blob: 71dd1d62a9872e9c9fcb62602228ef8a05abc4f8 [file]
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
import json
import unittest
from types import MappingProxyType
from typing import get_type_hints
import pyflink.dataframe as pf
from pyflink.dataframe import llm
from pyflink.table import TableEnvironment
class ModelProviderTests(unittest.TestCase):
def test_triton_options_follow_community_option_names(self):
self.assertEqual(pf.TritonProvider("http://localhost:8000").to_options(),
{"endpoint": "http://localhost:8000"})
headers = {"X-Request": "comma, colon: quote'", "X-Empty": ""}
provider = pf.TritonProvider(
"http://localhost:8000", model_name="model", model_version="2", timeout="10 s",
flatten_batch_dim=False, priority=0, sequence_id="18446744073709551615",
sequence_start=False, sequence_end=True, compression="gzip", auth_token="token",
custom_headers=headers, max_retries=0, retry_initial_backoff="100 ms",
retry_max_backoff="1 s", default_value="null", health_check_enabled=True,
health_check_interval="5 s", circuit_breaker_enabled=False,
circuit_breaker_failure_threshold=0.5, circuit_breaker_timeout="30 s",
circuit_breaker_half_open_requests=2,
)
self.assertEqual(provider.provider_identifier(), "triton")
self.assertEqual(provider.model_option_key(), "model-name")
options = provider.to_options()
self.assertEqual(json.loads(options.pop("custom-headers")), headers)
expected = {
"endpoint": "http://localhost:8000", "model-name": "model", "model-version": "2",
"timeout": "10 s", "flatten-batch-dim": "false", "priority": "0",
"sequence-id": "18446744073709551615", "sequence-start": "false",
"sequence-end": "true", "compression": "gzip", "auth-token": "token",
"max-retries": "0", "retry-initial-backoff": "100 ms", "retry-max-backoff": "1 s",
"default-value": "null", "health-check-enabled": "true", "health-check-interval": "5 s",
"circuit-breaker-enabled": "false", "circuit-breaker-failure-threshold": "0.5",
"circuit-breaker-timeout": "30 s", "circuit-breaker-half-open-requests": "2",
}
self.assertEqual(options, expected)
headers["X-Request"] = "changed"
self.assertEqual(json.loads(provider.to_options()["custom-headers"]),
{"X-Request": "comma, colon: quote'", "X-Empty": ""})
provider.to_options().clear()
self.assertEqual(set(provider.to_options()), set(expected) | {"custom-headers"})
def test_triton_rejects_invalid_typed_options(self):
for options, error in [
({"model_name": 1}, TypeError),
({"timeout": 10}, TypeError),
({"flatten_batch_dim": "false"}, TypeError),
({"sequence_start": 1}, TypeError),
({"priority": True}, TypeError),
({"priority": 0.5}, TypeError),
({"priority": -1}, ValueError),
({"priority": 256}, ValueError),
({"max_retries": -1}, ValueError),
({"compression": "deflate"}, ValueError),
({"default_value": [0.0]}, TypeError),
({"custom_headers": [("X-Header", "value")]}, TypeError),
({"custom_headers": {1: "value"}}, TypeError),
({"custom_headers": {"X-Header": 1}}, TypeError),
({"health_check_enabled": 1}, TypeError),
({"circuit_breaker_enabled": "false"}, TypeError),
({"circuit_breaker_failure_threshold": 0.0}, ValueError),
({"circuit_breaker_failure_threshold": 1.1}, ValueError),
({"circuit_breaker_failure_threshold": True}, TypeError),
({"circuit_breaker_failure_threshold": float("inf")}, ValueError),
({"circuit_breaker_half_open_requests": 0}, ValueError),
({"retry_initial_backoff": 100}, TypeError),
({"extra_body": "{}"}, TypeError),
]:
with self.subTest(options=options):
with self.assertRaises(error):
pf.TritonProvider("http://localhost:8000", **options)
def test_openai_connection_options_allow_deferred_model_selection(self):
endpoint = "https://example.test/v1/embeddings"
provider = pf.OpenAIProvider(endpoint=endpoint, api_key="key")
self.assertEqual(provider.provider_identifier(), "openai")
self.assertEqual(provider.model_option_key(), "model")
self.assertEqual(provider.to_options(), {"endpoint": endpoint, "api-key": "key"})
self.assertEqual(
pf.OpenAIProvider(endpoint, "key", model="embed-model").to_options(),
{"endpoint": endpoint, "api-key": "key", "model": "embed-model"},
)
def test_openai_options_follow_community_option_names(self):
provider = pf.OpenAIProvider(
"https://example.test/v1/chat/completions", "key", model="chat-model",
system_prompt="", temperature=0.0, top_p=1.0, max_tokens=100,
stop="END,STOP", presence_penalty=-1.0, n=2, seed=0,
response_format="json_object", dimension=3, max_context_size=1000,
context_overflow_action="truncated-head-log", error_handling_strategy="IGNORE",
retry_num=0, retry_fallback_strategy="IGNORE",
)
expected = {
"endpoint": "https://example.test/v1/chat/completions", "api-key": "key",
"model": "chat-model", "system-prompt": "", "temperature": "0.0",
"top-p": "1.0", "max-tokens": "100", "stop": "END,STOP",
"presence-penalty": "-1.0", "n": "2", "seed": "0",
"response-format": "json_object", "dimension": "3", "max-context-size": "1000",
"context-overflow-action": "truncated-head-log", "error-handling-strategy": "IGNORE",
"retry-num": "0", "retry-fallback-strategy": "IGNORE",
}
self.assertEqual(provider.to_options(), expected)
provider.to_options().clear()
self.assertEqual(provider.to_options(), expected)
def test_openai_rejects_invalid_typed_options(self):
for overrides, error in [
({"endpoint": None}, TypeError),
({"api_key": ""}, ValueError),
({"model": 1}, TypeError),
({"system_prompt": 1}, TypeError),
({"max_tokens": True}, TypeError),
({"dimension": "3"}, TypeError),
({"temperature": True}, TypeError),
({"temperature": float("nan")}, ValueError),
({"temperature": float("-inf")}, ValueError),
({"seed": True}, TypeError),
({"top_p": -0.1}, ValueError),
({"top_p": 1.1}, ValueError),
({"presence_penalty": -3.0}, ValueError),
({"presence_penalty": 3.0}, ValueError),
({"retry_num": -1}, ValueError),
({"n": 0}, ValueError),
({"max_context_size": 0}, ValueError),
({"stop": ["STOP"]}, TypeError),
({"response_format": "json_schema"}, ValueError),
({"context_overflow_action": "unknown"}, ValueError),
({"error_handling_strategy": "unknown"}, ValueError),
({"retry_fallback_strategy": "RETRY"}, ValueError),
({"task": "chat/completions"}, TypeError),
]:
with self.subTest(overrides=overrides):
arguments = {"endpoint": "https://example.test/chat/completions", "api_key": "key"}
arguments.update(overrides)
with self.assertRaises(error):
pf.OpenAIProvider(**arguments)
def test_numeric_options_accept_boundaries_and_unbounded_values(self):
for top_p, presence_penalty in [(0, -2), (1, 2)]:
with self.subTest(top_p=top_p, presence_penalty=presence_penalty):
provider = pf.OpenAIProvider(
"https://example.test/chat/completions", "key",
temperature=-0.5, seed=-100, top_p=top_p, presence_penalty=presence_penalty,
max_tokens=1, n=1, dimension=1, max_context_size=1, retry_num=0,
)
self.assertEqual(provider.to_options(), {
"endpoint": "https://example.test/chat/completions", "api-key": "key",
"temperature": "-0.5", "seed": "-100", "top-p": str(top_p),
"presence-penalty": str(presence_penalty), "max-tokens": "1", "n": "1",
"dimension": "1", "max-context-size": "1", "retry-num": "0",
})
provider = pf.TritonProvider(
"http://localhost:8000", priority=255, max_retries=0,
circuit_breaker_failure_threshold=1, circuit_breaker_half_open_requests=1,
)
self.assertEqual(provider.to_options(), {
"endpoint": "http://localhost:8000", "priority": "255", "max-retries": "0",
"circuit-breaker-failure-threshold": "1", "circuit-breaker-half-open-requests": "1",
})
def test_generic_provider_preserves_raw_options(self):
expected = {"endpoint": "HTTPS://example.test/v1", "api-key": "key",
"custom_option": "", "enabled": "false", "identifier": "raw",
"self": "raw", "options": "raw"}
for form in ("keywords", "mapping", "mixed", "none"):
with self.subTest(form=form):
options = dict(expected)
if form == "keywords":
provider = pf.GenericProvider("custom", **options)
elif form == "mapping":
provider = pf.GenericProvider("custom", MappingProxyType(options))
elif form == "mixed":
provider = pf.GenericProvider(
"custom", {"endpoint": options.pop("endpoint")}, **options)
else:
provider = pf.GenericProvider("custom", None, **options)
options.clear()
self.assertIsInstance(provider, pf.ModelProvider)
self.assertEqual(provider.provider_identifier(), "custom")
self.assertEqual(provider.model_option_key(), "model")
self.assertEqual(provider.to_options(), expected)
provider.to_options().clear()
self.assertEqual(provider.to_options(), expected)
def test_typed_providers_accept_raw_options(self):
for provider_type, connection, raw, typed, expected in [
(pf.OpenAIProvider,
{"endpoint": "https://example.test/chat/completions", "api_key": "key"},
{"model": "chat-model", "max-tokens": "100", "temperature": "0.0",
"system-prompt": "", "future-option": "Mixed CASE"},
{"max_tokens": 100, "temperature": 0.0, "system_prompt": ""},
{"endpoint": "https://example.test/chat/completions", "api-key": "key",
"model": "chat-model", "max-tokens": "100", "temperature": "0.0",
"system-prompt": "", "future-option": "Mixed CASE"}),
(pf.TritonProvider,
{"endpoint": "http://localhost:8000"},
{"model-name": "classifier", "max-retries": "0", "flatten-batch-dim": "false",
"future_option": ""},
{"max_retries": 0, "flatten_batch_dim": False},
{"endpoint": "http://localhost:8000", "model-name": "classifier",
"max-retries": "0", "flatten-batch-dim": "false", "future_option": ""}),
]:
for parameters in ({}, typed):
with self.subTest(provider=provider_type.__name__, parameters=parameters):
options = dict(raw)
provider = provider_type(
**connection, **parameters, options=MappingProxyType(options))
options.clear()
self.assertEqual(provider.to_options(), expected)
provider.to_options().clear()
self.assertEqual(provider.to_options(), expected)
def test_public_contracts_have_resolvable_type_hints(self):
for api in (pf.set_model_provider, pf.set_default_model_provider, pf.list_model_providers,
pf.ModelProvider.provider_identifier, pf.ModelProvider.to_options,
pf.ModelProvider.model_option_key, pf.OpenAIProvider.__init__,
pf.TritonProvider.__init__, pf.GenericProvider.__init__):
with self.subTest(api=api.__qualname__):
self.assertTrue(get_type_hints(api))
def test_providers_reject_conflicting_options(self):
for construct, key, value in [
(lambda options: pf.OpenAIProvider(
"https://example.test/chat/completions", "key", max_tokens=10, options=options),
"max-tokens", "10"),
(lambda options: pf.OpenAIProvider(
"https://example.test/chat/completions", "key", options=options),
"api-key", "key"),
(lambda options: pf.OpenAIProvider(
"https://example.test/chat/completions", "key", system_prompt="", options=options),
"system-prompt", ""),
(lambda options: pf.TritonProvider(
"http://localhost:8000", flatten_batch_dim=False, options=options),
"flatten-batch-dim", "false"),
(lambda options: pf.TritonProvider(
"http://localhost:8000", max_retries=0, options=options),
"max-retries", "0"),
(lambda options: pf.GenericProvider("custom", options, timeout="10 s"),
"timeout", "10 s"),
]:
with self.subTest(key=key):
options = {key: value}
self.assertEqual(construct(options).to_options()[key], value)
options[key] = "different"
with self.assertRaisesRegex(ValueError, "conflicting values for option"):
construct(options)
self.assertEqual(options, {key: "different"})
def test_generic_provider_rejects_invalid_configuration(self):
for identifier, options, error in [
(1, {}, TypeError),
("", {}, ValueError),
(" ", {}, ValueError),
("custom", {"enabled": True}, TypeError),
("custom", {"retries": 1}, TypeError),
("custom", {"endpoint": None}, TypeError),
]:
with self.subTest(identifier=identifier, options=options):
with self.assertRaises(error):
pf.GenericProvider(identifier, **options)
def test_providers_reject_invalid_raw_options(self):
for name, construct, expected in (
("openai", lambda options: pf.OpenAIProvider(
"https://example.test/chat/completions", "key", options=options),
{"endpoint": "https://example.test/chat/completions", "api-key": "key"}),
("triton", lambda options: pf.TritonProvider("http://localhost:8000", options=options),
{"endpoint": "http://localhost:8000"}),
("generic", lambda options: pf.GenericProvider("custom", options), {}),
):
for options in ([], "options", False, {1: "value"}, {"option": None},
{"option": False}, {"option": 1}):
with self.subTest(provider=name, options=options):
with self.assertRaises(TypeError):
construct(options)
self.assertEqual(construct({}).to_options(), expected)
with self.assertRaises(TypeError):
pf.OpenAIProvider("https://example.test/chat/completions", "key",
max_tokens=True, options={"max-tokens": "1"})
with self.assertRaises(ValueError):
pf.TritonProvider("http://localhost:8000", priority=256, options={"priority": "1"})
class ModelProviderRegistryTests(unittest.TestCase):
def setUp(self):
previous = dict(llm._provider_registry)
self.addCleanup(llm._provider_registry.update, previous)
self.addCleanup(llm._provider_registry.clear)
llm._provider_registry.clear()
previous_default = llm._default_provider
self.addCleanup(setattr, llm, "_default_provider", previous_default)
llm._default_provider = None
def test_registration_is_lazy_and_names_are_ordered(self):
class UnserializedProvider(pf.ModelProvider):
def provider_identifier(self):
raise AssertionError("Registration must not inspect the provider")
def to_options(self):
raise AssertionError("Registration must not serialize the provider")
pf.set_model_provider(name="chat", provider=UnserializedProvider())
pf.set_model_provider("embed", provider=pf.GenericProvider("custom"))
names = pf.list_model_providers()
self.assertEqual(names, ["chat", "embed"])
names.clear()
self.assertEqual(pf.list_model_providers(), ["chat", "embed"])
def test_single_provider_registration_derives_names_without_serializing(self):
class UnserializedProvider(pf.ModelProvider):
def provider_identifier(self):
return "custom"
def to_options(self):
raise AssertionError("Registration must not serialize the provider")
def model_option_key(self):
raise AssertionError("Registration must not inspect model options")
provider = UnserializedProvider()
pf.set_model_provider(provider)
self.assertEqual(pf.list_model_providers(), ["custom"])
self.assertIs(llm._resolve_provider(), provider)
pf.set_model_provider(provider=pf.OpenAIProvider("https://example.test/embeddings", "key"))
with self.assertRaisesRegex(ValueError, "Multiple model providers"):
llm._resolve_provider()
pf.set_default_model_provider("custom")
pf.set_model_provider(pf.TritonProvider("http://localhost:8000"))
self.assertEqual(pf.list_model_providers(), ["custom", "openai", "triton"])
self.assertIs(llm._resolve_provider(), provider)
def test_invalid_registration_preserves_registered_names(self):
class InvalidIdentifierProvider(pf.ModelProvider):
def __init__(self, identifier):
self.identifier = identifier
def provider_identifier(self):
if isinstance(self.identifier, RuntimeError):
raise self.identifier
return self.identifier
def to_options(self):
raise AssertionError("Registration must not serialize the provider")
provider = pf.GenericProvider("custom")
other = pf.GenericProvider("other")
pf.set_model_provider("chat", provider)
pf.set_model_provider("embed", other)
pf.set_default_model_provider("chat")
invalid_calls = [
(("", provider), {}, ValueError),
((" ", provider), {}, ValueError),
((1, provider), {}, TypeError),
((None, provider), {}, TypeError),
(("chat", None), {}, TypeError),
(("embed", "custom"), {}, TypeError),
((), {}, TypeError),
(("chat",), {}, TypeError),
((), {"provider": None}, TypeError),
((), {"name": provider}, TypeError),
((provider, provider), {}, TypeError),
((provider, None), {}, TypeError),
(("chat", provider, provider), {}, TypeError),
(("chat",), {"name": "embed", "provider": provider}, TypeError),
(("chat", provider), {"provider": other}, TypeError),
(("chat", provider), {"endpoint": "https://example.test"}, TypeError),
]
for identifier, error in [
("", ValueError), (" ", ValueError), (1, TypeError), (None, TypeError),
(RuntimeError("Identifier unavailable"), RuntimeError),
]:
invalid_calls.append(((), {"provider": InvalidIdentifierProvider(identifier)}, error))
for args, kwargs, error in invalid_calls:
with self.subTest(args=args, kwargs=kwargs):
with self.assertRaises(error):
pf.set_model_provider(*args, **kwargs)
self.assertEqual(pf.list_model_providers(), ["chat", "embed"])
self.assertIs(llm._resolve_provider(), provider)
self.assertIs(llm._resolve_provider("chat"), provider)
self.assertIs(llm._resolve_provider("embed"), other)
def test_replacement_preserves_order_and_defaults_bound_to_names(self):
original = pf.GenericProvider("custom", token="old")
pf.set_model_provider(original)
replacement = pf.GenericProvider("custom", token="new")
pf.set_model_provider(replacement)
self.assertEqual(pf.list_model_providers(), ["custom"])
self.assertIs(llm._resolve_provider(), replacement)
pf.set_model_provider("other", pf.GenericProvider("other"))
pf.set_default_model_provider("custom")
named = pf.GenericProvider("different-factory")
pf.set_model_provider(name="custom", provider=named)
self.assertEqual(pf.list_model_providers(), ["custom", "other"])
self.assertIs(llm._resolve_provider(), named)
pf.set_model_provider(provider=original)
other = pf.GenericProvider("another-factory")
pf.set_model_provider("other", other)
self.assertEqual(pf.list_model_providers(), ["custom", "other"])
self.assertIs(llm._resolve_provider(), original)
self.assertIs(llm._resolve_provider("other"), other)
def test_selection_requires_choice_only_with_multiple_providers(self):
chat = pf.GenericProvider("custom", task="chat")
embed = pf.GenericProvider("custom", task="embed")
with self.assertRaisesRegex(ValueError, "No model provider"):
llm._resolve_provider()
pf.set_model_provider("chat", chat)
self.assertIs(llm._resolve_provider(), chat)
pf.set_model_provider("embed", embed)
with self.assertRaisesRegex(ValueError, "Multiple model providers"):
llm._resolve_provider()
pf.set_default_model_provider("chat")
pf.set_model_provider("other", pf.GenericProvider("other"))
self.assertIs(llm._resolve_provider(), chat)
self.assertIs(llm._resolve_provider("embed"), embed)
def test_invalid_selection_preserves_default(self):
provider = pf.GenericProvider("custom")
pf.set_model_provider("chat", provider)
pf.set_default_model_provider("chat")
for name, error in [("unknown", ValueError), ("", ValueError),
(" ", ValueError), (1, TypeError), (None, TypeError)]:
with self.subTest(name=name):
with self.assertRaises(error):
pf.set_default_model_provider(name)
self.assertIs(llm._resolve_provider(), provider)
if name is not None:
with self.assertRaises(error):
llm._resolve_provider(name)
def test_registry_and_default_survive_environment_replacement(self):
previous_environment = pf.get_table_environment()
buffered = dict(pf.config._buffered)
self.addCleanup(pf.config._buffered.update, buffered)
self.addCleanup(pf.config._buffered.clear)
self.addCleanup(pf.set_table_environment, previous_environment)
pf.config._buffered.clear()
provider = pf.GenericProvider("custom")
pf.set_model_provider("chat", provider)
pf.set_model_provider("embed", pf.GenericProvider("custom"))
pf.set_default_model_provider("chat")
for environment in (object.__new__(TableEnvironment), None,
object.__new__(TableEnvironment)):
pf.set_table_environment(environment)
self.assertEqual(pf.list_model_providers(), ["chat", "embed"])
self.assertIs(llm._resolve_provider(), provider)
if __name__ == '__main__':
unittest.main()