| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| use t::APISIX 'no_plan'; |
| |
| log_level("info"); |
| repeat_each(1); |
| no_long_string(); |
| no_root_location(); |
| |
| |
| add_block_preprocessor(sub { |
| my ($block) = @_; |
| |
| if (!defined $block->request) { |
| $block->set_value("request", "GET /t"); |
| } |
| |
| my $user_yaml_config = <<_EOC_; |
| plugins: |
| - ai-proxy-multi |
| - prometheus |
| _EOC_ |
| $block->set_value("extra_yaml_config", $user_yaml_config); |
| }); |
| |
| run_tests(); |
| |
| __DATA__ |
| |
| === TEST 1: minimal viable configuration |
| --- config |
| location /t { |
| content_by_lua_block { |
| local plugin = require("apisix.plugins.ai-proxy-multi") |
| local ok, err = plugin.check_schema({ |
| instances = { |
| { |
| name = "openai-official", |
| provider = "openai", |
| options = { |
| model = "gpt-4", |
| }, |
| weight = 1, |
| auth = { |
| header = { |
| some_header = "some_value" |
| } |
| } |
| } |
| } |
| }) |
| |
| if not ok then |
| ngx.say(err) |
| else |
| ngx.say("passed") |
| end |
| } |
| } |
| --- response_body |
| passed |
| |
| |
| |
| === TEST 2: unsupported provider |
| --- config |
| location /t { |
| content_by_lua_block { |
| local plugin = require("apisix.plugins.ai-proxy-multi") |
| local ok, err = plugin.check_schema({ |
| instances = { |
| { |
| name = "self-hosted", |
| provider = "some-unique", |
| options = { |
| model = "gpt-4", |
| }, |
| weight = 1, |
| auth = { |
| header = { |
| some_header = "some_value" |
| } |
| } |
| } |
| } |
| }) |
| |
| if not ok then |
| ngx.say(err) |
| else |
| ngx.say("passed") |
| end |
| } |
| } |
| --- response_body eval |
| qr/.*property "provider" validation failed: matches none of the enum values*/ |
| |
| |
| |
| === TEST 3: set route with wrong auth header |
| --- config |
| location /t { |
| content_by_lua_block { |
| local t = require("lib.test_admin").test |
| local code, body = t('/apisix/admin/routes/1', |
| ngx.HTTP_PUT, |
| [[{ |
| "uri": "/anything", |
| "plugins": { |
| "ai-proxy-multi": { |
| "instances": [ |
| { |
| "name": "openai-official", |
| "provider": "openai", |
| "weight": 1, |
| "auth": { |
| "header": { |
| "Authorization": "Bearer wrongtoken" |
| } |
| }, |
| "options": { |
| "model": "gpt-4", |
| "max_tokens": 512, |
| "temperature": 1.0 |
| }, |
| "override": { |
| "endpoint": "http://127.0.0.1:1980" |
| } |
| } |
| ], |
| "ssl_verify": false |
| } |
| } |
| }]] |
| ) |
| |
| if code >= 300 then |
| ngx.status = code |
| end |
| ngx.say(body) |
| } |
| } |
| --- response_body |
| passed |
| |
| |
| |
| === TEST 4: send request (wrong auth should be rejected by upstream) |
| --- request |
| POST /anything |
| { "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] } |
| --- error_code: 401 |
| --- response_body |
| Unauthorized |
| |
| |
| |
| === TEST 5: set route with right auth header |
| --- config |
| location /t { |
| content_by_lua_block { |
| local t = require("lib.test_admin").test |
| local code, body = t('/apisix/admin/routes/1', |
| ngx.HTTP_PUT, |
| [[{ |
| "uri": "/anything", |
| "plugins": { |
| "ai-proxy-multi": { |
| "instances": [ |
| { |
| "name": "openai-official", |
| "provider": "openai", |
| "weight": 1, |
| "auth": { |
| "header": { |
| "Authorization": "Bearer token" |
| } |
| }, |
| "options": { |
| "model": "gpt-4", |
| "max_tokens": 512, |
| "temperature": 1.0 |
| }, |
| "override": { |
| "endpoint": "http://127.0.0.1:1980" |
| } |
| } |
| ], |
| "ssl_verify": false |
| } |
| } |
| }]] |
| ) |
| |
| if code >= 300 then |
| ngx.status = code |
| end |
| ngx.say(body) |
| } |
| } |
| --- response_body |
| passed |
| |
| |
| |
| === TEST 6: send request |
| --- request |
| POST /anything |
| { "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] } |
| --- more_headers |
| Authorization: Bearer token |
| X-AI-Fixture: openai/chat-basic.json |
| --- error_code: 200 |
| --- response_body eval |
| qr/\{ "content": "1 \+ 1 = 2\.", "role": "assistant" \}/ |
| |
| |
| |
| === TEST 7: send request with empty body |
| --- request |
| POST /anything |
| --- more_headers |
| Authorization: Bearer token |
| --- error_code: 400 |
| --- response_body_chomp |
| failed to get request body: request body is empty |
| |
| |
| |
| === TEST 8: send request with wrong method (GET) should work |
| --- request |
| GET /anything |
| { "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] } |
| --- more_headers |
| Authorization: Bearer token |
| X-AI-Fixture: openai/chat-basic.json |
| --- error_code: 200 |
| --- response_body eval |
| qr/\{ "content": "1 \+ 1 = 2\.", "role": "assistant" \}/ |
| |
| |
| |
| === TEST 9: wrong JSON in request body should give error |
| --- request |
| GET /anything |
| {}"messages": [ { "role": "system", "cont |
| --- error_code: 400 |
| --- response_body |
| {"message":"could not parse JSON request body: Expected the end but found T_STRING at character 3"} |
| |
| |
| |
| === TEST 10: content-type should be JSON |
| --- request |
| POST /anything |
| prompt%3Dwhat%2520is%25201%2520%252B%25201 |
| --- more_headers |
| Content-Type: application/x-www-form-urlencoded |
| --- error_code: 400 |
| --- response_body chomp |
| unsupported content-type: application/x-www-form-urlencoded, only application/json is supported |
| |
| |
| |
| === TEST 11: model options being merged to request body |
| --- config |
| location /t { |
| content_by_lua_block { |
| local t = require("lib.test_admin").test |
| local code, body = t('/apisix/admin/routes/1', |
| ngx.HTTP_PUT, |
| [[{ |
| "uri": "/anything", |
| "plugins": { |
| "ai-proxy-multi": { |
| "instances": [ |
| { |
| "name": "openai-official", |
| "provider": "openai", |
| "weight": 1, |
| "auth": { |
| "header": { |
| "Authorization": "Bearer token" |
| } |
| }, |
| "options": { |
| "model": "some-model", |
| "foo": "bar", |
| "temperature": 1.0 |
| }, |
| "override": { |
| "endpoint": "http://127.0.0.1:1980" |
| } |
| } |
| ], |
| "ssl_verify": false |
| } |
| } |
| }]] |
| ) |
| |
| if code >= 300 then |
| ngx.status = code |
| ngx.say(body) |
| return |
| end |
| |
| local code, body, actual_body = t("/anything", |
| ngx.HTTP_POST, |
| [[{ |
| "messages": [ |
| { "role": "system", "content": "You are a mathematician" }, |
| { "role": "user", "content": "What is 1+1?" } |
| ] |
| }]], |
| nil, |
| { |
| ["test-type"] = "options", |
| ["Content-Type"] = "application/json", |
| } |
| ) |
| |
| ngx.status = code |
| ngx.say(actual_body) |
| |
| } |
| } |
| --- error_code: 200 |
| --- response_body |
| options works |
| |
| |
| |
| === TEST 12: override path |
| --- config |
| location /t { |
| content_by_lua_block { |
| local t = require("lib.test_admin").test |
| local code, body = t('/apisix/admin/routes/1', |
| ngx.HTTP_PUT, |
| [[{ |
| "uri": "/anything", |
| "plugins": { |
| "ai-proxy-multi": { |
| "instances": [ |
| { |
| "name": "openai-official", |
| "provider": "openai", |
| "weight": 1, |
| "auth": { |
| "header": { |
| "Authorization": "Bearer token" |
| } |
| }, |
| "options": { |
| "model": "some-model", |
| "foo": "bar", |
| "temperature": 1.0 |
| }, |
| "override": { |
| "endpoint": "http://127.0.0.1:1980/random" |
| } |
| } |
| ], |
| "ssl_verify": false |
| } |
| } |
| }]] |
| ) |
| |
| if code >= 300 then |
| ngx.status = code |
| ngx.say(body) |
| return |
| end |
| |
| local code, body, actual_body = t("/anything", |
| ngx.HTTP_POST, |
| [[{ |
| "messages": [ |
| { "role": "system", "content": "You are a mathematician" }, |
| { "role": "user", "content": "What is 1+1?" } |
| ] |
| }]], |
| nil, |
| { |
| ["test-type"] = "path", |
| ["Content-Type"] = "application/json", |
| } |
| ) |
| |
| ngx.status = code |
| ngx.say(actual_body) |
| |
| } |
| } |
| --- response_body eval |
| qr/path override works/ |
| |
| |
| |
| === TEST 13: set route with stream = true (SSE) |
| --- config |
| location /t { |
| content_by_lua_block { |
| local t = require("lib.test_admin").test |
| local code, body = t('/apisix/admin/routes/1', |
| ngx.HTTP_PUT, |
| [[{ |
| "uri": "/anything", |
| "plugins": { |
| "ai-proxy-multi": { |
| "instances": [ |
| { |
| "name": "openai-official", |
| "provider": "openai", |
| "weight": 1, |
| "auth": { |
| "header": { |
| "Authorization": "Bearer token" |
| } |
| }, |
| "options": { |
| "model": "gpt-35-turbo-instruct", |
| "max_tokens": 512, |
| "temperature": 1.0, |
| "stream": true |
| }, |
| "override": { |
| "endpoint": "http://localhost:7737" |
| } |
| } |
| ], |
| "ssl_verify": false |
| } |
| } |
| }]] |
| ) |
| |
| if code >= 300 then |
| ngx.status = code |
| end |
| ngx.say(body) |
| } |
| } |
| --- response_body |
| passed |
| |
| |
| |
| === TEST 14: test is SSE works as expected |
| --- config |
| location /t { |
| content_by_lua_block { |
| local http = require("resty.http") |
| local httpc = http.new() |
| local core = require("apisix.core") |
| |
| local ok, err = httpc:connect({ |
| scheme = "http", |
| host = "localhost", |
| port = ngx.var.server_port, |
| }) |
| |
| if not ok then |
| ngx.status = 500 |
| ngx.say(err) |
| return |
| end |
| |
| local params = { |
| method = "POST", |
| headers = { |
| ["Content-Type"] = "application/json", |
| }, |
| path = "/anything", |
| body = [[{ |
| "messages": [ |
| { "role": "system", "content": "some content" } |
| ] |
| }]], |
| } |
| |
| local res, err = httpc:request(params) |
| if not res then |
| ngx.status = 500 |
| ngx.say(err) |
| return |
| end |
| |
| local final_res = {} |
| while true do |
| local chunk, err = res.body_reader() -- will read chunk by chunk |
| if err then |
| core.log.error("failed to read response chunk: ", err) |
| break |
| end |
| if not chunk then |
| break |
| end |
| core.table.insert_tail(final_res, chunk) |
| end |
| |
| ngx.print(#final_res .. final_res[6]) |
| } |
| } |
| --- response_body_eval |
| qr/6data: \[DONE\]\n\n/ |
| |
| |
| |
| === TEST 15: set route with wrong override endpoint |
| --- config |
| location /t { |
| content_by_lua_block { |
| local t = require("lib.test_admin").test |
| local code, body = t('/apisix/admin/routes/1', |
| ngx.HTTP_PUT, |
| [[{ |
| "uri": "/anything", |
| "plugins": { |
| "ai-proxy-multi": { |
| "instances": [ |
| { |
| "name": "openai-official", |
| "provider": "openai", |
| "weight": 1, |
| "auth": { |
| "header": { |
| "Authorization": "Bearer token" |
| } |
| }, |
| "options": { |
| "model": "gpt-4", |
| "max_tokens": 512, |
| "temperature": 1.0 |
| }, |
| "override": { |
| "endpoint": "http//127.0.0.1:1980" |
| } |
| } |
| ], |
| "ssl_verify": false |
| } |
| } |
| }]] |
| ) |
| |
| if code >= 300 then |
| ngx.status = code |
| end |
| ngx.say(body) |
| } |
| } |
| --- error_code: 400 |
| --- response_body eval |
| qr/.invalid endpoint.*/ |
