http: recognize QUERY as a well-known method QUERY (draft-ietf-httpbis-safe-method-w-body) was handled as an extension method, so it was tunneled, could not be named in an ip_allow.yaml or remap.config ACL, and was not counted separately. Register it in the well-known token table and give it the semantics the draft defines: safe, idempotent, and carrying request content, so over HTTP/1.1 it needs a Content-Length or Transfer-Encoding: chunked like POST and PUT do. QUERY is appended after PUSH in the method block so existing HTTP_WKSIDX_* values, and the ip_allow method bitmask derived from them, are unchanged. QUERY is kept off the retry-after-bytes-sent path that GET and HEAD use: its content is not kept for replay on that path, so once any of the request has been sent it is treated like POST there. A buffered QUERY can still be retried when an HTTP/2 origin reports the request was not processed. Adding QUERY grows the OPTIONS Allow value, which exposed that the value was appended through the string_view overload with a char * and no length, so it was measured with strlen on an unterminated buffer and could pick up trailing garbage. Pass the length explicitly. Caching of QUERY responses is not addressed here.
diff --git a/doc/admin-guide/monitoring/statistics/core/http-request-method.en.rst b/doc/admin-guide/monitoring/statistics/core/http-request-method.en.rst index 79c787d..909a5ab 100644 --- a/doc/admin-guide/monitoring/statistics/core/http-request-method.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/http-request-method.en.rst
@@ -86,6 +86,12 @@ Represents the total number of HTTP :literal:`PUT` requests received by the |TS| instance since statistics collection began. +.. ts:stat:: global proxy.process.http.query_requests integer + :type: counter + + Represents the total number of HTTP :literal:`QUERY` requests received by + the |TS| instance since statistics collection began. + .. ts:stat:: global proxy.process.http.trace_requests integer :type: counter
diff --git a/doc/release-notes/whats-new.en.rst b/doc/release-notes/whats-new.en.rst index 2c3da9f..d6bfd22 100644 --- a/doc/release-notes/whats-new.en.rst +++ b/doc/release-notes/whats-new.en.rst
@@ -35,6 +35,7 @@ * Added ``proxy.process.ssl.group.user_agent.P-224`` * Added ``proxy.process.ssl.group.user_agent.X448`` * Added ``proxy.process.ssl.group.user_agent.X25519MLKEM768`` +* Added ``proxy.process.http.query_requests`` Plugins ------- @@ -107,6 +108,13 @@ * Add an optional ``avg_obj_size`` to ``storage.yaml`` to control the directory entry sizing. * The ``proxy.config.http.cache.post_method`` is now an overridable config. +* The HTTP ``QUERY`` method + (`draft-ietf-httpbis-safe-method-w-body <https://datatracker.ietf.org/doc/draft-ietf-httpbis-safe-method-w-body/>`_) + is now a well-known method. It can be named in :file:`ip_allow.yaml` and + :file:`remap.config` ACLs, it is reported by + ``proxy.process.http.query_requests``, and like the other content-carrying + methods an HTTP/1.1 ``QUERY`` requires a ``Content-Length`` or + ``Transfer-Encoding: chunked``. * Defer deleting the copied plugin shared object file to startup to make it easier to debug crashes in plugins.
diff --git a/include/proxy/hdrs/HTTP.h b/include/proxy/hdrs/HTTP.h index 2539e4d..4e16bab 100644 --- a/include/proxy/hdrs/HTTP.h +++ b/include/proxy/hdrs/HTTP.h
@@ -363,6 +363,7 @@ extern c_str_view HTTP_METHOD_PUT; extern c_str_view HTTP_METHOD_TRACE; extern c_str_view HTTP_METHOD_PUSH; +extern c_str_view HTTP_METHOD_QUERY; extern int HTTP_WKSIDX_CONNECT; extern int HTTP_WKSIDX_DELETE; @@ -374,6 +375,7 @@ extern int HTTP_WKSIDX_PUT; extern int HTTP_WKSIDX_TRACE; extern int HTTP_WKSIDX_PUSH; +extern int HTTP_WKSIDX_QUERY; extern int HTTP_WKSIDX_METHODS_CNT; extern c_str_view HTTP_VALUE_BYTES;
diff --git a/include/proxy/http/HttpConfig.h b/include/proxy/http/HttpConfig.h index 2a52cd7..0807447 100644 --- a/include/proxy/http/HttpConfig.h +++ b/include/proxy/http/HttpConfig.h
@@ -248,6 +248,7 @@ Metrics::Counter::AtomicType *proxy_loop_detected; Metrics::Counter::AtomicType *proxy_mh_loop_detected; Metrics::Counter::AtomicType *purge_requests; + Metrics::Counter::AtomicType *query_requests; Metrics::Counter::AtomicType *push_requests; Metrics::Counter::AtomicType *pushed_document_total_size; Metrics::Counter::AtomicType *pushed_response_header_total_size;
diff --git a/include/proxy/http/HttpTransactHeaders.h b/include/proxy/http/HttpTransactHeaders.h index e2f3f47..2c33ff8 100644 --- a/include/proxy/http/HttpTransactHeaders.h +++ b/include/proxy/http/HttpTransactHeaders.h
@@ -104,7 +104,7 @@ return ((method == HTTP_WKSIDX_GET) || (method == HTTP_WKSIDX_POST) || (method == HTTP_WKSIDX_CONNECT) || (method == HTTP_WKSIDX_DELETE) || (method == HTTP_WKSIDX_PURGE) || (method == HTTP_WKSIDX_HEAD) || (method == HTTP_WKSIDX_OPTIONS) || (method == HTTP_WKSIDX_PUT) || (method == HTTP_WKSIDX_PUSH) || - (method == HTTP_WKSIDX_TRACE)); + (method == HTTP_WKSIDX_QUERY) || (method == HTTP_WKSIDX_TRACE)); } inline int
diff --git a/src/proxy/hdrs/HTTP.cc b/src/proxy/hdrs/HTTP.cc index 9eb0668..6b372e8 100644 --- a/src/proxy/hdrs/HTTP.cc +++ b/src/proxy/hdrs/HTTP.cc
@@ -59,6 +59,7 @@ c_str_view HTTP_METHOD_PUT; c_str_view HTTP_METHOD_TRACE; c_str_view HTTP_METHOD_PUSH; +c_str_view HTTP_METHOD_QUERY; int HTTP_WKSIDX_CONNECT; int HTTP_WKSIDX_DELETE; @@ -70,6 +71,7 @@ int HTTP_WKSIDX_PUT; int HTTP_WKSIDX_TRACE; int HTTP_WKSIDX_PUSH; +int HTTP_WKSIDX_QUERY; int HTTP_WKSIDX_METHODS_CNT = 0; c_str_view HTTP_VALUE_BYTES; @@ -154,6 +156,7 @@ HTTP_METHOD_PUT = hdrtoken_string_to_wks_sv("PUT"); HTTP_METHOD_TRACE = hdrtoken_string_to_wks_sv("TRACE"); HTTP_METHOD_PUSH = hdrtoken_string_to_wks_sv("PUSH"); + HTTP_METHOD_QUERY = hdrtoken_string_to_wks_sv("QUERY"); // HTTP methods index calculation. Don't forget to count them! // Don't change the order of calculation! Each index has related bitmask (see http quick filter) @@ -177,6 +180,8 @@ HTTP_WKSIDX_METHODS_CNT++; HTTP_WKSIDX_PUSH = hdrtoken_wks_to_index(HTTP_METHOD_PUSH.c_str()); HTTP_WKSIDX_METHODS_CNT++; + HTTP_WKSIDX_QUERY = hdrtoken_wks_to_index(HTTP_METHOD_QUERY.c_str()); + HTTP_WKSIDX_METHODS_CNT++; HTTP_VALUE_BYTES = hdrtoken_string_to_wks_sv("bytes"); HTTP_VALUE_CHUNKED = hdrtoken_string_to_wks_sv("chunked");
diff --git a/src/proxy/hdrs/HdrToken.cc b/src/proxy/hdrs/HdrToken.cc index d167fb4..d72c1f1 100644 --- a/src/proxy/hdrs/HdrToken.cc +++ b/src/proxy/hdrs/HdrToken.cc
@@ -105,7 +105,7 @@ "rtsp", "mmsu", "mmst", "mms", "wss", "ws", // HTTP methods - "CONNECT", "DELETE", "GET", "POST", "HEAD", "OPTIONS", "PURGE", "PUT", "TRACE", "PUSH", + "CONNECT", "DELETE", "GET", "POST", "HEAD", "OPTIONS", "PURGE", "PUT", "TRACE", "PUSH", "QUERY", // Header extensions "X-ID", "X-Forwarded-For", "TE", "Strict-Transport-Security", "100-continue", @@ -160,6 +160,7 @@ {"PUT", HdrTokenType::METHOD }, {"TRACE", HdrTokenType::METHOD }, {"PUSH", HdrTokenType::METHOD }, + {"QUERY", HdrTokenType::METHOD }, {"max-age", HdrTokenType::CACHE_CONTROL }, {"max-stale", HdrTokenType::CACHE_CONTROL },
diff --git a/src/proxy/http/HttpConfig.cc b/src/proxy/http/HttpConfig.cc index 7977cd1..ca15807 100644 --- a/src/proxy/http/HttpConfig.cc +++ b/src/proxy/http/HttpConfig.cc
@@ -402,6 +402,7 @@ http_rsb.proxy_loop_detected = Metrics::Counter::createPtr("proxy.process.http.http_proxy_loop_detected"); http_rsb.proxy_mh_loop_detected = Metrics::Counter::createPtr("proxy.process.http.http_proxy_mh_loop_detected"); http_rsb.purge_requests = Metrics::Counter::createPtr("proxy.process.http.purge_requests"); + http_rsb.query_requests = Metrics::Counter::createPtr("proxy.process.http.query_requests"); http_rsb.push_requests = Metrics::Counter::createPtr("proxy.process.http.push_requests"); http_rsb.pushed_document_total_size = Metrics::Counter::createPtr("proxy.process.http.pushed_document_total_size"); http_rsb.pushed_response_header_total_size = Metrics::Counter::createPtr("proxy.process.http.pushed_response_header_total_size");
diff --git a/src/proxy/http/HttpDebugNames.cc b/src/proxy/http/HttpDebugNames.cc index 66d2bfc..c017be8 100644 --- a/src/proxy/http/HttpDebugNames.cc +++ b/src/proxy/http/HttpDebugNames.cc
@@ -85,6 +85,8 @@ return ("HTTP_METHOD_POST"); } else if (method == HTTP_METHOD_PURGE.c_str()) { return ("HTTP_METHOD_PURGE"); + } else if (method == HTTP_METHOD_QUERY.c_str()) { + return ("HTTP_METHOD_QUERY"); } else if (method == HTTP_METHOD_PUT.c_str()) { return ("HTTP_METHOD_PUT"); } else if (method == HTTP_METHOD_TRACE.c_str()) {
diff --git a/src/proxy/http/HttpSM.cc b/src/proxy/http/HttpSM.cc index 454a015..07b6ef5 100644 --- a/src/proxy/http/HttpSM.cc +++ b/src/proxy/http/HttpSM.cc
@@ -785,7 +785,8 @@ if (t_state.hdr_info.client_request.version_get() == HTTP_1_1 && (t_state.hdr_info.client_request.method_get_wksidx() == HTTP_WKSIDX_POST || - t_state.hdr_info.client_request.method_get_wksidx() == HTTP_WKSIDX_PUT)) { + t_state.hdr_info.client_request.method_get_wksidx() == HTTP_WKSIDX_PUT || + t_state.hdr_info.client_request.method_get_wksidx() == HTTP_WKSIDX_QUERY)) { auto expect{t_state.hdr_info.client_request.value_get(static_cast<std::string_view>(MIME_FIELD_EXPECT))}; if (ts::iequals(expect, static_cast<std::string_view>(HTTP_VALUE_100_CONTINUE))) { // When receive an "Expect: 100-continue" request from client, ATS sends a "100 Continue" response to client @@ -5891,7 +5892,7 @@ will_be_private_ss = true; } - if (t_state.method == HTTP_WKSIDX_POST || t_state.method == HTTP_WKSIDX_PUT) { + if (t_state.method == HTTP_WKSIDX_POST || t_state.method == HTTP_WKSIDX_PUT || t_state.method == HTTP_WKSIDX_QUERY) { // don't share the session if keep-alive for post is not on if (t_state.txn_conf->keep_alive_post_out == 0) { SMDbg(dbg_ctl_http_ss, "Setting server session to private because of keep-alive post out");
diff --git a/src/proxy/http/HttpTransact.cc b/src/proxy/http/HttpTransact.cc index 442ce7d..63538b7 100644 --- a/src/proxy/http/HttpTransact.cc +++ b/src/proxy/http/HttpTransact.cc
@@ -5819,9 +5819,9 @@ return RequestError_t::UNACCEPTABLE_TE_REQUIRED; } - // Require Content-Length/Transfer-Encoding for POST/PUSH/PUT + // Require Content-Length/Transfer-Encoding for POST/PUSH/PUT/QUERY if ((scheme == URL_WKSIDX_HTTP || scheme == URL_WKSIDX_HTTPS) && - (method == HTTP_WKSIDX_POST || method == HTTP_WKSIDX_PUSH || method == HTTP_WKSIDX_PUT) && + (method == HTTP_WKSIDX_POST || method == HTTP_WKSIDX_PUSH || method == HTTP_WKSIDX_PUT || method == HTTP_WKSIDX_QUERY) && s->client_info.transfer_encoding != TransferEncoding_t::CHUNKED) { // In normal operation there will always be a get_ua_txn() at this point, but in one of the -R1 regression tests a request is // createdindependent of a transaction and this method is called, so we must null check @@ -6204,6 +6204,8 @@ Metrics::Counter::increment(http_rsb.delete_requests); } else if (method == HTTP_WKSIDX_PURGE) { Metrics::Counter::increment(http_rsb.purge_requests); + } else if (method == HTTP_WKSIDX_QUERY) { + Metrics::Counter::increment(http_rsb.query_requests); } else if (method == HTTP_WKSIDX_TRACE) { Metrics::Counter::increment(http_rsb.trace_requests); } else if (method == HTTP_WKSIDX_PUSH) { @@ -6945,11 +6947,15 @@ return false; } - // If safe requests are retryable, it should be safe to retry safe requests irrespective of bytes sent or connection state - // according to RFC the following methods are safe (https://tools.ietf.org/html/rfc7231#section-4.2.1) + // Safe methods (RFC 9110, section 9.2.1) can be retried irrespective of bytes sent or connection state. QUERY + // (draft-ietf-httpbis-safe-method-w-body) is safe too, but its content is not kept for replay on this path, so once + // any part of its request has been sent it is treated like an unsafe method, whether or not the content itself went + // out. A buffered QUERY can still be retried when the origin reports it was not processed, see + // HttpSM::prepare_for_origin_retry. // Otherwise, if there was no error establishing the connection (and we sent bytes)-- we cannot retry - if (!HttpTransactHeaders::is_method_safe(s->method) && s->current.state != CONNECTION_ERROR && - s->state_machine->server_request_hdr_bytes > 0) { + bool const retry_only_before_sending = !HttpTransactHeaders::is_method_safe(s->method) || s->method == HTTP_WKSIDX_QUERY; + + if (retry_only_before_sending && s->current.state != CONNECTION_ERROR && s->state_machine->server_request_hdr_bytes > 0) { return false; }
diff --git a/src/proxy/http/HttpTransactHeaders.cc b/src/proxy/http/HttpTransactHeaders.cc index e688f08..409c7ba 100644 --- a/src/proxy/http/HttpTransactHeaders.cc +++ b/src/proxy/http/HttpTransactHeaders.cc
@@ -100,8 +100,9 @@ bool HttpTransactHeaders::is_method_safe(int method) { - // See RFC 7231, section 4.2.1. - return (method == HTTP_WKSIDX_GET || method == HTTP_WKSIDX_OPTIONS || method == HTTP_WKSIDX_HEAD || method == HTTP_WKSIDX_TRACE); + // See RFC 9110, section 9.2.1, and draft-ietf-httpbis-safe-method-w-body for QUERY. + return (method == HTTP_WKSIDX_GET || method == HTTP_WKSIDX_OPTIONS || method == HTTP_WKSIDX_HEAD || method == HTTP_WKSIDX_QUERY || + method == HTTP_WKSIDX_TRACE); } bool @@ -115,7 +116,8 @@ HttpTransactHeaders::is_method_idempotent(int method) { return (method == HTTP_WKSIDX_CONNECT || method == HTTP_WKSIDX_DELETE || method == HTTP_WKSIDX_GET || - method == HTTP_WKSIDX_HEAD || method == HTTP_WKSIDX_PUT || method == HTTP_WKSIDX_OPTIONS || method == HTTP_WKSIDX_TRACE); + method == HTTP_WKSIDX_HEAD || method == HTTP_WKSIDX_PUT || method == HTTP_WKSIDX_OPTIONS || method == HTTP_WKSIDX_QUERY || + method == HTTP_WKSIDX_TRACE); } void @@ -125,9 +127,9 @@ const char *methods[] = { HTTP_METHOD_CONNECT.c_str(), HTTP_METHOD_DELETE.c_str(), HTTP_METHOD_GET.c_str(), HTTP_METHOD_HEAD.c_str(), HTTP_METHOD_OPTIONS.c_str(), HTTP_METHOD_POST.c_str(), HTTP_METHOD_PURGE.c_str(), HTTP_METHOD_PUT.c_str(), - HTTP_METHOD_PUSH.c_str(), HTTP_METHOD_TRACE.c_str(), + HTTP_METHOD_PUSH.c_str(), HTTP_METHOD_QUERY.c_str(), HTTP_METHOD_TRACE.c_str(), }; - char inline_buffer[64]; + char inline_buffer[128]; char *alloced_buffer, *value_buffer; int nmethods = sizeof(methods) / sizeof(methods[0]); @@ -190,7 +192,7 @@ // FIXME: do we really want to append to old list, or replace old list? // step 5: attach new allow list to end of previous list - field->value_append(response->m_heap, response->m_mime, value_buffer, bytes); + field->value_append(response->m_heap, response->m_mime, std::string_view{value_buffer, bytes}); // step 6: free up temp storage ats_free(alloced_buffer);
diff --git a/src/proxy/http2/Http2ConnectionState.cc b/src/proxy/http2/Http2ConnectionState.cc index 1fcae8a..b822d3e 100644 --- a/src/proxy/http2/Http2ConnectionState.cc +++ b/src/proxy/http2/Http2ConnectionState.cc
@@ -2714,7 +2714,8 @@ // explicit zero length and no end stream flag, causing the request to // fail. We emulate chromium behaviour here prevent such RSTs. Transfer-encoding // implies there’s a body, regardless of whether it is chunked or not. - bool content_method = method == HTTP_WKSIDX_POST || method == HTTP_WKSIDX_PUSH || method == HTTP_WKSIDX_PUT; + bool content_method = + method == HTTP_WKSIDX_POST || method == HTTP_WKSIDX_PUSH || method == HTTP_WKSIDX_PUT || method == HTTP_WKSIDX_QUERY; bool is_transfer_encoded = send_hdr->presence(MIME_PRESENCE_TRANSFER_ENCODING); bool has_content_header = send_hdr->presence(MIME_PRESENCE_CONTENT_LENGTH); bool explicit_zero_length = has_content_header && send_hdr->get_content_length() == 0;
diff --git a/tests/gold_tests/autest-site/ats_replay.test.ext b/tests/gold_tests/autest-site/ats_replay.test.ext index 205fac3..2fe6370 100644 --- a/tests/gold_tests/autest-site/ats_replay.test.ext +++ b/tests/gold_tests/autest-site/ats_replay.test.ext
@@ -131,8 +131,8 @@ # Configure sni.yaml if specified. sni_yaml = ats_config.get('sni_yaml') - if sni_yaml != None: - ts.Disk.sni_yaml.AddLines(yaml.dump(sni_yaml).split('\n')) + if sni_yaml is not None: + ts.Disk.sni_yaml.AddLines(yaml.safe_dump(sni_yaml).splitlines()) # Configure plugin_config if specified. plugin_config = ats_config.get('plugin_config', []) @@ -159,13 +159,18 @@ # Configure logging.yaml if specified. logging_yaml = ats_config.get('logging_yaml') - if logging_yaml != None: - ts.Disk.logging_yaml.AddLines(yaml.dump(logging_yaml).split('\n')) + if logging_yaml is not None: + ts.Disk.logging_yaml.AddLines(yaml.safe_dump(logging_yaml).splitlines()) + + # Configure ip_allow.yaml if specified. + ip_allow_yaml = ats_config.get('ip_allow_yaml') + if ip_allow_yaml is not None: + ts.Disk.ip_allow_yaml.AddLines(yaml.safe_dump(ip_allow_yaml).splitlines()) # Configure storage.yaml if specified. storage_yaml = ats_config.get('storage_yaml') if storage_yaml is not None: - ts.Disk.storage_yaml.AddLines(yaml.dump(storage_yaml).split('\n')) + ts.Disk.storage_yaml.AddLines(yaml.safe_dump(storage_yaml).splitlines()) remap_config = ats_config.get('remap_config', []) for remap_entry in remap_config:
diff --git a/tests/gold_tests/query/query-ip-allow.test.py b/tests/gold_tests/query/query-ip-allow.test.py new file mode 100644 index 0000000..bff99cb --- /dev/null +++ b/tests/gold_tests/query/query-ip-allow.test.py
@@ -0,0 +1,25 @@ +''' +Verify QUERY can be denied by an ip_allow.yaml ACL. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify QUERY is recognized as a well-known method by the ip_allow.yaml ACL +machinery. +''' + +Test.ATSReplayTest(replay_file="replay/query-ip-allow.replay.yaml")
diff --git a/tests/gold_tests/query/query-pass-through.test.py b/tests/gold_tests/query/query-pass-through.test.py new file mode 100644 index 0000000..d434126 --- /dev/null +++ b/tests/gold_tests/query/query-pass-through.test.py
@@ -0,0 +1,24 @@ +''' +Verify ATS forwards the HTTP QUERY method and its content. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify QUERY requests are proxied with their content. +''' + +Test.ATSReplayTest(replay_file="replay/query-pass-through.replay.yaml")
diff --git a/tests/gold_tests/query/query-remap-acl.test.py b/tests/gold_tests/query/query-remap-acl.test.py new file mode 100644 index 0000000..9c4c192 --- /dev/null +++ b/tests/gold_tests/query/query-remap-acl.test.py
@@ -0,0 +1,25 @@ +''' +Verify QUERY can be denied by a remap.config ACL. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify QUERY is recognized as a well-known method by the remap.config ACL +machinery. +''' + +Test.ATSReplayTest(replay_file="replay/query-remap-acl.replay.yaml")
diff --git a/tests/gold_tests/query/query-retry.test.py b/tests/gold_tests/query/query-retry.test.py new file mode 100644 index 0000000..9cf0644 --- /dev/null +++ b/tests/gold_tests/query/query-retry.test.py
@@ -0,0 +1,81 @@ +''' +Verify QUERY is not retried once its request, content included, has been sent +to an origin that then closes the connection, whether or not request buffering +is enabled. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import shlex +import sys + +from ports import get_port + +Test.Summary = ''' +Verify QUERY is not retried once its request, content included, has been sent +to an origin that then closes the connection, whether or not request buffering +is enabled. +''' + +Test.ContinueOnFail = True + +BODY = 'SELECT * FROM t WHERE a' + + +def add_scenario(name: str, method: str, buffering: bool) -> None: + tr = Test.AddTestRun(f'{method} after the origin closes, request buffering {"enabled" if buffering else "disabled"}') + tr.Setup.Copy('query_retry_origin.py') + + server = Test.Processes.Process(f'origin-{name}') + server_port = get_port(server, 'http_port') + server.Command = f'{sys.executable} {tr.RunDirectory}/query_retry_origin.py {server_port}' + server.Ready = When.PortOpen(server_port) + + ts = Test.MakeATSProcess(f'ts-{name}') + ts.Disk.records_config.update( + { + 'proxy.config.http.request_buffer_enabled': 1 if buffering else 0, + 'proxy.config.http.connect_attempts_max_retries': 3, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http_trans', + }) + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{server_port}/') + + content = f"-H 'Content-Type: application/sql' --data-binary {shlex.quote(BODY)} " if method == 'QUERY' else '' + tr.Processes.Default.StartBefore(server) + tr.Processes.Default.StartBefore(ts) + tr.Processes.Default.Command = ( + f"curl -s --max-time 30 -o /dev/null -w '%{{http_code}}\\n' -X {method} {content}http://127.0.0.1:{ts.Variables.port}/search" + ) + tr.Processes.Default.ReturnCode = 0 + + if method == 'QUERY': + server.Streams.stdout += Testers.ContainsExpression( + f'request attempt=1 method=QUERY body={re.escape(BODY)}', 'The origin must receive the QUERY and its content.') + server.Streams.stdout += Testers.ExcludesExpression( + 'request attempt=2 method=', 'ATS must not retry a QUERY after sending it.') + tr.Processes.Default.Streams.stdout += Testers.ContainsExpression('^502$', 'The failed QUERY must not be retried.') + else: + server.Streams.stdout += Testers.ContainsExpression( + f'request attempt=2 method={method}', f'ATS must retry a {method} after the origin closes.') + tr.Processes.Default.Streams.stdout += Testers.ContainsExpression('^200$', f'The retried {method} must succeed.') + + +# GET shows that the origin's close does lead ATS to retry a safe method. +add_scenario('get', 'GET', buffering=False) +add_scenario('query-buffered', 'QUERY', buffering=True) +add_scenario('query-unbuffered', 'QUERY', buffering=False)
diff --git a/tests/gold_tests/query/query_retry_origin.py b/tests/gold_tests/query/query_retry_origin.py new file mode 100644 index 0000000..b11e167 --- /dev/null +++ b/tests/gold_tests/query/query_retry_origin.py
@@ -0,0 +1,83 @@ +''' +An origin that closes the first connection after reading the request, without +responding, and answers every later request with a 200. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import socket +import sys + + +def read_request(conn: socket.socket) -> tuple[str, bytes]: + data = bytearray() + while b'\r\n\r\n' not in data: + chunk = conn.recv(4096) + if not chunk: + return '', b'' + data += chunk + head, _, rest = data.partition(b'\r\n\r\n') + body = bytearray(rest) + lines = head.decode('latin-1').split('\r\n') + method = lines[0].split(' ')[0] + length = 0 + for line in lines[1:]: + name, _, value = line.partition(':') + if name.strip().lower() == 'content-length': + length = int(value.strip()) + while len(body) < length: + chunk = conn.recv(4096) + if not chunk: + break + body += chunk + return method, bytes(body[:length]) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument('port', type=int) + args = parser.parse_args() + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(('127.0.0.1', args.port)) + listener.listen() + attempt = 0 + while True: + conn, _ = listener.accept() + with conn: + conn.settimeout(10) + try: + method, body = read_request(conn) + except OSError as e: + print(f'request read failed: {e}', flush=True) + continue + if not method: + continue + attempt += 1 + text = body.decode('utf-8', errors='replace') + print(f'request attempt={attempt} method={method} body={text}', flush=True) + if attempt == 1: + continue + payload = b'retried' + conn.sendall( + b'HTTP/1.1 200 OK\r\nContent-Length: ' + str(len(payload)).encode() + b'\r\nConnection: close\r\n\r\n' + + payload) + + +if __name__ == '__main__': + sys.exit(main())
diff --git a/tests/gold_tests/query/replay/query-ip-allow.replay.yaml b/tests/gold_tests/query/replay/query-ip-allow.replay.yaml new file mode 100644 index 0000000..7d5833e --- /dev/null +++ b/tests/gold_tests/query/replay/query-ip-allow.replay.yaml
@@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: 'Verify ip_allow.yaml can deny the QUERY method' + + server: + name: 'server' + + client: + name: 'client' + + ats: + name: 'ts' + process_config: + enable_cache: false + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'ip_allow' + ip_allow_yaml: + ip_allow: + - apply: in + ip_addrs: 0/0 + action: deny + methods: + - QUERY + - apply: in + ip_addrs: "::/0" + action: deny + methods: + - QUERY + remap_config: + - from: "http://example.com/" + to: "http://127.0.0.1:{SERVER_HTTP_PORT}/" + +sessions: +- transactions: + + # QUERY is denied by ip_allow, so it never reaches the origin. + - client-request: + method: QUERY + url: /denied + version: '1.1' + headers: + fields: + - [Host, example.com] + - [Content-Type, application/sql] + - [Content-Length, "8"] + - [uuid, ip-allow-denied] + content: + data: "select-a" + + proxy-request: + expect: absent + + proxy-response: + status: 403 + + # Methods other than QUERY are unaffected. + - client-request: + method: GET + url: /allowed + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, ip-allow-permitted] + + proxy-request: + method: GET + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, "2"] + content: + data: "ok" + + proxy-response: + status: 200
diff --git a/tests/gold_tests/query/replay/query-pass-through.replay.yaml b/tests/gold_tests/query/replay/query-pass-through.replay.yaml new file mode 100644 index 0000000..43eed77 --- /dev/null +++ b/tests/gold_tests/query/replay/query-pass-through.replay.yaml
@@ -0,0 +1,130 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: 'Verify QUERY requests are proxied with their content' + + server: + name: 'server' + + client: + name: 'client' + + ats: + name: 'ts' + process_config: + enable_cache: true + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'http_trans|http_cache' + remap_config: + - from: "http://example.com/" + to: "http://127.0.0.1:{SERVER_HTTP_PORT}/" + metric_checks: + - metric: 'proxy.process.http.query_requests' + value: 3 + +sessions: +- transactions: + + # A QUERY with content is forwarded to the origin, content and all. + - client-request: + method: QUERY + url: /search + version: '1.1' + headers: + fields: + - [Host, example.com] + - [Content-Type, application/sql] + - [Content-Length, "23"] + - [uuid, query-first] + content: + data: "SELECT * FROM t WHERE a" + + proxy-request: + method: QUERY + headers: + fields: + - [Content-Length, {value: "23", as: equal}] + content: + verify: {value: "SELECT * FROM t WHERE a", as: equal} + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, "8"] + - [Cache-Control, "max-age=300"] + content: + data: "result-1" + + proxy-response: + status: 200 + + # Caching of QUERY responses is off by default, even with the cache enabled, + # so the identical request goes to the origin again. + - client-request: + delay: 100ms + method: QUERY + url: /search + version: '1.1' + headers: + fields: + - [Host, example.com] + - [Content-Type, application/sql] + - [Content-Length, "23"] + - [uuid, query-second] + content: + data: "SELECT * FROM t WHERE a" + + proxy-request: + method: QUERY + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, "8"] + - [Cache-Control, "max-age=300"] + content: + data: "result-2" + + proxy-response: + status: 200 + content: + verify: {value: "result-2", as: equal} + + # A QUERY without a Content-Length and without chunked framing has no content, + # which ATS rejects the same way it rejects a length-less POST. + - client-request: + method: QUERY + url: /no-content + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, query-no-content] + + proxy-request: + expect: absent + + proxy-response: + status: 411
diff --git a/tests/gold_tests/query/replay/query-remap-acl.replay.yaml b/tests/gold_tests/query/replay/query-remap-acl.replay.yaml new file mode 100644 index 0000000..e95b9f8 --- /dev/null +++ b/tests/gold_tests/query/replay/query-remap-acl.replay.yaml
@@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: 'Verify a remap.config ACL can deny the QUERY method' + + server: + name: 'server' + + client: + name: 'client' + + ats: + name: 'ts' + process_config: + enable_cache: false + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'url_rewrite|ip_allow' + remap_config: + - 'map http://example.com/ http://127.0.0.1:{SERVER_HTTP_PORT}/ @action=deny @method=QUERY' + +sessions: +- transactions: + + - client-request: + method: QUERY + url: /denied + version: '1.1' + headers: + fields: + - [Host, example.com] + - [Content-Type, application/sql] + - [Content-Length, "8"] + - [uuid, remap-acl-denied] + content: + data: "select-a" + + proxy-request: + expect: absent + + proxy-response: + status: 403 + + - client-request: + method: GET + url: /allowed + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, remap-acl-permitted] + + proxy-request: + method: GET + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, "2"] + content: + data: "ok" + + proxy-response: + status: 200