Add a new configuration to ignore query params in cache keys (#9707)
This adds a new configuration option:
CONFIG proxy.config.http.cache.ignore_query INT 0
This is off by default, but it is overridable. The typical use case
for this would be to use e.g. conf_remap.so plugin, to set this for
services / URLs which shouldn't use the query parameters in the HTTP
cache key calculations.
This would be significantly less expensive (less CPU) than using e.g.
the cachekey.so plugin to simply strip the query parameters from the key.diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst
index 0cf49b0..a894302 100644
--- a/doc/admin-guide/files/records.yaml.en.rst
+++ b/doc/admin-guide/files/records.yaml.en.rst
@@ -2134,6 +2134,14 @@
used to purge the entire cache, or just a specific :file:`remap.config`
rule.
+.. ts:cv:: CONFIG proxy.config.http.cache.ignore_query INT 0
+ :reloadable:
+ :overridable:
+
+ If this value is set to ``1``, then the query string is ignored when
+ calculating the cach key for the request. This can be noticeably faster
+ than using e.g. the ``cachekey`` plugin to just remove the query parameters.
+
.. ts:cv:: CONFIG proxy.config.http.doc_in_cache_skip_dns INT 1
:reloadable:
:overridable:
diff --git a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst
index f2c27df..1a7aa78 100644
--- a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst
+++ b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst
@@ -84,6 +84,7 @@
:c:enumerator:`TS_CONFIG_HTTP_BACKGROUND_FILL_COMPLETED_THRESHOLD` :ts:cv:`proxy.config.http.background_fill_completed_threshold`
:c:enumerator:`TS_CONFIG_HTTP_CACHE_CACHE_RESPONSES_TO_COOKIES` :ts:cv:`proxy.config.http.cache.cache_responses_to_cookies`
:c:enumerator:`TS_CONFIG_HTTP_CACHE_CACHE_URLS_THAT_LOOK_DYNAMIC` :ts:cv:`proxy.config.http.cache.cache_urls_that_look_dynamic`
+:c:enumerator:`TS_CONFIG_HTTP_CACHE_IGNORE_QUERY` :ts:cv:`proxy.config.http.cache.ignore_query`
:c:enumerator:`TS_CONFIG_HTTP_CACHE_GENERATION` :ts:cv:`proxy.config.http.cache.generation`
:c:enumerator:`TS_CONFIG_HTTP_CACHE_GUARANTEED_MAX_LIFETIME` :ts:cv:`proxy.config.http.cache.guaranteed_max_lifetime`
:c:enumerator:`TS_CONFIG_HTTP_CACHE_GUARANTEED_MIN_LIFETIME` :ts:cv:`proxy.config.http.cache.guaranteed_min_lifetime`
diff --git a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst
index 287e4f5..fc9a920 100644
--- a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst
+++ b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst
@@ -155,6 +155,7 @@
.. c:enumerator:: TS_CONFIG_BODY_FACTORY_RESPONSE_SUPPRESSION_MODE
.. c:enumerator:: TS_CONFIG_NET_DEFAULT_INACTIVITY_TIMEOUT
.. c:enumerator:: TS_CONFIG_HTTP_NO_DNS_JUST_FORWARD_TO_PARENT
+.. c:enumerator:: TS_CONFIG_HTTP_CACHE_IGNORE_QUERY
Description
diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in
index fd0056a..d1fd899 100644
--- a/include/ts/apidefs.h.in
+++ b/include/ts/apidefs.h.in
@@ -888,7 +888,8 @@
TS_CONFIG_HTTP_DISABLE_PARENT_MARKDOWNS,
TS_CONFIG_NET_DEFAULT_INACTIVITY_TIMEOUT,
TS_CONFIG_HTTP_NO_DNS_JUST_FORWARD_TO_PARENT,
- TS_CONFIG_LAST_ENTRY,
+ TS_CONFIG_HTTP_CACHE_IGNORE_QUERY,
+ TS_CONFIG_LAST_ENTRY
} TSOverridableConfigKey;
/* The TASK pool of threads is the primary method of off-loading continuations from the
diff --git a/iocore/cache/P_CacheInternal.h b/iocore/cache/P_CacheInternal.h
index 63186f9..f175c0a 100644
--- a/iocore/cache/P_CacheInternal.h
+++ b/iocore/cache/P_CacheInternal.h
@@ -1003,7 +1003,7 @@
const CacheKey *key1 = nullptr, CacheFragType type = CACHE_FRAG_TYPE_HTTP, const char *hostname = nullptr,
int host_len = 0);
static void generate_key(CryptoHash *hash, CacheURL *url);
- static void generate_key(HttpCacheKey *hash, CacheURL *url, cache_generation_t generation = -1);
+ static void generate_key(HttpCacheKey *hash, CacheURL *url, bool ignore_query = false, cache_generation_t generation = -1);
void vol_initialized(bool result);
@@ -1024,10 +1024,10 @@
}
inline void
-Cache::generate_key(HttpCacheKey *key, CacheURL *url, cache_generation_t generation)
+Cache::generate_key(HttpCacheKey *key, CacheURL *url, bool ignore_query, cache_generation_t generation)
{
key->hostname = url->host_get(&key->hostlen);
- url->hash_get(&key->hash, generation);
+ url->hash_get(&key->hash, ignore_query, generation);
}
inline unsigned int
diff --git a/plugins/lua/ts_lua_http_config.c b/plugins/lua/ts_lua_http_config.c
index 36c8019..86f0079 100644
--- a/plugins/lua/ts_lua_http_config.c
+++ b/plugins/lua/ts_lua_http_config.c
@@ -146,6 +146,7 @@
TS_LUA_CONFIG_DISABLE_PARENT_MARKDOWNS = TS_CONFIG_HTTP_DISABLE_PARENT_MARKDOWNS,
TS_LUA_CONFIG_NET_DEFAULT_INACTIVITY_TIMEOUT = TS_CONFIG_NET_DEFAULT_INACTIVITY_TIMEOUT,
TS_LUA_CONFIG_HTTP_NO_DNS_JUST_FORWARD_TO_PARENT = TS_CONFIG_HTTP_NO_DNS_JUST_FORWARD_TO_PARENT,
+ TS_LUA_CONFIG_HTTP_CACHE_IGNORE_QUERY = TS_CONFIG_HTTP_CACHE_IGNORE_QUERY,
TS_LUA_CONFIG_LAST_ENTRY = TS_CONFIG_LAST_ENTRY,
} TSLuaOverridableConfigKey;
@@ -284,6 +285,7 @@
TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_DISABLE_PARENT_MARKDOWNS),
TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_NET_DEFAULT_INACTIVITY_TIMEOUT),
TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NO_DNS_JUST_FORWARD_TO_PARENT),
+ TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_IGNORE_QUERY),
TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_LAST_ENTRY),
};
diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc
index 3a1c3de..63aedcc 100644
--- a/proxy/hdrs/URL.cc
+++ b/proxy/hdrs/URL.cc
@@ -1801,7 +1801,8 @@
}
static inline void
-url_CryptoHash_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, cache_generation_t generation)
+url_CryptoHash_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, bool ignore_query,
+ cache_generation_t generation)
{
char buffer[BUFSIZE];
char *p, *e;
@@ -1833,11 +1834,19 @@
strs[9] = ";";
strs[10] = url->m_ptr_params;
strs[11] = "?";
- strs[12] = url->m_ptr_query;
+
+ // Special case for the query paramters, allowing us to ignore them if requested
+ if (!ignore_query) {
+ strs[12] = url->m_ptr_query;
+ ends[12] = strs[12] + url->m_len_query;
+ } else {
+ strs[12] = nullptr;
+ ends[12] = nullptr;
+ }
+
ends[9] = strs[9] + 1;
ends[10] = strs[10] + url->m_len_params;
ends[11] = strs[11] + 1;
- ends[12] = strs[12] + url->m_len_query;
p = buffer;
e = buffer + BUFSIZE;
@@ -1879,21 +1888,21 @@
}
void
-url_CryptoHash_get(const URLImpl *url, CryptoHash *hash, cache_generation_t generation)
+url_CryptoHash_get(const URLImpl *url, CryptoHash *hash, bool ignore_query, cache_generation_t generation)
{
URLHashContext ctx;
if ((url_hash_method != 0) && (url->m_url_type == URL_TYPE_HTTP) &&
- ((url->m_len_user + url->m_len_password + url->m_len_params + url->m_len_query) == 0) &&
+ ((url->m_len_user + url->m_len_password + url->m_len_params + (ignore_query ? 0 : url->m_len_query)) == 0) &&
(3 + 1 + 1 + 1 + 1 + 1 + 2 + url->m_len_scheme + url->m_len_host + url->m_len_path < BUFSIZE) &&
(memchr(url->m_ptr_host, '%', url->m_len_host) == nullptr) && (memchr(url->m_ptr_path, '%', url->m_len_path) == nullptr)) {
url_CryptoHash_get_fast(url, ctx, hash, generation);
#ifdef DEBUG
CryptoHash hash_general;
- url_CryptoHash_get_general(url, ctx, hash_general, generation);
+ url_CryptoHash_get_general(url, ctx, hash_general, ignore_query, generation);
ink_assert(*hash == hash_general);
#endif
} else {
- url_CryptoHash_get_general(url, ctx, *hash, generation);
+ url_CryptoHash_get_general(url, ctx, *hash, ignore_query, generation);
}
}
diff --git a/proxy/hdrs/URL.h b/proxy/hdrs/URL.h
index c89e868..294837c 100644
--- a/proxy/hdrs/URL.h
+++ b/proxy/hdrs/URL.h
@@ -216,7 +216,7 @@
void url_called_set(URLImpl *url);
char *url_string_get_buf(URLImpl *url, char *dstbuf, int dstbuf_size, int *length);
-void url_CryptoHash_get(const URLImpl *url, CryptoHash *hash, cache_generation_t generation = -1);
+void url_CryptoHash_get(const URLImpl *url, CryptoHash *hash, bool ignore_query = false, cache_generation_t generation = -1);
void url_host_CryptoHash_get(URLImpl *url, CryptoHash *hash);
constexpr bool USE_STRICT_URI_PARSING = true;
@@ -277,7 +277,7 @@
char *string_get(Arena *arena, int *length = nullptr) const;
char *string_get_ref(int *length = nullptr, unsigned normalization_flags = URLNormalize::NONE) const;
char *string_get_buf(char *dstbuf, int dsbuf_size, int *length = nullptr) const;
- void hash_get(CryptoHash *hash, cache_generation_t generation = -1) const;
+ void hash_get(CryptoHash *hash, bool ignore_query = false, cache_generation_t generation = -1) const;
void host_hash_get(CryptoHash *hash) const;
const char *scheme_get(int *length);
@@ -492,10 +492,10 @@
-------------------------------------------------------------------------*/
inline void
-URL::hash_get(CryptoHash *hash, cache_generation_t generation) const
+URL::hash_get(CryptoHash *hash, bool ignore_query, cache_generation_t generation) const
{
ink_assert(valid());
- url_CryptoHash_get(m_url_impl, hash, generation);
+ url_CryptoHash_get(m_url_impl, hash, ignore_query, generation);
}
/*-------------------------------------------------------------------------
diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc
index d8c3a34..8b167c1 100644
--- a/proxy/http/HttpConfig.cc
+++ b/proxy/http/HttpConfig.cc
@@ -1312,6 +1312,7 @@
HttpEstablishStaticConfigLongLong(c.oride.cache_guaranteed_max_lifetime, "proxy.config.http.cache.guaranteed_max_lifetime");
HttpEstablishStaticConfigLongLong(c.oride.cache_max_stale_age, "proxy.config.http.cache.max_stale_age");
+
HttpEstablishStaticConfigByte(c.oride.srv_enabled, "proxy.config.srv_enabled");
HttpEstablishStaticConfigByte(c.oride.allow_half_open, "proxy.config.http.allow_half_open");
@@ -1338,6 +1339,7 @@
HttpEstablishStaticConfigByte(c.oride.cache_ignore_auth, "proxy.config.http.cache.ignore_authentication");
HttpEstablishStaticConfigByte(c.oride.cache_urls_that_look_dynamic, "proxy.config.http.cache.cache_urls_that_look_dynamic");
+ HttpEstablishStaticConfigByte(c.oride.cache_ignore_query, "proxy.config.http.cache.ignore_query");
HttpEstablishStaticConfigByte(c.cache_post_method, "proxy.config.http.cache.post_method");
HttpEstablishStaticConfigByte(c.oride.ignore_accept_mismatch, "proxy.config.http.cache.ignore_accept_mismatch");
@@ -1630,6 +1632,7 @@
params->oride.cache_responses_to_cookies = m_master.oride.cache_responses_to_cookies;
params->oride.cache_ignore_auth = INT_TO_BOOL(m_master.oride.cache_ignore_auth);
params->oride.cache_urls_that_look_dynamic = INT_TO_BOOL(m_master.oride.cache_urls_that_look_dynamic);
+ params->oride.cache_ignore_query = INT_TO_BOOL(m_master.oride.cache_ignore_query);
params->cache_post_method = INT_TO_BOOL(m_master.cache_post_method);
params->oride.ignore_accept_mismatch = m_master.oride.ignore_accept_mismatch;
diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h
index c116976..ff293f5 100644
--- a/proxy/http/HttpConfig.h
+++ b/proxy/http/HttpConfig.h
@@ -566,6 +566,7 @@
MgmtByte cache_responses_to_cookies = 1;
MgmtByte cache_ignore_auth = 0;
MgmtByte cache_urls_that_look_dynamic = 1;
+ MgmtByte cache_ignore_query = 0;
MgmtByte cache_required_headers = 2;
MgmtByte cache_range_lookup = 1;
MgmtByte cache_range_write = 0;
@@ -619,6 +620,12 @@
/////////////////////////////////////////////////
MgmtByte response_suppression_mode = 0; // proxy.config.body_factory.response_suppression_mode
+ //////////////////
+ // Redirection //
+ //////////////////
+ MgmtByte redirect_use_orig_cache_key = 0;
+ MgmtInt number_of_redirections = 0;
+
//////////////////////////////
// server verification mode //
//////////////////////////////
@@ -626,12 +633,6 @@
char *ssl_client_verify_server_properties = nullptr;
char *ssl_client_sni_policy = nullptr;
- //////////////////
- // Redirection //
- //////////////////
- MgmtByte redirect_use_orig_cache_key = 0;
- MgmtInt number_of_redirections = 0;
-
MgmtInt proxy_response_hsts_max_age = -1;
////////////////////////////////
diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc
index c0087fa..99994ec 100644
--- a/proxy/http/HttpSM.cc
+++ b/proxy/http/HttpSM.cc
@@ -5047,7 +5047,7 @@
SMDebug("http_seq", "Issuing cache lookup for URL %s", c_url->string_get(&t_state.arena));
HttpCacheKey key;
- Cache::generate_key(&key, c_url, t_state.txn_conf->cache_generation_number);
+ Cache::generate_key(&key, c_url, t_state.txn_conf->cache_ignore_query, t_state.txn_conf->cache_generation_number);
pending_action = cache_sm.open_read(
&key, c_url, &t_state.hdr_info.client_request, t_state.txn_conf,
@@ -5072,7 +5072,8 @@
SMDebug("http_seq", "Issuing cache delete for %s", t_state.cache_info.lookup_url->string_get_ref());
HttpCacheKey key;
- Cache::generate_key(&key, t_state.cache_info.lookup_url, t_state.txn_conf->cache_generation_number);
+ Cache::generate_key(&key, t_state.cache_info.lookup_url, t_state.txn_conf->cache_ignore_query,
+ t_state.txn_conf->cache_generation_number);
pending_action = cacheProcessor.remove(cont, &key);
return;
@@ -5150,7 +5151,7 @@
SMDebug("http_cache_write", "writing to cache with URL %s", s_url->string_get(&t_state.arena));
HttpCacheKey key;
- Cache::generate_key(&key, s_url, t_state.txn_conf->cache_generation_number);
+ Cache::generate_key(&key, s_url, t_state.txn_conf->cache_ignore_query, t_state.txn_conf->cache_generation_number);
pending_action =
c_sm->open_write(&key, s_url, &t_state.hdr_info.client_request, object_read_info,
diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc
index c3213fe..46a858b 100644
--- a/src/records/RecordsConfig.cc
+++ b/src/records/RecordsConfig.cc
@@ -576,6 +576,8 @@
,
{RECT_CONFIG, "proxy.config.http.cache.cache_urls_that_look_dynamic", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}
,
+ {RECT_CONFIG, "proxy.config.http.cache.ignore_query", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}
+ ,
{RECT_CONFIG, "proxy.config.http.cache.post_method", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}
,
{RECT_CONFIG, "proxy.config.http.cache.max_open_read_retries", RECD_INT, "-1", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
diff --git a/src/shared/overridable_txn_vars.cc b/src/shared/overridable_txn_vars.cc
index fc50404..5cf1ea4 100644
--- a/src/shared/overridable_txn_vars.cc
+++ b/src/shared/overridable_txn_vars.cc
@@ -171,5 +171,6 @@
{TS_CONFIG_HTTP_ENABLE_PARENT_TIMEOUT_MARKDOWNS, TS_RECORDDATATYPE_INT} },
{"proxy.config.http.parent_proxy.disable_parent_markdowns", {TS_CONFIG_HTTP_DISABLE_PARENT_MARKDOWNS, TS_RECORDDATATYPE_INT} },
{"proxy.config.http.parent_proxy.disable_parent_markdowns", {TS_CONFIG_HTTP_DISABLE_PARENT_MARKDOWNS, TS_RECORDDATATYPE_INT} },
- {"proxy.config.net.default_inactivity_timeout", {TS_CONFIG_NET_DEFAULT_INACTIVITY_TIMEOUT, TS_RECORDDATATYPE_INT} }
+ {"proxy.config.net.default_inactivity_timeout", {TS_CONFIG_NET_DEFAULT_INACTIVITY_TIMEOUT, TS_RECORDDATATYPE_INT} },
+ {"proxy.config.http.cache.ignore_query", {TS_CONFIG_HTTP_CACHE_IGNORE_QUERY, TS_RECORDDATATYPE_INT} }
});
diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc
index 6f43a07..d62a9d2 100644
--- a/src/traffic_server/InkAPI.cc
+++ b/src/traffic_server/InkAPI.cc
@@ -8550,8 +8550,8 @@
case TS_CONFIG_HTTP_CACHE_IGNORE_AUTHENTICATION:
ret = _memberp_to_generic(&overridableHttpConfig->cache_ignore_auth, conv);
break;
- case TS_CONFIG_HTTP_CACHE_CACHE_URLS_THAT_LOOK_DYNAMIC:
- ret = _memberp_to_generic(&overridableHttpConfig->cache_urls_that_look_dynamic, conv);
+ case TS_CONFIG_HTTP_CACHE_IGNORE_QUERY:
+ ret = _memberp_to_generic(&overridableHttpConfig->cache_ignore_query, conv);
break;
case TS_CONFIG_HTTP_CACHE_REQUIRED_HEADERS:
ret = _memberp_to_generic(&overridableHttpConfig->cache_required_headers, conv);
@@ -8829,6 +8829,9 @@
case TS_CONFIG_NET_DEFAULT_INACTIVITY_TIMEOUT:
ret = _memberp_to_generic(&overridableHttpConfig->default_inactivity_timeout, conv);
break;
+ case TS_CONFIG_HTTP_CACHE_CACHE_URLS_THAT_LOOK_DYNAMIC:
+ ret = _memberp_to_generic(&overridableHttpConfig->cache_urls_that_look_dynamic, conv);
+ break;
// This helps avoiding compiler warnings, yet detect unhandled enum members.
case TS_CONFIG_NULL:
diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc
index 6e1e0a7..dbf1c00 100644
--- a/src/traffic_server/InkAPITest.cc
+++ b/src/traffic_server/InkAPITest.cc
@@ -8645,7 +8645,8 @@
"proxy.config.plugin.vc.default_buffer_water_mark", "proxy.config.net.sock_notsent_lowat",
"proxy.config.body_factory.response_suppression_mode", "proxy.config.http.parent_proxy.enable_parent_timeout_markdowns",
"proxy.config.http.parent_proxy.disable_parent_markdowns", "proxy.config.net.default_inactivity_timeout",
- "proxy.config.http.no_dns_just_forward_to_parent", }
+ "proxy.config.http.no_dns_just_forward_to_parent", "proxy.config.http.cache.ignore_query",
+ }
};
extern ClassAllocator<HttpSM> httpSMAllocator;
