blob: d5cc5e6d2b3b34926a8878a9de5392e775416c8d [file] [log] [blame]
/* caching.c : in-memory caching
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
#include "fs.h"
#include "fs_x.h"
#include "id.h"
#include "dag_cache.h"
#include "index.h"
#include "changes.h"
#include "noderevs.h"
#include "temp_serializer.h"
#include "reps.h"
#include "../libsvn_fs/fs-loader.h"
#include "svn_config.h"
#include "svn_cache_config.h"
#include "svn_private_config.h"
#include "svn_hash.h"
#include "svn_pools.h"
#include "private/svn_debug.h"
#include "private/svn_subr_private.h"
/* Take the ORIGINAL string and replace all occurrences of ":" without
* limiting the key space. Allocate the result in RESULT_POOL.
*/
static const char *
normalize_key_part(const char *original,
apr_pool_t *result_pool)
{
apr_size_t i;
apr_size_t len = strlen(original);
svn_stringbuf_t *normalized = svn_stringbuf_create_ensure(len,
result_pool);
for (i = 0; i < len; ++i)
{
char c = original[i];
switch (c)
{
case ':': svn_stringbuf_appendbytes(normalized, "%_", 2);
break;
case '%': svn_stringbuf_appendbytes(normalized, "%%", 2);
break;
default : svn_stringbuf_appendbyte(normalized, c);
}
}
return normalized->data;
}
/* *CACHE_TXDELTAS, *CACHE_FULLTEXTS, *CACHE_REVPROPS and *CACHE_NODEPROPS
flags will be set according to FS->CONFIG. *CACHE_NAMESPACE receives
the cache prefix to use.
Allocate CACHE_NAMESPACE in RESULT_POOL. */
static svn_error_t *
read_config(const char **cache_namespace,
svn_boolean_t *cache_txdeltas,
svn_boolean_t *cache_fulltexts,
svn_boolean_t *cache_revprops,
svn_boolean_t *cache_nodeprops,
svn_fs_t *fs,
apr_pool_t *result_pool)
{
/* No cache namespace by default. I.e. all FS instances share the
* cached data. If you specify different namespaces, the data will
* share / compete for the same cache memory but keys will not match
* across namespaces and, thus, cached data will not be shared between
* namespaces.
*
* Since the namespace will be concatenated with other elements to form
* the complete key prefix, we must make sure that the resulting string
* is unique and cannot be created by any other combination of elements.
*/
*cache_namespace
= normalize_key_part(svn_hash__get_cstring(fs->config,
SVN_FS_CONFIG_FSFS_CACHE_NS,
""),
result_pool);
/* don't cache text deltas by default.
* Once we reconstructed the fulltexts from the deltas,
* these deltas are rarely re-used. Therefore, only tools
* like svnadmin will activate this to speed up operations
* dump and verify.
*/
*cache_txdeltas
= svn_hash__get_bool(fs->config,
SVN_FS_CONFIG_FSFS_CACHE_DELTAS,
TRUE);
/* by default, cache fulltexts.
* Most SVN tools care about reconstructed file content.
* Thus, this is a reasonable default.
* SVN admin tools may set that to FALSE because fulltexts
* won't be re-used rendering the cache less effective
* by squeezing wanted data out.
*/
*cache_fulltexts
= svn_hash__get_bool(fs->config,
SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS,
TRUE);
/* don't cache revprops by default.
* Revprop caching significantly speeds up operations like
* svn ls -v. However, it requires synchronization that may
* not be available or efficient in the current server setup.
* Option "2" is equivalent to "1".
*/
if (strcmp(svn_hash__get_cstring(fs->config,
SVN_FS_CONFIG_FSFS_CACHE_REVPROPS,
""), "2"))
*cache_revprops
= svn_hash__get_bool(fs->config,
SVN_FS_CONFIG_FSFS_CACHE_REVPROPS,
FALSE);
else
*cache_revprops = TRUE;
/* by default, cache nodeprops: this will match pre-1.10
* behavior where node properties caching was controlled
* by SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS configuration option.
*/
*cache_nodeprops
= svn_hash__get_bool(fs->config,
SVN_FS_CONFIG_FSFS_CACHE_NODEPROPS,
TRUE);
return SVN_NO_ERROR;
}
/* Implements svn_cache__error_handler_t
* This variant clears the error after logging it.
*/
static svn_error_t *
warn_and_continue_on_cache_errors(svn_error_t *err,
void *baton,
apr_pool_t *pool)
{
svn_fs_t *fs = baton;
(fs->warning)(fs->warning_baton, err);
svn_error_clear(err);
return SVN_NO_ERROR;
}
/* Implements svn_cache__error_handler_t
* This variant logs the error and passes it on to the callers.
*/
static svn_error_t *
warn_and_fail_on_cache_errors(svn_error_t *err,
void *baton,
apr_pool_t *pool)
{
svn_fs_t *fs = baton;
(fs->warning)(fs->warning_baton, err);
return err;
}
#ifdef SVN_DEBUG_CACHE_DUMP_STATS
/* Baton to be used for the dump_cache_statistics() pool cleanup function, */
typedef struct dump_cache_baton_t
{
/* the pool about to be cleaned up. Will be used for temp. allocations. */
apr_pool_t *pool;
/* the cache to dump the statistics for */
svn_cache__t *cache;
} dump_cache_baton_t;
/* APR pool cleanup handler that will printf the statistics of the
cache referenced by the baton in BATON_VOID. */
static apr_status_t
dump_cache_statistics(void *baton_void)
{
dump_cache_baton_t *baton = baton_void;
apr_status_t result = APR_SUCCESS;
svn_cache__info_t info;
svn_string_t *text_stats;
apr_array_header_t *lines;
int i;
svn_error_t *err = svn_cache__get_info(baton->cache,
&info,
TRUE,
baton->pool);
/* skip unused caches */
if (! err && (info.gets > 0 || info.sets > 0))
{
text_stats = svn_cache__format_info(&info, TRUE, baton->pool);
lines = svn_cstring_split(text_stats->data, "\n", FALSE, baton->pool);
for (i = 0; i < lines->nelts; ++i)
{
const char *line = APR_ARRAY_IDX(lines, i, const char *);
#ifdef SVN_DEBUG
SVN_DBG(("%s\n", line));
#endif
}
}
/* process error returns */
if (err)
{
result = err->apr_err;
svn_error_clear(err);
}
return result;
}
static apr_status_t
dump_global_cache_statistics(void *baton_void)
{
apr_pool_t *pool = baton_void;
svn_cache__info_t *info = svn_cache__membuffer_get_global_info(pool);
svn_string_t *text_stats = svn_cache__format_info(info, FALSE, pool);
apr_array_header_t *lines = svn_cstring_split(text_stats->data, "\n",
FALSE, pool);
int i;
for (i = 0; i < lines->nelts; ++i)
{
const char *line = APR_ARRAY_IDX(lines, i, const char *);
#ifdef SVN_DEBUG
SVN_DBG(("%s\n", line));
#endif
}
return APR_SUCCESS;
}
#endif /* SVN_DEBUG_CACHE_DUMP_STATS */
/* This function sets / registers the required callbacks for a given
* not transaction-specific CACHE object in FS, if CACHE is not NULL.
*
* All these svn_cache__t instances shall be handled uniformly. Unless
* ERROR_HANDLER is NULL, register it for the given CACHE in FS.
*/
static svn_error_t *
init_callbacks(svn_cache__t *cache,
svn_fs_t *fs,
svn_cache__error_handler_t error_handler,
apr_pool_t *pool)
{
#ifdef SVN_DEBUG_CACHE_DUMP_STATS
/* schedule printing the access statistics upon pool cleanup,
* i.e. end of FSX session.
*/
dump_cache_baton_t *baton;
baton = apr_palloc(pool, sizeof(*baton));
baton->pool = pool;
baton->cache = cache;
apr_pool_cleanup_register(pool,
baton,
dump_cache_statistics,
apr_pool_cleanup_null);
#endif
if (error_handler)
SVN_ERR(svn_cache__set_error_handler(cache,
error_handler,
fs,
pool));
return SVN_NO_ERROR;
}
/* Sets *CACHE_P to cache instance based on provided options.
*
* If DUMMY_CACHE is set, create a null cache. Otherwise, creates a memcache
* if MEMCACHE is not NULL or a membuffer cache if MEMBUFFER is not NULL.
* Falls back to inprocess cache if no other cache type has been selected
* and PAGES is not 0. Create a null cache otherwise.
*
* Use the given PRIORITY class for the new cache. If PRIORITY is 0, then
* use the default priority class. HAS_NAMESPACE indicates whether we
* prefixed this cache instance with a namespace.
*
* Unless NO_HANDLER is true, register an error handler that reports errors
* as warnings to the FS warning callback.
*
* Cache is allocated in RESULT_POOL, temporaries in SCRATCH_POOL.
* */
static svn_error_t *
create_cache(svn_cache__t **cache_p,
svn_memcache_t *memcache,
svn_membuffer_t *membuffer,
apr_int64_t pages,
apr_int64_t items_per_page,
svn_cache__serialize_func_t serializer,
svn_cache__deserialize_func_t deserializer,
apr_ssize_t klen,
const char *prefix,
apr_uint32_t priority,
svn_boolean_t has_namespace,
svn_fs_t *fs,
svn_boolean_t no_handler,
svn_boolean_t dummy_cache,
apr_pool_t *result_pool,
apr_pool_t *scratch_pool)
{
svn_cache__error_handler_t error_handler = no_handler
? NULL
: warn_and_fail_on_cache_errors;
if (priority == 0)
priority = SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY;
if (dummy_cache)
{
SVN_ERR(svn_cache__create_null(cache_p, prefix, result_pool));
}
else if (memcache)
{
SVN_ERR(svn_cache__create_memcache(cache_p, memcache,
serializer, deserializer, klen,
prefix, result_pool));
error_handler = no_handler
? NULL
: warn_and_continue_on_cache_errors;
}
else if (membuffer)
{
/* We assume caches with namespaces to be relatively short-lived,
* i.e. their data will not be needed after a while. */
SVN_ERR(svn_cache__create_membuffer_cache(
cache_p, membuffer, serializer, deserializer,
klen, prefix, priority, FALSE, has_namespace,
result_pool, scratch_pool));
}
else if (pages)
{
SVN_ERR(svn_cache__create_inprocess(
cache_p, serializer, deserializer, klen, pages,
items_per_page, FALSE, prefix, result_pool));
}
else
{
SVN_ERR(svn_cache__create_null(cache_p, prefix, result_pool));
}
SVN_ERR(init_callbacks(*cache_p, fs, error_handler, result_pool));
return SVN_NO_ERROR;
}
svn_error_t *
svn_fs_x__initialize_caches(svn_fs_t *fs,
apr_pool_t *scratch_pool)
{
svn_fs_x__data_t *ffd = fs->fsap_data;
const char *prefix = apr_pstrcat(scratch_pool,
"fsx:", fs->uuid,
"--", ffd->instance_id,
"/", normalize_key_part(fs->path,
scratch_pool),
":",
SVN_VA_NULL);
svn_membuffer_t *membuffer;
svn_boolean_t no_handler = ffd->fail_stop;
svn_boolean_t cache_txdeltas;
svn_boolean_t cache_fulltexts;
svn_boolean_t cache_revprops;
svn_boolean_t cache_nodeprops;
const char *cache_namespace;
svn_boolean_t has_namespace;
/* Evaluating the cache configuration. */
SVN_ERR(read_config(&cache_namespace,
&cache_txdeltas,
&cache_fulltexts,
&cache_revprops,
&cache_nodeprops,
fs,
scratch_pool));
prefix = apr_pstrcat(scratch_pool, "ns:", cache_namespace, ":", prefix,
SVN_VA_NULL);
has_namespace = strlen(cache_namespace) > 0;
membuffer = svn_cache__get_global_membuffer_cache();
/* General rules for assigning cache priorities:
*
* - Data that can be reconstructed from other elements has low prio
* (e.g. fulltexts, directories etc.)
* - Index data required to find any of the other data has high prio
* (e.g. noderevs, L2P and P2L index pages)
* - everthing else should use default prio
*/
#ifdef SVN_DEBUG_CACHE_DUMP_STATS
/* schedule printing the global access statistics upon pool cleanup,
* i.e. end of FSX session.
*/
if (membuffer)
apr_pool_cleanup_register(fs->pool,
fs->pool,
dump_global_cache_statistics,
apr_pool_cleanup_null);
#endif
/* 1st level DAG node cache */
ffd->dag_node_cache = svn_fs_x__create_dag_cache(fs->pool);
/* Very rough estimate: 1K per directory. */
SVN_ERR(create_cache(&(ffd->dir_cache),
NULL,
membuffer,
1024, 8,
svn_fs_x__serialize_dir_entries,
svn_fs_x__deserialize_dir_entries,
sizeof(svn_fs_x__id_t),
apr_pstrcat(scratch_pool, prefix, "DIR", SVN_VA_NULL),
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
has_namespace,
fs,
no_handler, FALSE,
fs->pool, scratch_pool));
/* initialize node revision cache, if caching has been enabled */
SVN_ERR(create_cache(&(ffd->node_revision_cache),
NULL,
membuffer,
32, 32, /* ~200 byte / entry; 1k entries total */
svn_fs_x__serialize_node_revision,
svn_fs_x__deserialize_node_revision,
sizeof(svn_fs_x__pair_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "NODEREVS",
SVN_VA_NULL),
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
has_namespace,
fs,
no_handler, FALSE,
fs->pool, scratch_pool));
/* initialize representation header cache, if caching has been enabled */
SVN_ERR(create_cache(&(ffd->rep_header_cache),
NULL,
membuffer,
1, 1000, /* ~8 bytes / entry; 1k entries total */
svn_fs_x__serialize_rep_header,
svn_fs_x__deserialize_rep_header,
sizeof(svn_fs_x__representation_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "REPHEADER",
SVN_VA_NULL),
SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
has_namespace,
fs,
no_handler, FALSE,
fs->pool, scratch_pool));
/* initialize node change list cache, if caching has been enabled */
SVN_ERR(create_cache(&(ffd->changes_cache),
NULL,
membuffer,
1, 8, /* 1k / entry; 8 entries total, rarely used */
svn_fs_x__serialize_changes,
svn_fs_x__deserialize_changes,
sizeof(svn_fs_x__pair_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "CHANGES",
SVN_VA_NULL),
0,
has_namespace,
fs,
no_handler, FALSE,
fs->pool, scratch_pool));
/* if enabled, cache fulltext and other derived information */
SVN_ERR(create_cache(&(ffd->fulltext_cache),
ffd->memcache,
membuffer,
0, 0, /* Do not use inprocess cache */
/* Values are svn_stringbuf_t */
NULL, NULL,
sizeof(svn_fs_x__pair_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "TEXT",
SVN_VA_NULL),
SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
has_namespace,
fs,
no_handler, !cache_fulltexts,
fs->pool, scratch_pool));
SVN_ERR(create_cache(&(ffd->properties_cache),
NULL,
membuffer,
0, 0, /* Do not use inprocess cache */
svn_fs_x__serialize_properties,
svn_fs_x__deserialize_properties,
sizeof(svn_fs_x__pair_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "PROP",
SVN_VA_NULL),
SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
has_namespace,
fs,
no_handler, !cache_nodeprops,
fs->pool, scratch_pool));
/* if enabled, cache revprops */
SVN_ERR(create_cache(&(ffd->revprop_cache),
NULL,
membuffer,
0, 0, /* Do not use inprocess cache */
svn_fs_x__serialize_properties,
svn_fs_x__deserialize_properties,
sizeof(svn_fs_x__pair_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "REVPROP",
SVN_VA_NULL),
SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
has_namespace,
fs,
no_handler, !cache_revprops,
fs->pool, scratch_pool));
/* if enabled, cache text deltas and their combinations */
SVN_ERR(create_cache(&(ffd->txdelta_window_cache),
NULL,
membuffer,
0, 0, /* Do not use inprocess cache */
svn_fs_x__serialize_txdelta_window,
svn_fs_x__deserialize_txdelta_window,
sizeof(svn_fs_x__window_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "TXDELTA_WINDOW",
SVN_VA_NULL),
SVN_CACHE__MEMBUFFER_LOW_PRIORITY,
has_namespace,
fs,
no_handler, !cache_txdeltas,
fs->pool, scratch_pool));
SVN_ERR(create_cache(&(ffd->combined_window_cache),
NULL,
membuffer,
0, 0, /* Do not use inprocess cache */
/* Values are svn_stringbuf_t */
NULL, NULL,
sizeof(svn_fs_x__window_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "COMBINED_WINDOW",
SVN_VA_NULL),
SVN_CACHE__MEMBUFFER_LOW_PRIORITY,
has_namespace,
fs,
no_handler, !cache_txdeltas,
fs->pool, scratch_pool));
/* Caches for our various container types. */
SVN_ERR(create_cache(&(ffd->noderevs_container_cache),
NULL,
membuffer,
16, 4, /* Important, largish objects */
svn_fs_x__serialize_noderevs_container,
svn_fs_x__deserialize_noderevs_container,
sizeof(svn_fs_x__pair_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "NODEREVSCNT",
SVN_VA_NULL),
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
has_namespace,
fs,
no_handler, FALSE,
fs->pool, scratch_pool));
SVN_ERR(create_cache(&(ffd->changes_container_cache),
NULL,
membuffer,
0, 0, /* Do not use inprocess cache */
svn_fs_x__serialize_changes_container,
svn_fs_x__deserialize_changes_container,
sizeof(svn_fs_x__pair_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "CHANGESCNT",
SVN_VA_NULL),
0,
has_namespace,
fs,
no_handler, FALSE,
fs->pool, scratch_pool));
SVN_ERR(create_cache(&(ffd->reps_container_cache),
NULL,
membuffer,
0, 0, /* Do not use inprocess cache */
svn_fs_x__serialize_reps_container,
svn_fs_x__deserialize_reps_container,
sizeof(svn_fs_x__pair_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "REPSCNT",
SVN_VA_NULL),
0,
has_namespace,
fs,
no_handler, FALSE,
fs->pool, scratch_pool));
/* Cache index info. */
SVN_ERR(create_cache(&(ffd->l2p_header_cache),
NULL,
membuffer,
64, 16, /* entry size varies but we must cover
a reasonable number of revisions (1k) */
svn_fs_x__serialize_l2p_header,
svn_fs_x__deserialize_l2p_header,
sizeof(svn_fs_x__pair_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "L2P_HEADER",
SVN_VA_NULL),
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
has_namespace,
fs,
no_handler, FALSE,
fs->pool, scratch_pool));
SVN_ERR(create_cache(&(ffd->l2p_page_cache),
NULL,
membuffer,
64, 16, /* entry size varies but we must cover
a reasonable number of revisions (1k) */
svn_fs_x__serialize_l2p_page,
svn_fs_x__deserialize_l2p_page,
sizeof(svn_fs_x__page_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "L2P_PAGE",
SVN_VA_NULL),
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
has_namespace,
fs,
no_handler, FALSE,
fs->pool, scratch_pool));
SVN_ERR(create_cache(&(ffd->p2l_header_cache),
NULL,
membuffer,
4, 1, /* Large entries. Rarely used. */
svn_fs_x__serialize_p2l_header,
svn_fs_x__deserialize_p2l_header,
sizeof(svn_fs_x__pair_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "P2L_HEADER",
SVN_VA_NULL),
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
has_namespace,
fs,
no_handler, FALSE,
fs->pool, scratch_pool));
SVN_ERR(create_cache(&(ffd->p2l_page_cache),
NULL,
membuffer,
4, 16, /* Variably sized entries. Rarely used. */
svn_fs_x__serialize_p2l_page,
svn_fs_x__deserialize_p2l_page,
sizeof(svn_fs_x__page_cache_key_t),
apr_pstrcat(scratch_pool, prefix, "P2L_PAGE",
SVN_VA_NULL),
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
has_namespace,
fs,
no_handler, FALSE,
fs->pool, scratch_pool));
return SVN_NO_ERROR;
}