On the fsx branch: final sync with /trunk. Conflicts in spillbuf.c and spillbuf-test.c resolved. git-svn-id: https://svn.apache.org/repos/asf/subversion/branches/fsx@1509911 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/BRANCH-README b/BRANCH-README new file mode 100644 index 0000000..3a5d4a1 --- /dev/null +++ b/BRANCH-README
@@ -0,0 +1,6 @@ +This is an integration branch to prepare FSX for merger into /trunk +and it should be short-lived. The main tasks here are + +* extract FSX and necessary new APIs from /branches/fsfs-format7 +* add tests for new libsvn_subr APIs +
diff --git a/build.conf b/build.conf index 5e16b64..71234e7 100644 --- a/build.conf +++ b/build.conf
@@ -49,6 +49,7 @@ private-built-includes = subversion/svn_private_config.h subversion/libsvn_fs_fs/rep-cache-db.h + subversion/libsvn_fs_x/rep-cache-db.h subversion/libsvn_wc/wc-metadata.h subversion/libsvn_wc/wc-queries.h subversion/libsvn_wc/wc-checks.h @@ -281,6 +282,13 @@ libs = libsvn_delta libsvn_subr aprutil apriconv apr libsvn_fs_util msvc-static = yes +[libsvn_fs_x] +type = fs-module +path = subversion/libsvn_fs_x +install = fsmod-lib +libs = libsvn_delta libsvn_subr aprutil apriconv apr libsvn_fs_util +msvc-static = yes + # Low-level grab bag of utilities [libsvn_fs_util] type = lib @@ -408,12 +416,18 @@ # CONSTRUCTED HEADERS # -[rep_cache] -description = Schema for the rep-sharing feature +[rep_cache_fs_fs] +description = Schema for the FSFS rep-sharing feature type = sql-header path = subversion/libsvn_fs_fs sources = rep-cache-db.sql +[rep_cache_fs_x] +description = Schema for the FSX rep-sharing feature +type = sql-header +path = subversion/libsvn_fs_x +sources = rep-cache-db.sql + [wc_queries] desription = Queries on the WC database type = sql-header @@ -751,16 +765,35 @@ # ---------------------------------------------------------------------------- # Tests for libsvn_fs_fs -[fs-pack-test] +[fs-fs-pack-test] description = Test fsfs packing in libsvn_fs_fs type = exe path = subversion/tests/libsvn_fs_fs -sources = fs-pack-test.c +sources = fs-fs-pack-test.c install = test libs = libsvn_test libsvn_fs libsvn_fs_fs libsvn_delta libsvn_subr apriconv apr # ---------------------------------------------------------------------------- +# Tests for libsvn_fs_x +[fs-x-pack-test] +description = Test fsx packing in libsvn_fs_fs +type = exe +path = subversion/tests/libsvn_fs_x +sources = fs-x-pack-test.c +install = test +libs = libsvn_test libsvn_fs libsvn_fs_x libsvn_delta + libsvn_subr apriconv apr + +[string-table-test] +description = Test fsfs string tables +type = exe +path = subversion/tests/libsvn_fs_x +sources = string-table-test.c +install = test +libs = libsvn_test libsvn_fs_x libsvn_subr apr + +# ---------------------------------------------------------------------------- # Tests for libsvn_fs [locks-test] @@ -914,6 +947,14 @@ install = sub-test libs = libsvn_subr apr +[packed-data-test] +description = Test path library +type = exe +path = subversion/tests/libsvn_subr +sources = packed-data-test.c +install = test +libs = libsvn_test libsvn_subr apriconv apr + [path-test] description = Test path library type = exe @@ -922,6 +963,22 @@ install = test libs = libsvn_test libsvn_subr apriconv apr +[prefix-string-test] +description = Test path library +type = exe +path = subversion/tests/libsvn_subr +sources = prefix-string-test.c +install = test +libs = libsvn_test libsvn_subr apriconv apr + +[priority-queue-test] +description = Test path library +type = exe +path = subversion/tests/libsvn_subr +sources = priority-queue-test.c +install = test +libs = libsvn_test libsvn_subr apriconv apr + [revision-test] description = Test revision library type = exe @@ -1282,7 +1339,7 @@ [fs-libs] type = lib external-lib = $(SVN_FS_LIB_LINK) -libs = libsvn_fs_base libsvn_fs_fs +libs = libsvn_fs_base libsvn_fs_fs libsvn_fs_x [__ALL__] type = project @@ -1296,10 +1353,12 @@ type = project path = build/win32 libs = __ALL__ - fs-test fs-base-test fs-fsfs-test fs-pack-test skel-test - strings-reps-test changes-test locks-test repos-test + fs-test fs-base-test fs-fsfs-test fs-fs-pack-test fs-x-pack-test + string-table-test + skel-test strings-reps-test changes-test locks-test repos-test checksum-test compat-test config-test hashdump-test mergeinfo-test - opt-test path-test stream-test string-test time-test utf-test + opt-test packed-data-test path-test prefix-string-test + priority-queue-test stream-test string-test time-test utf-test error-test error-code-test cache-test spillbuf-test crypto-test named_atomic-test named_atomic-proc-test revision-test subst_translate-test io-test
diff --git a/configure.ac b/configure.ac index 5383fd4..9e02de0 100644 --- a/configure.ac +++ b/configure.ac
@@ -1389,6 +1389,12 @@ svn_fs_lib_install_deps="install-fsmod-lib" svn_fs_lib_link="\$(FS_FS_LINK)" + AC_DEFINE(SVN_LIBSVN_FS_LINKS_FS_X, 1, + [Defined if libsvn_fs should link against libsvn_fs_x]) + svn_fs_lib_deps="$svn_fs_lib_deps \$(FS_X_DEPS)" + svn_fs_lib_install_deps="$svn_fs_lib_install_deps install-fsx-lib" + svn_fs_lib_link="$svn_fs_lib_link \$(FS_X_LINK)" + if test "$svn_lib_berkeley_db" = "yes"; then AC_DEFINE(SVN_LIBSVN_FS_LINKS_FS_BASE, 1, [Defined if libsvn_fs should link against libsvn_fs_base])
diff --git a/subversion/include/private/svn_packed_data.h b/subversion/include/private/svn_packed_data.h new file mode 100644 index 0000000..838c9f6 --- /dev/null +++ b/subversion/include/private/svn_packed_data.h
@@ -0,0 +1,242 @@ +/* packed_data.h : Interface to the packed binary stream data structure + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "svn_string.h" +#include "svn_io.h" + +/* This API provides Yet Another Serialization Framework. + * + * It is geared towards efficiently encoding collections of structured + * binary data (e.g. an array of noderev objects). The basic idea is to + * transform them into hierarchies of streams with each stream usually + * corresponding to a single attribute in the original data structures. + * The user is free model the mapping structure <-> streams mapping as she + * sees fit. + * + * With all data inside the same (sub-)stream carrying similar attribute + * values, the whole stream lends itself to data compression. Strings / + * plain byte sequences will be stored as is. Numbers use a 7b/8b encoding + * scheme to eliminate leading zeros. Because values are often dependent + * (increasing offsets, roughly similar revision number, etc.), streams + * can be configured as storing (hopefully shorter) deltas instead of the + * original value. + * + * Two stream types are provided: integer and byte streams. While the + * first store 64 bit integers only and can be configured to assume + * signed and / or deltifyable data, the second will store arbitrary + * byte sequences including their length. At the root level, you may + * create an arbitrary number of integer and byte streams. Any stream + * may have an arbitrary number of sub-streams of the same kind. You + * should create the full stream hierarchy before writing any data to it. + * + * As a convenience, when an integer stream has sub-streams, you may write + * to the parent stream instead of all sub-streams individually and the + * values will be passed down automatically in a round-robin fashion. + * Reading from the parent stream is similarly supported. + * + * When all data has been added to the stream, it can be written to an + * ordinary svn_stream_t. First, we write a description of the stream + * structure (types, sub-streams, sizes and configurations) followed by + * zlib compressed stream content. For each top-level stream, all sub- + * stream data will be concatenated and then compressed as a single block. + * To maximize the effect of this, make sure all data in that stream + * hierarchy has a similar value distribution. + * + * Reading data starts with an svn_stream_t and automatically recreates + * the stream hierarchies. You only need to extract data from it in the + * same order as you wrote it. + * + * Although not enforced programmatically, you may either only write to a + * stream hierarchy or only read from it but you cannot do both on the + * same data structure. + */ + + + +/* We pack / unpack integers en block to minimize calling and setup overhead. + * This is the number of integers we put into a buffer before writing them + * them to / after reading them from the 7b/8b stream. Under 64 bits, this + * value creates a 128 byte data structure (14 + 2 integers, 8 bytes each). + */ +#define SVN__PACKED_DATA_BUFFER_SIZE 14 + + +/* Data types. */ + +/* Opaque type for the root object. + */ +typedef struct svn_packed__data_root_t svn_packed__data_root_t; + +/* Opaque type for byte streams. + */ +typedef struct svn_packed__byte_stream_t svn_packed__byte_stream_t; + +/* Semi-opaque type for integer streams. We expose the unpacked buffer + * to allow for replacing svn_packed__add_uint and friends by macros. + */ +typedef struct svn_packed__int_stream_t +{ + /* pointer to the remainder of the data structure */ + void *private_data; + + /* number of value entries in BUFFER */ + apr_size_t buffer_used; + + /* unpacked integers (either yet to be packed or pre-fetched from the + * packed buffers). Only the first BUFFER_USED entries are valid. */ + apr_uint64_t buffer[SVN__PACKED_DATA_BUFFER_SIZE]; +} svn_packed__int_stream_t; + + +/* Writing data. */ + +/* Return a new serialization root object, allocated in POOL. + */ +svn_packed__data_root_t * +svn_packed__data_create_root(apr_pool_t *pool); + +/* Create and return a new top-level integer stream in ROOT. If signed, + * negative numbers will be put into that stream, SIGNED_INTS should be + * TRUE as a more efficient encoding will be used in that case. Set + * DIFF to TRUE if you expect the difference between consecutive numbers + * to be much smaller (~100 times) than the actual numbers. + */ +svn_packed__int_stream_t * +svn_packed__create_int_stream(svn_packed__data_root_t *root, + svn_boolean_t diff, + svn_boolean_t signed_ints); + +/* Create and return a sub-stream to the existing integer stream PARENT. + * If signed, negative numbers will be put into that stream, SIGNED_INTS + * should be TRUE as a more efficient encoding will be used in that case. + * Set DIFF to TRUE if you expect the difference between consecutive numbers + * to be much smaller (~100 times) than the actual numbers. + */ +svn_packed__int_stream_t * +svn_packed__create_int_substream(svn_packed__int_stream_t *parent, + svn_boolean_t diff, + svn_boolean_t signed_ints); + +/* Create and return a new top-level byte sequence stream in ROOT. + */ +svn_packed__byte_stream_t * +svn_packed__create_bytes_stream(svn_packed__data_root_t *root); + +/* Write the unsigned integer VALUE to STEAM. + */ +void +svn_packed__add_uint(svn_packed__int_stream_t *stream, + apr_uint64_t value); + +/* Write the signed integer VALUE to STEAM. + */ +void +svn_packed__add_int(svn_packed__int_stream_t *stream, + apr_int64_t value); + +/* Write the sequence stating at DATA containing LEN bytes to STEAM. + */ +void +svn_packed__add_bytes(svn_packed__byte_stream_t *stream, + const char *data, + apr_size_t len); + +/* Write all contents of ROOT (including all sub-streams) to STREAM. + * Use SCRATCH_POOL for temporary allocations. + */ +svn_error_t * +svn_packed__data_write(svn_stream_t *stream, + svn_packed__data_root_t *root, + apr_pool_t *scratch_pool); + + +/* Reading data. */ + +/* Return the first integer stream in ROOT. Returns NULL in case there + * aren't any. + */ +svn_packed__int_stream_t * +svn_packed__first_int_stream(svn_packed__data_root_t *root); + +/* Return the first byte sequence stream in ROOT. Returns NULL in case + * there aren't any. + */ +svn_packed__byte_stream_t * +svn_packed__first_byte_stream(svn_packed__data_root_t *root); + +/* Return the next (sibling) integer stream to STREAM. Returns NULL in + * case there isn't any. + */ +svn_packed__int_stream_t * +svn_packed__next_int_stream(svn_packed__int_stream_t *stream); + +/* Return the next (sibling) byte sequence stream to STREAM. Returns NULL + * in case there isn't any. + */ +svn_packed__byte_stream_t * +svn_packed__next_byte_stream(svn_packed__byte_stream_t *stream); + +/* Return the first sub-stream of STREAM. Returns NULL in case there + * isn't any. + */ +svn_packed__int_stream_t * +svn_packed__first_int_substream(svn_packed__int_stream_t *stream); + +/* Return the number of integers left to read from STREAM. + */ +apr_size_t +svn_packed__int_count(svn_packed__int_stream_t *stream); + +/* Return the number of bytes left to read from STREAM. + */ +apr_size_t +svn_packed__byte_count(svn_packed__byte_stream_t *stream); + +/* Return the next number from STREAM as unsigned integer. Returns 0 when + * reading beyond the end of the stream. + */ +apr_uint64_t +svn_packed__get_uint(svn_packed__int_stream_t *stream); + +/* Return the next number from STREAM as signed integer. Returns 0 when + * reading beyond the end of the stream. + */ +apr_int64_t +svn_packed__get_int(svn_packed__int_stream_t *stream); + +/* Return the next byte sequence from STREAM and set *LEN to the length + * of that sequence. Sets *LEN to 0 when reading beyond the end of the + * stream. + */ +const char * +svn_packed__get_bytes(svn_packed__byte_stream_t *stream, + apr_size_t *len); + +/* Allocate a new packed data root in RESULT_POOL, read its structure and + * stream contents from STREAM and return it in *ROOT_P. Use SCRATCH_POOL + * for temporary allocations. + */ +svn_error_t * +svn_packed__data_read(svn_packed__data_root_t **root_p, + svn_stream_t *stream, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool);
diff --git a/subversion/include/private/svn_string_private.h b/subversion/include/private/svn_string_private.h index 49287f1..a8e06d6 100644 --- a/subversion/include/private/svn_string_private.h +++ b/subversion/include/private/svn_string_private.h
@@ -264,6 +264,58 @@ /** @} */ +/** Prefix trees. + * + * Prefix trees allow for a space-efficient representation of a set of path- + * like strings, i.e. those that share common prefixes. Any given string + * value will be stored only once, i.e. two strings stored in the same tree + * are equal if and only if the point to the same #svn_prefix_string__t. + * + * @defgroup svn_prefix_string Strings in prefix trees. +* @{ + */ + +/** + * Opaque data type for prefix-tree-based strings. + */ +typedef struct svn_prefix_string__t svn_prefix_string__t; + +/** + * Opaque data type representing a prefix tree + */ +typedef struct svn_prefix_tree__t svn_prefix_tree__t; + +/** + * Return a new prefix tree allocated in @a pool. + */ +svn_prefix_tree__t * +svn_prefix_tree__create(apr_pool_t *pool); + +/** + * Return a string with the value @a s stored in @a tree. If no such string + * exists yet, add it automatically. + */ +svn_prefix_string__t * +svn_prefix_string__create(svn_prefix_tree__t *tree, + const char *s); + +/** + * Return the contents of @a s as a new string object allocated in @a pool. + */ +svn_string_t * +svn_prefix_string__expand(const svn_prefix_string__t *s, + apr_pool_t *pool); + +/** + * Compare the two strings @a lhs and @a rhs that must be part of the same + * tree. + */ +int +svn_prefix_string__compare(const svn_prefix_string__t *lhs, + const svn_prefix_string__t *rhs); + +/** @} */ + /** @} */
diff --git a/subversion/include/private/svn_subr_private.h b/subversion/include/private/svn_subr_private.h index 71735e2..e21f795 100644 --- a/subversion/include/private/svn_subr_private.h +++ b/subversion/include/private/svn_subr_private.h
@@ -215,8 +215,7 @@ but implements the same basic sematics of a spillbuf for the underlying storage. */ svn_stream_t * -svn_stream__from_spillbuf(apr_size_t blocksize, - apr_size_t maxsize, +svn_stream__from_spillbuf(svn_spillbuf_t *buf, apr_pool_t *result_pool); /** @} */
diff --git a/subversion/include/svn_error_codes.h b/subversion/include/svn_error_codes.h index 2025970..f910666 100644 --- a/subversion/include/svn_error_codes.h +++ b/subversion/include/svn_error_codes.h
@@ -806,6 +806,36 @@ SVN_ERR_FS_CATEGORY_START + 52, "Could not initialize the revprop caching infrastructure.") + /** @since New in 1.9. */ + SVN_ERRDEF(SVN_ERR_FS_MALFORMED_TXN_ID, + SVN_ERR_FS_CATEGORY_START + 53, + "Malformed transaction ID string.") + + /** @since New in 1.9. */ + SVN_ERRDEF(SVN_ERR_FS_ITEM_INDEX_CORRUPTION, + SVN_ERR_FS_CATEGORY_START + 54, + "Corrupt index file.") + + /** @since New in 1.9. */ + SVN_ERRDEF(SVN_ERR_FS_ITEM_INDEX_REVISION, + SVN_ERR_FS_CATEGORY_START + 55, + "Revision not covered by index.") + + /** @since New in 1.9. */ + SVN_ERRDEF(SVN_ERR_FS_ITEM_INDEX_OVERFLOW, + SVN_ERR_FS_CATEGORY_START + 56, + "Item index too large for this revision.") + + /** @since New in 1.9. */ + SVN_ERRDEF(SVN_ERR_FS_CONTAINER_INDEX, + SVN_ERR_FS_CATEGORY_START + 57, + "Container index out of range.") + + /** @since New in 1.9. */ + SVN_ERRDEF(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT, + SVN_ERR_FS_CATEGORY_START + 58, + "Index files are inconsistent.") + /* repos errors */ SVN_ERRDEF(SVN_ERR_REPOS_LOCKED, @@ -1355,6 +1385,11 @@ SVN_ERR_MISC_CATEGORY_START + 40, "Bad arguments to SQL operators GLOB or LIKE") + /** @since New in 1.9. */ + SVN_ERRDEF(SVN_ERR_CORRUPT_PACKED_DATA, + SVN_ERR_MISC_CATEGORY_START + 41, + "Packed data stream is corrupt") + /* command-line client errors */ SVN_ERRDEF(SVN_ERR_CL_ARG_PARSING_ERROR,
diff --git a/subversion/include/svn_fs.h b/subversion/include/svn_fs.h index d1b1b84..1663632 100644 --- a/subversion/include/svn_fs.h +++ b/subversion/include/svn_fs.h
@@ -121,6 +121,16 @@ /** @since New in 1.1. */ #define SVN_FS_TYPE_FSFS "fsfs" +/** + * EXPERIMENTAL filesystem backend. + * + * It is not ready for general production use. Please consult the + * respective release notes on suggested usage scenarios. + * + * @since New in 1.9. + */ +#define SVN_FS_TYPE_FSX "fsx" + /** Create repository format compatible with Subversion versions * earlier than 1.4. * @@ -148,6 +158,18 @@ * @since New in 1.8. */ #define SVN_FS_CONFIG_PRE_1_8_COMPATIBLE "pre-1.8-compatible" + +/** Create repository format compatible with Subversion versions + * earlier than 1.9. The value must be a version in the same format + * as #SVN_VER_NUMBER. + * + * @note The @c patch component would often be ignored, due to our forward + * compatibility promises within minor release lines. It should therefore + * usually be set to @c 0. + * + * @since New in 1.9. + */ +#define SVN_FS_CONFIG_COMPATIBLE_VERSION "compatible-version" /** @} */ @@ -222,6 +244,7 @@ * * SVN_FS_TYPE_BDB Berkeley-DB implementation * SVN_FS_TYPE_FSFS Native-filesystem implementation + * SVN_FS_TYPE_FSX Experimental filesystem implementation * * If @a fs_config is @c NULL or does not contain a value for * #SVN_FS_CONFIG_FS_TYPE then the default filesystem type will be used. @@ -2662,6 +2685,32 @@ or svn_fs_info_dup(). */ } svn_fs_fsfs_info_t; +/** + * A structure that provides some information about a filesystem. + * Returned by svn_fs_info() for #SVN_FS_TYPE_FSX filesystems. + * + * @note Fields may be added to the end of this structure in future + * versions. Therefore, users shouldn't allocate structures of this + * type, to preserve binary compatibility. + * + * @since New in 1.9. + */ +typedef struct svn_fs_fsx_info_t { + + /** Filesystem backend (#fs_type), i.e., the string #SVN_FS_TYPE_FSX. */ + const char *fs_type; + + /** Shard size, always > 0. */ + int shard_size; + + /** The smallest revision which is not in a pack file. */ + svn_revnum_t min_unpacked_rev; + + /* If you add fields here, check whether you need to extend svn_fs_info() + or svn_fs_info_dup(). */ + +} svn_fs_fsx_info_t; + /** @see svn_fs_info() * @since New in 1.9. */ typedef struct svn_fs_info_placeholder_t {
diff --git a/subversion/include/svn_sorts.h b/subversion/include/svn_sorts.h index 31ad8c5..558383a 100644 --- a/subversion/include/svn_sorts.h +++ b/subversion/include/svn_sorts.h
@@ -217,6 +217,77 @@ svn_sort__array_reverse(apr_array_header_t *array, apr_pool_t *scratch_pool); +/** Priority queues. + * + * @defgroup svn_priority_queue__t Priority Queues + * @{ + */ + +/** + * We implement priority queues on top of existing ELEMENTS arrays. They + * provide us with memory management and very basic element type information. + * + * The extraction order is being defined by a comparison function similar + * to the ones used with qsort. The first element in the queue is always + * on with COMPARISON_FUNC(first,element) <= 0, for all elements in the + * queue. + */ + +/** + * Opaque data type for priority queues. + */ +typedef struct svn_priority_queue__t svn_priority_queue__t; + +/** + * Return a priority queue containing all provided @a elements and prioritize + * them according to @a compare_func. + * + * @note The priority queue will use the existing @a elements array for data + * storage. So, you must not manipulate that array while using the queue. + * Also, the lifetime of the queue is bound to that of the array. + */ +svn_priority_queue__t * +svn_priority_queue__create(apr_array_header_t *elements, + int (*compare_func)(const void *, const void *)); + +/** + * Returns the number of elements in the @a queue. + */ +apr_size_t +svn_priority_queue__size(svn_priority_queue__t *queue); + +/** + * Returns a reference to the first element in the @a queue. The queue + * contents remains unchanged. If the @a queue is empty, #NULL will be + * returned. + */ +void * +svn_priority_queue__peek(svn_priority_queue__t *queue); + +/** + * Notify the @a queue after modifying the first item as returned by + * #svn_priority_queue__peek. + */ +void +svn_priority_queue__update(svn_priority_queue__t *queue); + +/** + * Remove the first element from the @a queue. This is a no-op for empty + * queues. + */ +void +svn_priority_queue__pop(svn_priority_queue__t *queue); + +/** + * Append the new @a element to the @a queue. @a element must neither be + * #NULL nor the first element as returned by #svn_priority_queue__peek. + */ +void +svn_priority_queue__push(svn_priority_queue__t *queue, const void *element); + +/** @} */ + + #ifdef __cplusplus } #endif /* __cplusplus */
diff --git a/subversion/include/svn_string.h b/subversion/include/svn_string.h index d532d57..9130ee6 100644 --- a/subversion/include/svn_string.h +++ b/subversion/include/svn_string.h
@@ -313,6 +313,16 @@ const char *bytes, apr_size_t count); +/** Append @a byte @a count times onto @a targetstr. + * + * reallocs if necessary. @a targetstr is affected, nothing else is. + * @since New in 1.9. + */ +void +svn_stringbuf_appendfill(svn_stringbuf_t *targetstr, + char byte, + apr_size_t count); + /** Append the stringbuf @c appendstr onto @a targetstr. * * reallocs if necessary. @a targetstr is affected, nothing else is.
diff --git a/subversion/libsvn_fs/fs-loader.c b/subversion/libsvn_fs/fs-loader.c index 22f88bc..4de9c84 100644 --- a/subversion/libsvn_fs/fs-loader.c +++ b/subversion/libsvn_fs/fs-loader.c
@@ -84,6 +84,17 @@ NULL }; +static struct fs_type_defn fsx_defn = + { + SVN_FS_TYPE_FSX, "x", +#ifdef SVN_LIBSVN_FS_LINKS_FS_X + svn_fs_x__init, +#else + NULL, +#endif + &base_defn + }; + static struct fs_type_defn fsfs_defn = { SVN_FS_TYPE_FSFS, "fs", @@ -92,7 +103,7 @@ #else NULL, #endif - &base_defn + &fsx_defn }; static struct fs_type_defn *fs_modules = &fsfs_defn;
diff --git a/subversion/libsvn_fs/fs-loader.h b/subversion/libsvn_fs/fs-loader.h index 664019e..5ded9e8 100644 --- a/subversion/libsvn_fs/fs-loader.h +++ b/subversion/libsvn_fs/fs-loader.h
@@ -171,6 +171,9 @@ svn_error_t *svn_fs_fs__init(const svn_version_t *loader_version, fs_library_vtable_t **vtable, apr_pool_t* common_pool); +svn_error_t *svn_fs_x__init(const svn_version_t *loader_version, + fs_library_vtable_t **vtable, + apr_pool_t* common_pool);
diff --git a/subversion/libsvn_fs_base/lock.c b/subversion/libsvn_fs_base/lock.c index 451f322..c15c7f8 100644 --- a/subversion/libsvn_fs_base/lock.c +++ b/subversion/libsvn_fs_base/lock.c
@@ -465,8 +465,9 @@ args.path = svn_fs__canonicalize_abspath(path, pool); args.depth = depth; /* Enough for 100+ locks if the comments are small. */ - args.stream = svn_stream__from_spillbuf(4 * 1024 /* blocksize */, - 64 * 1024 /* maxsize */, + args.stream = svn_stream__from_spillbuf(svn_spillbuf__create(4 * 1024 /* blocksize */, + 64 * 1024 /* maxsize */, + pool), pool); SVN_ERR(svn_fs_base__retry_txn(fs, txn_body_get_locks, &args, FALSE, pool));
diff --git a/subversion/libsvn_fs_x/cached_data.c b/subversion/libsvn_fs_x/cached_data.c new file mode 100644 index 0000000..bc7d9b6 --- /dev/null +++ b/subversion/libsvn_fs_x/cached_data.c
@@ -0,0 +1,2752 @@ +/* cached_data.c --- cached (read) access to FSX data + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "cached_data.h" + +#include <assert.h> + +#include "svn_hash.h" +#include "svn_ctype.h" +#include "private/svn_temp_serializer.h" + +#include "fs_x.h" +#include "low_level.h" +#include "util.h" +#include "pack.h" +#include "temp_serializer.h" +#include "index.h" +#include "changes.h" +#include "noderevs.h" +#include "reps.h" + +#include "../libsvn_fs/fs-loader.h" + +#include "svn_private_config.h" + +/* forward-declare */ +static svn_error_t * +block_read(void **result, + svn_fs_t *fs, + svn_revnum_t revision, + apr_uint64_t item_index, + apr_file_t *revision_file, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + + +/* Defined this to enable access logging via dgb__log_access +#define SVN_FS_X__LOG_ACCESS +*/ + +/* When SVN_FS_X__LOG_ACCESS has been defined, write a line to console + * showing where REVISION, ITEM_INDEX is located in FS and use ITEM to + * show details on it's contents if not NULL. To support format 6 and + * earlier repos, ITEM_TYPE (SVN_FS_X__ITEM_TYPE_*) must match ITEM. + * Use SCRATCH_POOL for temporary allocations. + * + * For pre-format7 repos, the display will be restricted. + */ +static svn_error_t * +dgb__log_access(svn_fs_t *fs, + svn_revnum_t revision, + apr_uint64_t item_index, + void *item, + int item_type, + apr_pool_t *scratch_pool) +{ + /* no-op if this macro is not defined */ +#ifdef SVN_FS_X__LOG_ACCESS + fs_x_data_t *ffd = fs->fsap_data; + apr_off_t offset = -1; + apr_off_t end_offset = 0; + apr_uint32_t sub_item = 0; + apr_array_header_t *entries; + svn_fs_x__p2l_entry_t *entry = NULL; + int i; + static const char *types[] = {"<n/a>", "frep ", "drep ", "fprop", "dprop", + "node ", "chgs ", "rep ", "c:", "n:", "r:"}; + const char *description = ""; + const char *type = types[item_type]; + const char *pack = ""; + + /* determine rev / pack file offset */ + SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, revision, NULL, + item_index, scratch_pool)); + + /* constructing the pack file description */ + if (revision < ffd->min_unpacked_rev) + pack = apr_psprintf(scratch_pool, "%4ld|", + revision / ffd->max_files_per_dir); + + /* construct description if possible */ + if (item_type == SVN_FS_X__ITEM_TYPE_NODEREV && item != NULL) + { + node_revision_t *node = item; + const char *data_rep + = node->data_rep + ? apr_psprintf(scratch_pool, " d=%ld/%" APR_UINT64_T_FMT, + node->data_rep->revision, + node->data_rep->item_index) + : ""; + const char *prop_rep + = node->prop_rep + ? apr_psprintf(scratch_pool, " p=%ld/%" APR_UINT64_T_FMT, + node->prop_rep->revision, + node->prop_rep->item_index) + : ""; + description = apr_psprintf(scratch_pool, "%s (pc=%d%s%s)", + node->created_path, + node->predecessor_count, + data_rep, + prop_rep); + } + else if (item_type == SVN_FS_X__ITEM_TYPE_ANY_REP) + { + svn_fs_x__rep_header_t *header = item; + if (header == NULL) + description = " (txdelta window)"; + else if (header->type == svn_fs_x__rep_self_delta) + description = " DELTA"; + else + description = apr_psprintf(scratch_pool, + " DELTA against %ld/%" APR_UINT64_T_FMT, + header->base_revision, + header->base_item_index); + } + else if (item_type == SVN_FS_X__ITEM_TYPE_CHANGES && item != NULL) + { + apr_array_header_t *changes = item; + switch (changes->nelts) + { + case 0: description = " no change"; + break; + case 1: description = " 1 change"; + break; + default: description = apr_psprintf(scratch_pool, " %d changes", + changes->nelts); + } + } + + /* some info is only available in format7 repos */ + if (ffd->format >= SVN_FS_X__MIN_LOG_ADDRESSING_FORMAT) + { + /* reverse index lookup: get item description in ENTRY */ + SVN_ERR(svn_fs_x__p2l_entry_lookup(&entry, fs, revision, offset, + scratch_pool)); + if (entry) + { + /* more details */ + end_offset = offset + entry->size; + type = types[entry->type]; + + /* merge the sub-item number with the container type */ + if ( entry->type == SVN_FS_X__ITEM_TYPE_CHANGES_CONT + || entry->type == SVN_FS_X__ITEM_TYPE_NODEREVS_CONT + || entry->type == SVN_FS_X__ITEM_TYPE_REPS_CONT) + type = apr_psprintf(scratch_pool, "%s%-3d", type, sub_item); + } + + /* line output */ + printf("%5s%4lx:%04lx -%4lx:%04lx %s %7ld %5"APR_UINT64_T_FMT" %s\n", + pack, (long)(offset / ffd->block_size), + (long)(offset % ffd->block_size), + (long)(end_offset / ffd->block_size), + (long)(end_offset % ffd->block_size), + type, revision, item_index, description); + } + else + { + /* reduced logging for format 6 and earlier */ + printf("%5s%10" APR_UINT64_T_HEX_FMT " %s %7ld %7" APR_UINT64_T_FMT \ + " %s\n", + pack, (apr_uint64_t)(offset), type, revision, item_index, + description); + } + +#endif + + return SVN_NO_ERROR; +} + +/* Convenience wrapper around svn_io_file_aligned_seek, taking filesystem + FS instead of a block size. */ +static svn_error_t * +aligned_seek(svn_fs_t *fs, + apr_file_t *file, + apr_off_t *buffer_start, + apr_off_t offset, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + return svn_error_trace(svn_io_file_aligned_seek(file, ffd->block_size, + buffer_start, offset, + pool)); +} + +/* Open the revision file for revision REV in filesystem FS and store + the newly opened file in FILE. Seek to location OFFSET before + returning. Perform temporary allocations in POOL. */ +static svn_error_t * +open_and_seek_revision(apr_file_t **file, + svn_fs_t *fs, + svn_revnum_t rev, + apr_uint64_t item, + apr_pool_t *pool) +{ + apr_file_t *rev_file; + apr_off_t offset = -1; + apr_uint32_t sub_item = 0; + + SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, pool)); + + SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, rev, pool)); + SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, rev, NULL, item, + pool)); + SVN_ERR(aligned_seek(fs, rev_file, NULL, offset, pool)); + + *file = rev_file; + + return SVN_NO_ERROR; +} + +/* Open the representation REP for a node-revision in filesystem FS, seek + to its position and store the newly opened file in FILE. Perform + temporary allocations in POOL. */ +static svn_error_t * +open_and_seek_transaction(apr_file_t **file, + svn_fs_t *fs, + representation_t *rep, + apr_pool_t *pool) +{ + apr_file_t *rev_file; + apr_off_t offset; + apr_uint32_t sub_item = 0; + + SVN_ERR(svn_io_file_open(&rev_file, + svn_fs_x__path_txn_proto_rev(fs, &rep->txn_id, + pool), + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool)); + + SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, SVN_INVALID_REVNUM, + &rep->txn_id, rep->item_index, pool)); + SVN_ERR(aligned_seek(fs, rev_file, NULL, offset, pool)); + + *file = rev_file; + + return SVN_NO_ERROR; +} + +/* Given a node-id ID, and a representation REP in filesystem FS, open + the correct file and seek to the correction location. Store this + file in *FILE_P. Perform any allocations in POOL. */ +static svn_error_t * +open_and_seek_representation(apr_file_t **file_p, + svn_fs_t *fs, + representation_t *rep, + apr_pool_t *pool) +{ + if (! svn_fs_x__id_txn_used(&rep->txn_id)) + return open_and_seek_revision(file_p, fs, rep->revision, rep->item_index, + pool); + else + return open_and_seek_transaction(file_p, fs, rep, pool); +} + + + +static svn_error_t * +err_dangling_id(svn_fs_t *fs, const svn_fs_id_t *id) +{ + svn_string_t *id_str = svn_fs_x__id_unparse(id, fs->pool); + return svn_error_createf + (SVN_ERR_FS_ID_NOT_FOUND, 0, + _("Reference to non-existent node '%s' in filesystem '%s'"), + id_str->data, fs->path); +} + +/* Get the node-revision for the node ID in FS. + Set *NODEREV_P to the new node-revision structure, allocated in POOL. + See svn_fs_x__get_node_revision, which wraps this and adds another + error. */ +static svn_error_t * +get_node_revision_body(node_revision_t **noderev_p, + svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool) +{ + apr_file_t *revision_file; + svn_error_t *err; + svn_boolean_t is_cached = FALSE; + fs_x_data_t *ffd = fs->fsap_data; + + if (svn_fs_x__id_is_txn(id)) + { + /* This is a transaction node-rev. Its storage logic is very + different from that of rev / pack files. */ + err = svn_io_file_open(&revision_file, + svn_fs_x__path_txn_node_rev(fs, id, pool), + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool); + if (err) + { + if (APR_STATUS_IS_ENOENT(err->apr_err)) + { + svn_error_clear(err); + return svn_error_trace(err_dangling_id(fs, id)); + } + + return svn_error_trace(err); + } + + SVN_ERR(svn_fs_x__read_noderev(noderev_p, + svn_stream_from_aprfile2(revision_file, + FALSE, + pool), + pool)); + } + else + { + /* noderevs in rev / pack files can be cached */ + const svn_fs_x__id_part_t *rev_item = svn_fs_x__id_rev_item(id); + pair_cache_key_t key; + + /* First, try a noderevs container cache lookup. */ + if ( svn_fs_x__is_packed_rev(fs, rev_item->revision) + && ffd->noderevs_container_cache) + { + apr_off_t offset; + apr_uint32_t sub_item; + SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, + rev_item->revision, NULL, + rev_item->number, pool)); + key.revision = svn_fs_x__packed_base_rev(fs, rev_item->revision); + key.second = offset; + + SVN_ERR(svn_cache__get_partial((void **)noderev_p, &is_cached, + ffd->noderevs_container_cache, &key, + svn_fs_x__noderevs_get_func, + &sub_item, pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + key.revision = rev_item->revision; + key.second = rev_item->number; + + /* Not found or not applicable. Try a noderev cache lookup. + * If that succeeds, we are done here. */ + if (ffd->node_revision_cache) + { + SVN_ERR(svn_cache__get((void **) noderev_p, + &is_cached, + ffd->node_revision_cache, + &key, + pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + /* someone needs to read the data from this file: */ + err = open_and_seek_revision(&revision_file, fs, + rev_item->revision, + rev_item->number, + pool); + + /* block-read will parse the whole block and will also return + the one noderev that we need right now. */ + SVN_ERR(block_read((void **)noderev_p, fs, + rev_item->revision, + rev_item->number, + revision_file, + pool, + pool)); + SVN_ERR(svn_io_file_close(revision_file, pool)); + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__get_node_revision(node_revision_t **noderev_p, + svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool) +{ + const svn_fs_x__id_part_t *rev_item = svn_fs_x__id_rev_item(id); + + svn_error_t *err = get_node_revision_body(noderev_p, fs, id, pool); + if (err && err->apr_err == SVN_ERR_FS_CORRUPT) + { + svn_string_t *id_string = svn_fs_x__id_unparse(id, pool); + return svn_error_createf(SVN_ERR_FS_CORRUPT, err, + "Corrupt node-revision '%s'", + id_string->data); + } + + SVN_ERR(dgb__log_access(fs, + rev_item->revision, + rev_item->number, + *noderev_p, + SVN_FS_X__ITEM_TYPE_NODEREV, + pool)); + + return svn_error_trace(err); +} + + +svn_error_t * +svn_fs_x__rev_get_root(svn_fs_id_t **root_id_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, pool)); + *root_id_p = svn_fs_x__id_create_root(rev, pool); + + return SVN_NO_ERROR; +} + +/* Describes a lazily opened rev / pack file. Instances will be shared + between multiple instances of rep_state_t. */ +typedef struct shared_file_t +{ + /* The opened file. NULL while file is not open, yet. */ + apr_file_t *file; + + /* Stream wrapper around FILE. NULL while file is not open, yet. */ + svn_stream_t *stream; + + /* file system to open the file in */ + svn_fs_t *fs; + + /* revision contained in the file */ + svn_revnum_t revision; + + /* pool to use when creating the FILE. This guarantees that the file + remains open / valid beyond the respective local context that required + the file to be opened eventually. */ + apr_pool_t *pool; +} shared_file_t; + +/* Represents where in the current svndiff data block each + representation is. */ +typedef struct rep_state_t +{ + /* shared lazy-open rev/pack file structure */ + shared_file_t *file; + /* The txdelta window cache to use or NULL. */ + svn_cache__t *window_cache; + /* Caches un-deltified windows. May be NULL. */ + svn_cache__t *combined_cache; + /* revision containing the representation */ + svn_revnum_t revision; + /* representation's item index in REVISION */ + apr_uint64_t item_index; + /* length of the header at the start of the rep. + 0 iff this is rep is stored in a container + (i.e. does not have a header) */ + apr_size_t header_size; + apr_off_t start; /* The starting offset for the raw + svndiff data minus header. + -1 if the offset is yet unknwon. */ + /* sub-item index in case the rep is containered */ + apr_uint32_t sub_item; + apr_off_t current;/* The current offset relative to start. */ + apr_off_t size; /* Final value of CURRENT. */ + int ver; /* If a delta, what svndiff version? + -1 for unknown delta version. */ + int chunk_index; /* number of the window to read */ +} rep_state_t; + +/* See create_rep_state, which wraps this and adds another error. */ +static svn_error_t * +create_rep_state_body(rep_state_t **rep_state, + svn_fs_x__rep_header_t **rep_header, + shared_file_t **shared_file, + representation_t *rep, + svn_fs_t *fs, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + rep_state_t *rs = apr_pcalloc(pool, sizeof(*rs)); + svn_fs_x__rep_header_t *rh; + svn_boolean_t is_cached = FALSE; + + /* If the hint is + * - given, + * - refers to a valid revision, + * - refers to a packed revision, + * - as does the rep we want to read, and + * - refers to the same pack file as the rep + * we can re-use the same, already open file object + */ + svn_boolean_t reuse_shared_file + = shared_file && *shared_file && (*shared_file)->file + && SVN_IS_VALID_REVNUM((*shared_file)->revision) + && (*shared_file)->revision < ffd->min_unpacked_rev + && rep->revision < ffd->min_unpacked_rev + && ( ((*shared_file)->revision / ffd->max_files_per_dir) + == (rep->revision / ffd->max_files_per_dir)); + + representation_cache_key_t key; + key.revision = rep->revision; + key.is_packed = rep->revision < ffd->min_unpacked_rev; + key.item_index = rep->item_index; + + /* continue constructing RS and RA */ + rs->size = rep->size; + rs->revision = rep->revision; + rs->item_index = rep->item_index; + rs->window_cache = ffd->txdelta_window_cache; + rs->combined_cache = ffd->combined_window_cache; + rs->ver = -1; + rs->start = -1; + + if (ffd->rep_header_cache && !svn_fs_x__id_txn_used(&rep->txn_id)) + SVN_ERR(svn_cache__get((void **) &rh, &is_cached, + ffd->rep_header_cache, &key, pool)); + + if (is_cached) + { + if (reuse_shared_file) + { + rs->file = *shared_file; + } + else + { + shared_file_t *file = apr_pcalloc(pool, sizeof(*file)); + file->revision = rep->revision; + file->pool = pool; + file->fs = fs; + rs->file = file; + + /* remember the current file, if suggested by the caller */ + if (shared_file) + *shared_file = file; + } + } + else + { + /* we will need the on-disk location for non-txn reps */ + apr_off_t offset; + apr_uint32_t sub_item; + if (! svn_fs_x__id_txn_used(&rep->txn_id)) + SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, + fs, rep->revision, NULL, + rep->item_index, pool)); + + /* is rep stored in some star-deltified container? */ + if (! svn_fs_x__id_txn_used(&rep->txn_id)) + { + svn_boolean_t in_container = TRUE; + if (sub_item == 0) + { + svn_fs_x__p2l_entry_t *entry; + SVN_ERR(svn_fs_x__p2l_entry_lookup(&entry, fs, rep->revision, + offset, pool)); + in_container = entry->type == SVN_FS_X__ITEM_TYPE_REPS_CONT; + } + + if (in_container) + { + /* construct a container rep header */ + *rep_header = apr_pcalloc(pool, sizeof(**rep_header)); + (*rep_header)->type = svn_fs_x__rep_container; + + /* provide an empty shared file struct */ + rs->file = apr_pcalloc(pool, sizeof(*rs->file)); + rs->file->revision = rep->revision; + rs->file->pool = pool; + rs->file->fs = fs; + + /* exit to caller */ + *rep_state = rs; + return SVN_NO_ERROR; + } + } + + if (reuse_shared_file) + { + /* ... we can re-use the same, already open file object + */ + SVN_ERR_ASSERT(sub_item == 0); + SVN_ERR(aligned_seek(fs, (*shared_file)->file, NULL, offset, pool)); + + rs->file = *shared_file; + } + else + { + shared_file_t *file = apr_pcalloc(pool, sizeof(*file)); + file->revision = rep->revision; + file->pool = pool; + file->fs = fs; + + /* otherwise, create a new file object + */ + SVN_ERR(open_and_seek_representation(&file->file, fs, rep, pool)); + file->stream = svn_stream_from_aprfile2(file->file, TRUE, + file->pool); + rs->file = file; + + /* remember the current file, if suggested by the caller */ + if (shared_file) + *shared_file = file; + } + + SVN_ERR(svn_fs_x__read_rep_header(&rh, rs->file->stream, pool)); + SVN_ERR(svn_fs_x__get_file_offset(&rs->start, rs->file->file, pool)); + + if (! svn_fs_x__id_txn_used(&rep->txn_id)) + { + SVN_ERR(block_read(NULL, fs, rep->revision, rep->item_index, + rs->file->file, pool, pool)); + if (ffd->rep_header_cache) + SVN_ERR(svn_cache__set(ffd->rep_header_cache, &key, rh, pool)); + } + } + + SVN_ERR(dgb__log_access(fs, rep->revision, rep->item_index, rh, + SVN_FS_X__ITEM_TYPE_ANY_REP, pool)); + + rs->header_size = rh->header_size; + *rep_state = rs; + *rep_header = rh; + + /* We are dealing with a delta, find out what version. */ + rs->chunk_index = 0; + rs->current = 4; + + return SVN_NO_ERROR; +} + +/* Read the rep args for REP in filesystem FS and create a rep_state + for reading the representation. Return the rep_state in *REP_STATE + and the rep args in *REP_ARGS, both allocated in POOL. + + When reading multiple reps, i.e. a skip delta chain, you may provide + non-NULL SHARED_FILE. (If SHARED_FILE is not NULL, in the first + call it should be a pointer to NULL.) The function will use this + variable to store the previous call results and tries to re-use it. + This may result in significant savings in I/O for packed files and + number of open file handles. + */ +static svn_error_t * +create_rep_state(rep_state_t **rep_state, + svn_fs_x__rep_header_t **rep_header, + shared_file_t **shared_file, + representation_t *rep, + svn_fs_t *fs, + apr_pool_t *pool) +{ + svn_error_t *err = create_rep_state_body(rep_state, rep_header, + shared_file, rep, fs, pool); + if (err && err->apr_err == SVN_ERR_FS_CORRUPT) + { + fs_x_data_t *ffd = fs->fsap_data; + + /* ### This always returns "-1" for transaction reps, because + ### this particular bit of code doesn't know if the rep is + ### stored in the protorev or in the mutable area (for props + ### or dir contents). It is pretty rare for FSX to *read* + ### from the protorev file, though, so this is probably OK. + ### And anyone going to debug corruption errors is probably + ### going to jump straight to this comment anyway! */ + return svn_error_createf(SVN_ERR_FS_CORRUPT, err, + "Corrupt representation '%s'", + rep + ? svn_fs_x__unparse_representation + (rep, ffd->format, TRUE, pool)->data + : "(null)"); + } + /* ### Call representation_string() ? */ + return svn_error_trace(err); +} + +svn_error_t * +svn_fs_x__check_rep(representation_t *rep, + svn_fs_t *fs, + void **hint, + apr_pool_t *pool) +{ + rep_state_t *rs; + svn_fs_x__rep_header_t *rep_header; + + /* ### Should this be using read_rep_line() directly? */ + SVN_ERR(create_rep_state(&rs, &rep_header, (shared_file_t**)hint, rep, + fs, pool)); + + return SVN_NO_ERROR; +} + +/* . + Do any allocations in POOL. */ +svn_error_t * +svn_fs_x__rep_chain_length(int *chain_length, + representation_t *rep, + svn_fs_t *fs, + apr_pool_t *pool) +{ + int count = 0; + apr_pool_t *sub_pool = svn_pool_create(pool); + svn_boolean_t is_delta = FALSE; + + /* Check whether the length of the deltification chain is acceptable. + * Otherwise, shared reps may form a non-skipping delta chain in + * extreme cases. */ + representation_t base_rep = *rep; + + /* re-use open files between iterations */ + shared_file_t *file_hint = NULL; + + svn_fs_x__rep_header_t *header; + + /* follow the delta chain towards the end but for at most + * MAX_CHAIN_LENGTH steps. */ + do + { + rep_state_t *rep_state; + SVN_ERR(create_rep_state_body(&rep_state, + &header, + &file_hint, + &base_rep, + fs, + sub_pool)); + + base_rep.revision = header->base_revision; + base_rep.item_index = header->base_item_index; + base_rep.size = header->base_length; + svn_fs_x__id_txn_reset(&base_rep.txn_id); + is_delta = header->type == svn_fs_x__rep_delta; + + ++count; + if (count % 16 == 0) + { + file_hint = NULL; + svn_pool_clear(sub_pool); + } + } + while (is_delta && base_rep.revision); + + *chain_length = count; + svn_pool_destroy(sub_pool); + + return SVN_NO_ERROR; +} + + +struct rep_read_baton +{ + /* The FS from which we're reading. */ + svn_fs_t *fs; + + /* If not NULL, this is the base for the first delta window in rs_list */ + svn_stringbuf_t *base_window; + + /* The state of all prior delta representations. */ + apr_array_header_t *rs_list; + + /* The plaintext state, if there is a plaintext. */ + rep_state_t *src_state; + + /* The index of the current delta chunk, if we are reading a delta. */ + int chunk_index; + + /* The buffer where we store undeltified data. */ + char *buf; + apr_size_t buf_pos; + apr_size_t buf_len; + + /* A checksum context for summing the data read in order to verify it. + Note: we don't need to use the sha1 checksum because we're only doing + data verification, for which md5 is perfectly safe. */ + svn_checksum_ctx_t *md5_checksum_ctx; + + svn_boolean_t checksum_finalized; + + /* The stored checksum of the representation we are reading, its + length, and the amount we've read so far. Some of this + information is redundant with rs_list and src_state, but it's + convenient for the checksumming code to have it here. */ + unsigned char md5_digest[APR_MD5_DIGESTSIZE]; + + svn_filesize_t len; + svn_filesize_t off; + + /* The key for the fulltext cache for this rep, if there is a + fulltext cache. */ + pair_cache_key_t fulltext_cache_key; + /* The text we've been reading, if we're going to cache it. */ + svn_stringbuf_t *current_fulltext; + + /* Used for temporary allocations during the read. */ + apr_pool_t *pool; + + /* Pool used to store file handles and other data that is persistant + for the entire stream read. */ + apr_pool_t *filehandle_pool; +}; + +/* Set window key in *KEY to address the window described by RS. + For convenience, return the KEY. */ +static window_cache_key_t * +get_window_key(window_cache_key_t *key, rep_state_t *rs) +{ + assert(rs->revision <= APR_UINT32_MAX); + key->revision = (apr_uint32_t)rs->revision; + key->item_index = rs->item_index; + key->chunk_index = rs->chunk_index; + + return key; +} + +/* Read the WINDOW_P number CHUNK_INDEX for the representation given in + * rep state RS from the current FSX session's cache. This will be a + * no-op and IS_CACHED will be set to FALSE if no cache has been given. + * If a cache is available IS_CACHED will inform the caller about the + * success of the lookup. Allocations (of the window in particualar) will + * be made from POOL. + * + * If the information could be found, put RS to CHUNK_INDEX. + */ + +/* Return data type for get_cached_window_sizes_func. + */ +typedef struct window_sizes_t +{ + /* length of the txdelta window in its on-disk format */ + svn_filesize_t packed_len; + + /* expanded (and combined) window length */ + svn_filesize_t target_len; +} window_sizes_t; + +/* Implements svn_cache__partial_getter_func_t extracting the packed + * and expanded window sizes from a cached window and return the size + * info as a window_sizes_t* in *OUT. + */ +static svn_error_t * +get_cached_window_sizes_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool) +{ + const svn_fs_x__txdelta_cached_window_t *window = data; + const svn_txdelta_window_t *txdelta_window + = svn_temp_deserializer__ptr(window, (const void **)&window->window); + + window_sizes_t *result = apr_palloc(pool, sizeof(*result)); + result->packed_len = window->end_offset - window->start_offset; + result->target_len = txdelta_window->tview_len; + + *out = result; + + return SVN_NO_ERROR; +} + +/* Return the packed & expanded sizes of the window addressed by RS. If the + * window cannot be found in the window cache, set *IS_CACHED to FALSE. + * Otherwise, set it to TRUE and return the data in *SIZES, allocated in POOL. + */ +static svn_error_t * +get_cached_window_sizes(window_sizes_t **sizes, + rep_state_t *rs, + svn_boolean_t *is_cached, + apr_pool_t *pool) +{ + if (! rs->window_cache) + { + /* txdelta window has not been enabled */ + *is_cached = FALSE; + } + else + { + window_cache_key_t key = { 0 }; + SVN_ERR(svn_cache__get_partial((void **)sizes, + is_cached, + rs->window_cache, + get_window_key(&key, rs), + get_cached_window_sizes_func, + NULL, + pool)); + } + + return SVN_NO_ERROR; +} + +static svn_error_t * +get_cached_window(svn_txdelta_window_t **window_p, + rep_state_t *rs, + int chunk_index, + svn_boolean_t *is_cached, + apr_pool_t *pool) +{ + if (! rs->window_cache) + { + /* txdelta window has not been enabled */ + *is_cached = FALSE; + } + else + { + /* ask the cache for the desired txdelta window */ + svn_fs_x__txdelta_cached_window_t *cached_window; + window_cache_key_t key = { 0 }; + get_window_key(&key, rs); + key.chunk_index = chunk_index; + SVN_ERR(svn_cache__get((void **) &cached_window, + is_cached, + rs->window_cache, + &key, + pool)); + + if (*is_cached) + { + /* found it. Pass it back to the caller. */ + *window_p = cached_window->window; + + /* manipulate the RS as if we just read the data */ + rs->current = cached_window->end_offset; + rs->chunk_index = chunk_index; + } + } + + return SVN_NO_ERROR; +} + +/* Store the WINDOW read for the rep state RS with the given START_OFFSET + * within the pack / rev file in the current FSX session's cache. This + * will be a no-op if no cache has been given. + * Temporary allocations will be made from SCRATCH_POOL. */ +static svn_error_t * +set_cached_window(svn_txdelta_window_t *window, + rep_state_t *rs, + apr_off_t start_offset, + apr_pool_t *scratch_pool) +{ + if (rs->window_cache) + { + /* store the window and the first offset _past_ it */ + svn_fs_x__txdelta_cached_window_t cached_window; + window_cache_key_t key = {0}; + + cached_window.window = window; + cached_window.start_offset = start_offset - rs->start; + cached_window.end_offset = rs->current; + + /* but key it with the start offset because that is the known state + * when we will look it up */ + SVN_ERR(svn_cache__set(rs->window_cache, + get_window_key(&key, rs), + &cached_window, + scratch_pool)); + } + + return SVN_NO_ERROR; +} + +/* Read the WINDOW_P for the rep state RS from the current FSX session's + * cache. This will be a no-op and IS_CACHED will be set to FALSE if no + * cache has been given. If a cache is available IS_CACHED will inform + * the caller about the success of the lookup. Allocations (of the window + * in particualar) will be made from POOL. + */ +static svn_error_t * +get_cached_combined_window(svn_stringbuf_t **window_p, + rep_state_t *rs, + svn_boolean_t *is_cached, + apr_pool_t *pool) +{ + if (! rs->combined_cache) + { + /* txdelta window has not been enabled */ + *is_cached = FALSE; + } + else + { + /* ask the cache for the desired txdelta window */ + window_cache_key_t key = { 0 }; + return svn_cache__get((void **)window_p, + is_cached, + rs->combined_cache, + get_window_key(&key, rs), + pool); + } + + return SVN_NO_ERROR; +} + +/* Store the WINDOW read for the rep state RS in the current FSX session's + * cache. This will be a no-op if no cache has been given. + * Temporary allocations will be made from SCRATCH_POOL. */ +static svn_error_t * +set_cached_combined_window(svn_stringbuf_t *window, + rep_state_t *rs, + apr_pool_t *scratch_pool) +{ + if (rs->combined_cache) + { + /* but key it with the start offset because that is the known state + * when we will look it up */ + window_cache_key_t key = { 0 }; + return svn_cache__set(rs->combined_cache, + get_window_key(&key, rs), + window, + scratch_pool); + } + + return SVN_NO_ERROR; +} + +/* Build an array of rep_state structures in *LIST giving the delta + reps from first_rep to a self-compressed rep. Set *SRC_STATE to + the container rep we find at the end of the chain, or to NULL if + the final delta representation is self-compressed. + The representation to start from is designated by filesystem FS, id + ID, and representation REP. + Also, set *WINDOW_P to the base window content for *LIST, if it + could be found in cache. Otherwise, *LIST will contain the base + representation for the whole delta chain. + */ +static svn_error_t * +build_rep_list(apr_array_header_t **list, + svn_stringbuf_t **window_p, + rep_state_t **src_state, + svn_fs_t *fs, + representation_t *first_rep, + apr_pool_t *pool) +{ + representation_t rep; + rep_state_t *rs = NULL; + svn_fs_x__rep_header_t *rep_header; + svn_boolean_t is_cached = FALSE; + shared_file_t *shared_file = NULL; + + *list = apr_array_make(pool, 1, sizeof(rep_state_t *)); + rep = *first_rep; + + /* for the top-level rep, we need the rep_args */ + SVN_ERR(create_rep_state(&rs, &rep_header, &shared_file, &rep, fs, pool)); + + while (1) + { + /* fetch state, if that has not been done already */ + if (!rs) + SVN_ERR(create_rep_state(&rs, &rep_header, &shared_file, + &rep, fs, pool)); + + /* for txn reps and containered reps, there won't be a cached + * combined window */ + if (!svn_fs_x__id_txn_used(&rep.txn_id) + && rep_header->type != svn_fs_x__rep_container) + SVN_ERR(get_cached_combined_window(window_p, rs, &is_cached, pool)); + + if (is_cached) + { + /* We already have a reconstructed window in our cache. + Write a pseudo rep_state with the full length. */ + rs->start = 0; + rs->current = 0; + rs->size = (*window_p)->len; + *src_state = rs; + return SVN_NO_ERROR; + } + + if (rep_header->type == svn_fs_x__rep_container) + { + /* This is a container item, so just return the current rep_state. */ + *src_state = rs; + return SVN_NO_ERROR; + } + + /* Push this rep onto the list. If it's self-compressed, we're done. */ + APR_ARRAY_PUSH(*list, rep_state_t *) = rs; + if (rep_header->type == svn_fs_x__rep_self_delta) + { + *src_state = NULL; + return SVN_NO_ERROR; + } + + rep.revision = rep_header->base_revision; + rep.item_index = rep_header->base_item_index; + rep.size = rep_header->base_length; + svn_fs_x__id_txn_reset(&rep.txn_id); + + rs = NULL; + } +} + + +/* Create a rep_read_baton structure for node revision NODEREV in + filesystem FS and store it in *RB_P. If FULLTEXT_CACHE_KEY is not + NULL, it is the rep's key in the fulltext cache, and a stringbuf + must be allocated to store the text. Perform all allocations in + POOL. If rep is mutable, it must be for file contents. */ +static svn_error_t * +rep_read_get_baton(struct rep_read_baton **rb_p, + svn_fs_t *fs, + representation_t *rep, + pair_cache_key_t fulltext_cache_key, + apr_pool_t *pool) +{ + struct rep_read_baton *b; + + b = apr_pcalloc(pool, sizeof(*b)); + b->fs = fs; + b->base_window = NULL; + b->chunk_index = 0; + b->buf = NULL; + b->md5_checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); + b->checksum_finalized = FALSE; + memcpy(b->md5_digest, rep->md5_digest, sizeof(rep->md5_digest)); + b->len = rep->expanded_size; + b->off = 0; + b->fulltext_cache_key = fulltext_cache_key; + b->pool = svn_pool_create(pool); + b->filehandle_pool = svn_pool_create(pool); + + SVN_ERR(build_rep_list(&b->rs_list, &b->base_window, &b->src_state, + fs, rep, b->filehandle_pool)); + + if (SVN_IS_VALID_REVNUM(fulltext_cache_key.revision)) + b->current_fulltext = svn_stringbuf_create_ensure + ((apr_size_t)b->len, + b->filehandle_pool); + else + b->current_fulltext = NULL; + + /* Save our output baton. */ + *rb_p = b; + + return SVN_NO_ERROR; +} + +/* Open FILE->FILE and FILE->STREAM if they haven't been opened, yet. */ +static svn_error_t* +auto_open_shared_file(shared_file_t *file) +{ + if (file->file == NULL) + { + SVN_ERR(svn_fs_x__open_pack_or_rev_file(&file->file, file->fs, + file->revision, file->pool)); + file->stream = svn_stream_from_aprfile2(file->file, TRUE, file->pool); + } + + return SVN_NO_ERROR; +} + +/* Set RS->START to the begin of the representation raw in RS->FILE->FILE, + if that hasn't been done yet. Use POOL for temporary allocations. */ +static svn_error_t* +auto_set_start_offset(rep_state_t *rs, apr_pool_t *pool) +{ + if (rs->start == -1) + { + SVN_ERR(svn_fs_x__item_offset(&rs->start, &rs->sub_item, + rs->file->fs, rs->revision, NULL, + rs->item_index, pool)); + rs->start += rs->header_size; + } + + return SVN_NO_ERROR; +} + +/* Set RS->VER depending on what is found in the already open RS->FILE->FILE + if the diff version is still unknown. Use POOL for temporary allocations. + */ +static svn_error_t* +auto_read_diff_version(rep_state_t *rs, apr_pool_t *pool) +{ + if (rs->ver == -1) + { + char buf[4]; + SVN_ERR(aligned_seek(rs->file->fs, rs->file->file, NULL, rs->start, + pool)); + SVN_ERR(svn_io_file_read_full2(rs->file->file, buf, sizeof(buf), + NULL, NULL, pool)); + + /* ### Layering violation */ + if (! ((buf[0] == 'S') && (buf[1] == 'V') && (buf[2] == 'N'))) + return svn_error_create + (SVN_ERR_FS_CORRUPT, NULL, + _("Malformed svndiff data in representation")); + rs->ver = buf[3]; + + rs->chunk_index = 0; + rs->current = 4; + } + + return SVN_NO_ERROR; +} + +/* Skip forwards to THIS_CHUNK in REP_STATE and then read the next delta + window into *NWIN. */ +static svn_error_t * +read_delta_window(svn_txdelta_window_t **nwin, int this_chunk, + rep_state_t *rs, apr_pool_t *pool) +{ + svn_boolean_t is_cached; + apr_off_t start_offset; + apr_off_t end_offset; + SVN_ERR_ASSERT(rs->chunk_index <= this_chunk); + + SVN_ERR(dgb__log_access(rs->file->fs, rs->revision, rs->item_index, + NULL, SVN_FS_X__ITEM_TYPE_ANY_REP, pool)); + + /* Read the next window. But first, try to find it in the cache. */ + SVN_ERR(get_cached_window(nwin, rs, this_chunk, &is_cached, pool)); + if (is_cached) + return SVN_NO_ERROR; + + /* someone has to actually read the data from file. Open it */ + SVN_ERR(auto_open_shared_file(rs->file)); + + /* invoke the 'block-read' feature for non-txn data. + However, don't do that if we are in the middle of some representation, + because the block is unlikely to contain other data. */ + if (rs->chunk_index == 0 && SVN_IS_VALID_REVNUM(rs->revision)) + { + SVN_ERR(block_read(NULL, rs->file->fs, rs->revision, rs->item_index, + rs->file->file, pool, pool)); + + /* reading the whole block probably also provided us with the + desired txdelta window */ + SVN_ERR(get_cached_window(nwin, rs, this_chunk, &is_cached, pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + /* data is still not cached -> we need to read it. + Make sure we have all the necessary info. */ + SVN_ERR(auto_set_start_offset(rs, pool)); + SVN_ERR(auto_read_diff_version(rs, pool)); + + /* RS->FILE may be shared between RS instances -> make sure we point + * to the right data. */ + start_offset = rs->start + rs->current; + SVN_ERR(aligned_seek(rs->file->fs, rs->file->file, NULL, start_offset, + pool)); + + /* Skip windows to reach the current chunk if we aren't there yet. */ + while (rs->chunk_index < this_chunk) + { + SVN_ERR(svn_txdelta_skip_svndiff_window(rs->file->file, rs->ver, + pool)); + rs->chunk_index++; + SVN_ERR(svn_fs_x__get_file_offset(&start_offset, rs->file->file, pool)); + rs->current = start_offset - rs->start; + if (rs->current >= rs->size) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Reading one svndiff window read " + "beyond the end of the " + "representation")); + } + + /* Actually read the next window. */ + SVN_ERR(svn_txdelta_read_svndiff_window(nwin, rs->file->stream, rs->ver, + pool)); + SVN_ERR(svn_fs_x__get_file_offset(&end_offset, rs->file->file, pool)); + rs->current = end_offset - rs->start; + if (rs->current > rs->size) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Reading one svndiff window read beyond " + "the end of the representation")); + + /* the window has not been cached before, thus cache it now + * (if caching is used for them at all) */ + if (SVN_IS_VALID_REVNUM(rs->revision)) + SVN_ERR(set_cached_window(*nwin, rs, start_offset, pool)); + + return SVN_NO_ERROR; +} + +/* Read the whole representation RS and return it in *NWIN. */ +static svn_error_t * +read_container_window(svn_stringbuf_t **nwin, + rep_state_t *rs, + apr_size_t size, + apr_pool_t *pool) +{ + svn_fs_x__rep_extractor_t *extractor = NULL; + svn_fs_t *fs = rs->file->fs; + fs_x_data_t *ffd = fs->fsap_data; + pair_cache_key_t key; + + SVN_ERR(auto_set_start_offset(rs, pool)); + key.revision = svn_fs_x__packed_base_rev(fs, rs->revision); + key.second = rs->start; + + /* already in cache? */ + if (ffd->reps_container_cache) + { + svn_boolean_t is_cached = FALSE; + svn_fs_x__reps_baton_t baton; + baton.fs = fs; + baton.idx = rs->sub_item; + + SVN_ERR(svn_cache__get_partial((void**)&extractor, &is_cached, + ffd->reps_container_cache, &key, + svn_fs_x__reps_get_func, &baton, + pool)); + } + + /* read from disk, if necessary */ + if (extractor == NULL) + { + SVN_ERR(auto_open_shared_file(rs->file)); + SVN_ERR(block_read((void **)&extractor, fs, rs->revision, + rs->item_index, rs->file->file, pool, pool)); + } + + SVN_ERR(svn_fs_x__extractor_drive(nwin, extractor, rs->current, size, + pool, pool)); + + /* Update RS. */ + rs->current += (apr_off_t)size; + + return SVN_NO_ERROR; +} + +/* Get the undeltified window that is a result of combining all deltas + from the current desired representation identified in *RB with its + base representation. Store the window in *RESULT. */ +static svn_error_t * +get_combined_window(svn_stringbuf_t **result, + struct rep_read_baton *rb) +{ + apr_pool_t *pool, *new_pool, *window_pool; + int i; + apr_array_header_t *windows; + svn_stringbuf_t *source, *buf = rb->base_window; + rep_state_t *rs; + + /* Read all windows that we need to combine. This is fine because + the size of each window is relatively small (100kB) and skip- + delta limits the number of deltas in a chain to well under 100. + Stop early if one of them does not depend on its predecessors. */ + window_pool = svn_pool_create(rb->pool); + windows = apr_array_make(window_pool, 0, sizeof(svn_txdelta_window_t *)); + for (i = 0; i < rb->rs_list->nelts; ++i) + { + svn_txdelta_window_t *window; + + rs = APR_ARRAY_IDX(rb->rs_list, i, rep_state_t *); + SVN_ERR(read_delta_window(&window, rb->chunk_index, rs, window_pool)); + + APR_ARRAY_PUSH(windows, svn_txdelta_window_t *) = window; + if (window->src_ops == 0) + { + ++i; + break; + } + } + + /* Combine in the windows from the other delta reps. */ + pool = svn_pool_create(rb->pool); + for (--i; i >= 0; --i) + { + svn_txdelta_window_t *window; + + rs = APR_ARRAY_IDX(rb->rs_list, i, rep_state_t *); + window = APR_ARRAY_IDX(windows, i, svn_txdelta_window_t *); + + /* Maybe, we've got a start representation in a container. If we do, + read as much data from it as the needed for the txdelta window's + source view. + Note that BUF / SOURCE may only be NULL in the first iteration. */ + source = buf; + if (source == NULL && rb->src_state != NULL) + SVN_ERR(read_container_window(&source, rb->src_state, + window->sview_len, pool)); + + /* Combine this window with the current one. */ + new_pool = svn_pool_create(rb->pool); + buf = svn_stringbuf_create_ensure(window->tview_len, new_pool); + buf->len = window->tview_len; + + svn_txdelta_apply_instructions(window, source ? source->data : NULL, + buf->data, &buf->len); + if (buf->len != window->tview_len) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("svndiff window length is " + "corrupt")); + + /* Cache windows only if the whole rep content could be read as a + single chunk. Only then will no other chunk need a deeper RS + list than the cached chunk. */ + if ( (rb->chunk_index == 0) && (rs->current == rs->size) + && SVN_IS_VALID_REVNUM(rs->revision)) + SVN_ERR(set_cached_combined_window(buf, rs, new_pool)); + + rs->chunk_index++; + + /* Cycle pools so that we only need to hold three windows at a time. */ + svn_pool_destroy(pool); + pool = new_pool; + } + + svn_pool_destroy(window_pool); + + *result = buf; + return SVN_NO_ERROR; +} + +/* Returns whether or not the expanded fulltext of the file is cachable + * based on its size SIZE. The decision depends on the cache used by RB. + */ +static svn_boolean_t +fulltext_size_is_cachable(fs_x_data_t *ffd, svn_filesize_t size) +{ + return (size < APR_SIZE_MAX) + && svn_cache__is_cachable(ffd->fulltext_cache, (apr_size_t)size); +} + +/* Close method used on streams returned by read_representation(). + */ +static svn_error_t * +rep_read_contents_close(void *baton) +{ + struct rep_read_baton *rb = baton; + + svn_pool_destroy(rb->pool); + svn_pool_destroy(rb->filehandle_pool); + + return SVN_NO_ERROR; +} + +/* Inialize the representation read state RS for the given REP_HEADER and + * p2l index ENTRY. If not NULL, assign FILE and STREAM to RS. + * Use POOL for allocations. + */ +static svn_error_t * +init_rep_state(rep_state_t *rs, + svn_fs_x__rep_header_t *rep_header, + svn_fs_t *fs, + apr_file_t *file, + svn_stream_t *stream, + svn_fs_x__p2l_entry_t* entry, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + shared_file_t *shared_file = apr_pcalloc(pool, sizeof(*shared_file)); + + /* this function does not apply to representation containers */ + SVN_ERR_ASSERT(entry->type >= SVN_FS_X__ITEM_TYPE_FILE_REP + && entry->type <= SVN_FS_X__ITEM_TYPE_DIR_PROPS); + SVN_ERR_ASSERT(entry->item_count == 1); + + shared_file->file = file; + shared_file->stream = stream; + shared_file->fs = fs; + shared_file->revision = entry->items[0].revision; + shared_file->pool = pool; + + rs->file = shared_file; + rs->revision = entry->items[0].revision; + rs->item_index = entry->items[0].number; + rs->header_size = rep_header->header_size; + rs->start = entry->offset + rs->header_size; + rs->current = 4; + rs->size = entry->size - rep_header->header_size - 7; + rs->ver = 1; + rs->chunk_index = 0; + rs->window_cache = ffd->txdelta_window_cache; + rs->combined_cache = ffd->combined_window_cache; + + return SVN_NO_ERROR; +} + +/* Walk through all windows in the representation addressed by RS in FS + * (excluding the delta bases) and put those not already cached into the + * window caches. As a side effect, return the total sum of all expanded + * window sizes in *FULLTEXT_LEN. Use POOL for temporary allocations. + */ +static svn_error_t * +cache_windows(svn_filesize_t *fulltext_len, + svn_fs_t *fs, + rep_state_t *rs, + apr_pool_t *pool) +{ + *fulltext_len = 0; + + while (rs->current < rs->size) + { + svn_boolean_t is_cached = FALSE; + window_sizes_t *window_sizes; + + /* efficiently skip windows that are still being cached instead + * of fully decoding them */ + SVN_ERR(get_cached_window_sizes(&window_sizes, rs, &is_cached, pool)); + if (is_cached) + { + *fulltext_len += window_sizes->target_len; + rs->current += window_sizes->packed_len; + } + else + { + svn_txdelta_window_t *window; + apr_off_t start_offset = rs->start + rs->current; + apr_off_t end_offset; + apr_off_t block_start; + + /* navigate to & read the current window */ + SVN_ERR(aligned_seek(fs, rs->file->file, &block_start, + start_offset, pool)); + SVN_ERR(svn_txdelta_read_svndiff_window(&window, rs->file->stream, + rs->ver, pool)); + + /* aggregate expanded window size */ + *fulltext_len += window->tview_len; + + /* determine on-disk window size */ + SVN_ERR(svn_fs_x__get_file_offset(&end_offset, rs->file->file, + pool)); + rs->current = end_offset - rs->start; + if (rs->current > rs->size) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Reading one svndiff window read beyond " + "the end of the representation")); + + /* if the window has not been cached before, cache it now + * (if caching is used for them at all) */ + if (!is_cached) + SVN_ERR(set_cached_window(window, rs, start_offset, pool)); + } + + rs->chunk_index++; + } + + return SVN_NO_ERROR; +} + +/* Try to get the representation header identified by KEY from FS's cache. + * If it has not been cached, read it from the current position in STREAM + * and put it into the cache (if caching has been enabled for rep headers). + * Return the result in *REP_HEADER. Use POOL for allocations. + */ +static svn_error_t * +read_rep_header(svn_fs_x__rep_header_t **rep_header, + svn_fs_t *fs, + svn_stream_t *stream, + representation_cache_key_t *key, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_boolean_t is_cached = FALSE; + + if (ffd->rep_header_cache) + { + SVN_ERR(svn_cache__get((void**)rep_header, &is_cached, + ffd->rep_header_cache, key, pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + SVN_ERR(svn_fs_x__read_rep_header(rep_header, stream, pool)); + + if (ffd->rep_header_cache) + SVN_ERR(svn_cache__set(ffd->rep_header_cache, key, *rep_header, pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__get_representation_length(svn_filesize_t *packed_len, + svn_filesize_t *expanded_len, + svn_fs_t *fs, + apr_file_t *file, + svn_stream_t *stream, + svn_fs_x__p2l_entry_t* entry, + apr_pool_t *pool) +{ + representation_cache_key_t key = { 0 }; + rep_state_t rs = { 0 }; + svn_fs_x__rep_header_t *rep_header; + + /* this function does not apply to representation containers */ + SVN_ERR_ASSERT(entry->type >= SVN_FS_X__ITEM_TYPE_FILE_REP + && entry->type <= SVN_FS_X__ITEM_TYPE_DIR_PROPS); + SVN_ERR_ASSERT(entry->item_count == 1); + + /* get / read the representation header */ + key.revision = entry->items[0].revision; + key.is_packed = svn_fs_x__is_packed_rev(fs, key.revision); + key.item_index = entry->items[0].number; + SVN_ERR(read_rep_header(&rep_header, fs, stream, &key, pool)); + + /* prepare representation reader state (rs) structure */ + SVN_ERR(init_rep_state(&rs, rep_header, fs, file, stream, entry, pool)); + + /* RS->FILE may be shared between RS instances -> make sure we point + * to the right data. */ + *packed_len = rs.size; + SVN_ERR(cache_windows(expanded_len, fs, &rs, pool)); + + return SVN_NO_ERROR; +} + +/* Return the next *LEN bytes of the rep and store them in *BUF. */ +static svn_error_t * +get_contents(struct rep_read_baton *rb, + char *buf, + apr_size_t *len) +{ + apr_size_t copy_len, remaining = *len; + char *cur = buf; + rep_state_t *rs; + + /* Special case for when there are no delta reps, only a + containered text. */ + if (rb->rs_list->nelts == 0 && rb->buf == NULL) + { + copy_len = remaining; + rs = rb->src_state; + + /* reps in containers don't have a header */ + if (rs->header_size == 0 && rb->base_window == NULL) + { + /* RS->SIZE is unreliable here because it is based upon + * the delta rep size _before_ putting the data into a + * a container. */ + SVN_ERR(read_container_window(&rb->base_window, rs, + rb->len, rb->pool)); + rs->current -= rb->base_window->len; + } + + if (rb->base_window != NULL) + { + /* We got the desired rep directly from the cache. + This is where we need the pseudo rep_state created + by build_rep_list(). */ + apr_size_t offset = (apr_size_t)rs->current; + if (copy_len + offset > rb->base_window->len) + copy_len = offset < rb->base_window->len + ? rb->base_window->len - offset + : 0ul; + + memcpy (cur, rb->base_window->data + offset, copy_len); + } + + rs->current += copy_len; + *len = copy_len; + return SVN_NO_ERROR; + } + + while (remaining > 0) + { + /* If we have buffered data from a previous chunk, use that. */ + if (rb->buf) + { + /* Determine how much to copy from the buffer. */ + copy_len = rb->buf_len - rb->buf_pos; + if (copy_len > remaining) + copy_len = remaining; + + /* Actually copy the data. */ + memcpy(cur, rb->buf + rb->buf_pos, copy_len); + rb->buf_pos += copy_len; + cur += copy_len; + remaining -= copy_len; + + /* If the buffer is all used up, clear it and empty the + local pool. */ + if (rb->buf_pos == rb->buf_len) + { + svn_pool_clear(rb->pool); + rb->buf = NULL; + } + } + else + { + svn_stringbuf_t *sbuf = NULL; + + rs = APR_ARRAY_IDX(rb->rs_list, 0, rep_state_t *); + if (rs->current == rs->size) + break; + + /* Get more buffered data by evaluating a chunk. */ + SVN_ERR(get_combined_window(&sbuf, rb)); + + rb->chunk_index++; + rb->buf_len = sbuf->len; + rb->buf = sbuf->data; + rb->buf_pos = 0; + } + } + + *len = cur - buf; + + return SVN_NO_ERROR; +} + +/* BATON is of type `rep_read_baton'; read the next *LEN bytes of the + representation and store them in *BUF. Sum as we read and verify + the MD5 sum at the end. */ +static svn_error_t * +rep_read_contents(void *baton, + char *buf, + apr_size_t *len) +{ + struct rep_read_baton *rb = baton; + + /* Get the next block of data. */ + SVN_ERR(get_contents(rb, buf, len)); + + if (rb->current_fulltext) + svn_stringbuf_appendbytes(rb->current_fulltext, buf, *len); + + /* Perform checksumming. We want to check the checksum as soon as + the last byte of data is read, in case the caller never performs + a short read, but we don't want to finalize the MD5 context + twice. */ + if (!rb->checksum_finalized) + { + SVN_ERR(svn_checksum_update(rb->md5_checksum_ctx, buf, *len)); + rb->off += *len; + if (rb->off == rb->len) + { + svn_checksum_t *md5_checksum; + svn_checksum_t expected; + expected.kind = svn_checksum_md5; + expected.digest = rb->md5_digest; + + rb->checksum_finalized = TRUE; + SVN_ERR(svn_checksum_final(&md5_checksum, rb->md5_checksum_ctx, + rb->pool)); + if (!svn_checksum_match(md5_checksum, &expected)) + return svn_error_create(SVN_ERR_FS_CORRUPT, + svn_checksum_mismatch_err(&expected, md5_checksum, + rb->pool, + _("Checksum mismatch while reading representation")), + NULL); + } + } + + if (rb->off == rb->len && rb->current_fulltext) + { + fs_x_data_t *ffd = rb->fs->fsap_data; + SVN_ERR(svn_cache__set(ffd->fulltext_cache, &rb->fulltext_cache_key, + rb->current_fulltext, rb->pool)); + rb->current_fulltext = NULL; + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__get_contents(svn_stream_t **contents_p, + svn_fs_t *fs, + representation_t *rep, + apr_pool_t *pool) +{ + if (! rep) + { + *contents_p = svn_stream_empty(pool); + } + else + { + fs_x_data_t *ffd = fs->fsap_data; + pair_cache_key_t fulltext_cache_key = { 0 }; + svn_filesize_t len = rep->expanded_size; + struct rep_read_baton *rb; + + fulltext_cache_key.revision = rep->revision; + fulltext_cache_key.second = rep->item_index; + if (ffd->fulltext_cache && SVN_IS_VALID_REVNUM(rep->revision) + && fulltext_size_is_cachable(ffd, len)) + { + svn_stringbuf_t *fulltext; + svn_boolean_t is_cached; + SVN_ERR(svn_cache__get((void **) &fulltext, &is_cached, + ffd->fulltext_cache, &fulltext_cache_key, + pool)); + if (is_cached) + { + *contents_p = svn_stream_from_stringbuf(fulltext, pool); + return SVN_NO_ERROR; + } + } + else + fulltext_cache_key.revision = SVN_INVALID_REVNUM; + + SVN_ERR(rep_read_get_baton(&rb, fs, rep, fulltext_cache_key, pool)); + + *contents_p = svn_stream_create(rb, pool); + svn_stream_set_read(*contents_p, rep_read_contents); + svn_stream_set_close(*contents_p, rep_read_contents_close); + } + + return SVN_NO_ERROR; +} + + +/* Baton for cache_access_wrapper. Wraps the original parameters of + * svn_fs_x__try_process_file_content(). + */ +typedef struct cache_access_wrapper_baton_t +{ + svn_fs_process_contents_func_t func; + void* baton; +} cache_access_wrapper_baton_t; + +/* Wrapper to translate between svn_fs_process_contents_func_t and + * svn_cache__partial_getter_func_t. + */ +static svn_error_t * +cache_access_wrapper(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool) +{ + cache_access_wrapper_baton_t *wrapper_baton = baton; + + SVN_ERR(wrapper_baton->func((const unsigned char *)data, + data_len - 1, /* cache adds terminating 0 */ + wrapper_baton->baton, + pool)); + + /* non-NULL value to signal the calling cache that all went well */ + *out = baton; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__try_process_file_contents(svn_boolean_t *success, + svn_fs_t *fs, + node_revision_t *noderev, + svn_fs_process_contents_func_t processor, + void* baton, + apr_pool_t *pool) +{ + representation_t *rep = noderev->data_rep; + if (rep) + { + fs_x_data_t *ffd = fs->fsap_data; + pair_cache_key_t fulltext_cache_key = { 0 }; + + fulltext_cache_key.revision = rep->revision; + fulltext_cache_key.second = rep->item_index; + if (ffd->fulltext_cache && SVN_IS_VALID_REVNUM(rep->revision) + && fulltext_size_is_cachable(ffd, rep->expanded_size)) + { + cache_access_wrapper_baton_t wrapper_baton; + void *dummy = NULL; + + wrapper_baton.func = processor; + wrapper_baton.baton = baton; + return svn_cache__get_partial(&dummy, success, + ffd->fulltext_cache, + &fulltext_cache_key, + cache_access_wrapper, + &wrapper_baton, + pool); + } + } + + *success = FALSE; + return SVN_NO_ERROR; +} + +/* Baton used when reading delta windows. */ +struct delta_read_baton +{ + struct rep_state_t *rs; + unsigned char md5_digest[APR_MD5_DIGESTSIZE]; +}; + +/* This implements the svn_txdelta_next_window_fn_t interface. */ +static svn_error_t * +delta_read_next_window(svn_txdelta_window_t **window, void *baton, + apr_pool_t *pool) +{ + struct delta_read_baton *drb = baton; + + *window = NULL; + if (drb->rs->current < drb->rs->size) + { + SVN_ERR(read_delta_window(window, drb->rs->chunk_index, drb->rs, pool)); + drb->rs->chunk_index++; + } + + return SVN_NO_ERROR; +} + +/* This implements the svn_txdelta_md5_digest_fn_t interface. */ +static const unsigned char * +delta_read_md5_digest(void *baton) +{ + struct delta_read_baton *drb = baton; + return drb->md5_digest; +} + +svn_error_t * +svn_fs_x__get_file_delta_stream(svn_txdelta_stream_t **stream_p, + svn_fs_t *fs, + node_revision_t *source, + node_revision_t *target, + apr_pool_t *pool) +{ + svn_stream_t *source_stream, *target_stream; + + /* Try a shortcut: if the target is stored as a delta against the source, + then just use that delta. */ + if (source && source->data_rep && target->data_rep) + { + rep_state_t *rep_state; + svn_fs_x__rep_header_t *rep_header; + + /* Read target's base rep if any. */ + SVN_ERR(create_rep_state(&rep_state, &rep_header, NULL, + target->data_rep, fs, pool)); + /* If that matches source, then use this delta as is. */ + if (rep_header->type == svn_fs_x__rep_self_delta + || (rep_header->type == svn_fs_x__rep_delta + && rep_header->base_revision == source->data_rep->revision + && rep_header->base_item_index == source->data_rep->item_index)) + { + /* Create the delta read baton. */ + struct delta_read_baton *drb = apr_pcalloc(pool, sizeof(*drb)); + + drb->rs = rep_state; + memcpy(drb->md5_digest, target->data_rep->md5_digest, + sizeof(drb->md5_digest)); + *stream_p = svn_txdelta_stream_create(drb, delta_read_next_window, + delta_read_md5_digest, pool); + return SVN_NO_ERROR; + } + else + if (rep_state->file->file) + SVN_ERR(svn_io_file_close(rep_state->file->file, pool)); + } + + /* Read both fulltexts and construct a delta. */ + if (source) + SVN_ERR(svn_fs_x__get_contents(&source_stream, fs, source->data_rep, pool)); + else + source_stream = svn_stream_empty(pool); + SVN_ERR(svn_fs_x__get_contents(&target_stream, fs, target->data_rep, pool)); + + /* Because source and target stream will already verify their content, + * there is no need to do this once more. In particular if the stream + * content is being fetched from cache. */ + svn_txdelta2(stream_p, source_stream, target_stream, FALSE, pool); + + return SVN_NO_ERROR; +} + + +/* Fetch the contents of a directory into ENTRIES. Values are stored + as filename to string mappings; further conversion is necessary to + convert them into svn_fs_dirent_t values. */ +static svn_error_t * +get_dir_contents(apr_hash_t *entries, + svn_fs_t *fs, + node_revision_t *noderev, + apr_pool_t *pool) +{ + svn_stream_t *contents; + + if (noderev->data_rep && svn_fs_x__id_txn_used(&noderev->data_rep->txn_id)) + { + const char *filename + = svn_fs_x__path_txn_node_children(fs, noderev->id, pool); + + /* The representation is mutable. Read the old directory + contents from the mutable children file, followed by the + changes we've made in this transaction. */ + SVN_ERR(svn_stream_open_readonly(&contents, filename, pool, pool)); + SVN_ERR(svn_hash_read2(entries, contents, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_hash_read_incremental(entries, contents, NULL, pool)); + SVN_ERR(svn_stream_close(contents)); + } + else if (noderev->data_rep) + { + /* use a temporary pool for temp objects. + * Also undeltify content before parsing it. Otherwise, we could only + * parse it byte-by-byte. + */ + apr_pool_t *text_pool = svn_pool_create(pool); + apr_size_t len = noderev->data_rep->expanded_size; + svn_stringbuf_t *text = svn_stringbuf_create_ensure(len, text_pool); + text->len = len; + + /* The representation is immutable. Read it normally. */ + SVN_ERR(svn_fs_x__get_contents(&contents, fs, noderev->data_rep, text_pool)); + SVN_ERR(svn_stream_read(contents, text->data, &text->len)); + SVN_ERR(svn_stream_close(contents)); + + /* de-serialize hash */ + contents = svn_stream_from_stringbuf(text, text_pool); + SVN_ERR(svn_hash_read2(entries, contents, SVN_HASH_TERMINATOR, pool)); + + svn_pool_destroy(text_pool); + } + + return SVN_NO_ERROR; +} + + +/* Given a hash STR_ENTRIES with values as svn_string_t as specified + in an FSX directory contents listing, return a hash of dirents in + *ENTRIES_P. Use ID to generate more helpful error messages. + Perform allocations in POOL. */ +static svn_error_t * +parse_dir_entries(apr_hash_t **entries_p, + apr_hash_t *str_entries, + const svn_fs_id_t *id, + apr_pool_t *pool) +{ + apr_hash_index_t *hi; + + *entries_p = apr_hash_make(pool); + + /* Translate the string dir entries into real entries. */ + for (hi = apr_hash_first(pool, str_entries); hi; hi = apr_hash_next(hi)) + { + const char *name = svn__apr_hash_index_key(hi); + svn_string_t *str_val = svn__apr_hash_index_val(hi); + char *str, *last_str; + svn_fs_dirent_t *dirent = apr_pcalloc(pool, sizeof(*dirent)); + + last_str = apr_pstrdup(pool, str_val->data); + dirent->name = apr_pstrdup(pool, name); + + str = svn_cstring_tokenize(" ", &last_str); + if (str == NULL) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Directory entry corrupt in '%s'"), + svn_fs_x__id_unparse(id, pool)->data); + + if (strcmp(str, SVN_FS_X__KIND_FILE) == 0) + { + dirent->kind = svn_node_file; + } + else if (strcmp(str, SVN_FS_X__KIND_DIR) == 0) + { + dirent->kind = svn_node_dir; + } + else + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Directory entry corrupt in '%s'"), + svn_fs_x__id_unparse(id, pool)->data); + } + + str = svn_cstring_tokenize(" ", &last_str); + if (str == NULL) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Directory entry corrupt in '%s'"), + svn_fs_x__id_unparse(id, pool)->data); + + dirent->id = svn_fs_x__id_parse(str, strlen(str), pool); + + svn_hash_sets(*entries_p, dirent->name, dirent); + } + + return SVN_NO_ERROR; +} + +/* Return the cache object in FS responsible to storing the directory the + * NODEREV plus the corresponding *KEY. If no cache exists, return NULL. + * PAIR_KEY must point to some key struct, which does not need to be + * initialized. We use it to avoid dynamic allocation. + */ +static svn_cache__t * +locate_dir_cache(svn_fs_t *fs, + const void **key, + pair_cache_key_t *pair_key, + node_revision_t *noderev, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + if (svn_fs_x__id_is_txn(noderev->id)) + { + /* data in txns requires the expensive fs_id-based addressing mode */ + *key = svn_fs_x__id_unparse(noderev->id, pool)->data; + return ffd->txn_dir_cache; + } + else + { + /* committed data can use simple rev,item pairs */ + if (noderev->data_rep) + { + pair_key->revision = noderev->data_rep->revision; + pair_key->second = noderev->data_rep->item_index; + *key = pair_key; + } + else + { + /* no data rep -> empty directory. + A NULL key causes a cache miss. */ + *key = NULL; + } + + return ffd->dir_cache; + } +} + +svn_error_t * +svn_fs_x__rep_contents_dir(apr_hash_t **entries_p, + svn_fs_t *fs, + node_revision_t *noderev, + apr_pool_t *pool) +{ + pair_cache_key_t pair_key = { 0 }; + const void *key; + apr_hash_t *unparsed_entries, *parsed_entries; + + /* find the cache we may use */ + svn_cache__t *cache = locate_dir_cache(fs, &key, &pair_key, noderev, pool); + if (cache) + { + svn_boolean_t found; + + SVN_ERR(svn_cache__get((void **)entries_p, &found, cache, key, pool)); + if (found) + return SVN_NO_ERROR; + } + + /* Read in the directory hash. */ + unparsed_entries = apr_hash_make(pool); + SVN_ERR(get_dir_contents(unparsed_entries, fs, noderev, pool)); + SVN_ERR(parse_dir_entries(&parsed_entries, unparsed_entries, + noderev->id, pool)); + + /* Update the cache, if we are to use one. */ + if (cache) + SVN_ERR(svn_cache__set(cache, key, parsed_entries, pool)); + + *entries_p = parsed_entries; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__rep_contents_dir_entry(svn_fs_dirent_t **dirent, + svn_fs_t *fs, + node_revision_t *noderev, + const char *name, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + svn_boolean_t found = FALSE; + + /* find the cache we may use */ + pair_cache_key_t pair_key = { 0 }; + const void *key; + svn_cache__t *cache = locate_dir_cache(fs, &key, &pair_key, noderev, + scratch_pool); + if (cache) + { + /* Cache lookup. */ + SVN_ERR(svn_cache__get_partial((void **)dirent, + &found, + cache, + key, + svn_fs_x__extract_dir_entry, + (void*)name, + result_pool)); + } + + /* fetch data from disk if we did not find it in the cache */ + if (! found) + { + apr_hash_t *entries; + svn_fs_dirent_t *entry; + svn_fs_dirent_t *entry_copy = NULL; + + /* read the dir from the file system. It will probably be put it + into the cache for faster lookup in future calls. */ + SVN_ERR(svn_fs_x__rep_contents_dir(&entries, fs, noderev, + scratch_pool)); + + /* find desired entry and return a copy in POOL, if found */ + entry = svn_hash_gets(entries, name); + if (entry != NULL) + { + entry_copy = apr_palloc(result_pool, sizeof(*entry_copy)); + entry_copy->name = apr_pstrdup(result_pool, entry->name); + entry_copy->id = svn_fs_x__id_copy(entry->id, result_pool); + entry_copy->kind = entry->kind; + } + + *dirent = entry_copy; + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__get_proplist(apr_hash_t **proplist_p, + svn_fs_t *fs, + node_revision_t *noderev, + apr_pool_t *pool) +{ + apr_hash_t *proplist; + svn_stream_t *stream; + + if (noderev->prop_rep && svn_fs_x__id_txn_used(&noderev->prop_rep->txn_id)) + { + const char *filename + = svn_fs_x__path_txn_node_props(fs, noderev->id, pool); + proplist = apr_hash_make(pool); + + SVN_ERR(svn_stream_open_readonly(&stream, filename, pool, pool)); + SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_stream_close(stream)); + } + else if (noderev->prop_rep) + { + fs_x_data_t *ffd = fs->fsap_data; + representation_t *rep = noderev->prop_rep; + pair_cache_key_t key = { 0 }; + + key.revision = rep->revision; + key.second = rep->item_index; + if (ffd->properties_cache && SVN_IS_VALID_REVNUM(rep->revision)) + { + svn_boolean_t is_cached; + SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached, + ffd->properties_cache, &key, pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + proplist = apr_hash_make(pool); + SVN_ERR(svn_fs_x__get_contents(&stream, fs, noderev->prop_rep, pool)); + SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_stream_close(stream)); + + if (ffd->properties_cache && SVN_IS_VALID_REVNUM(rep->revision)) + SVN_ERR(svn_cache__set(ffd->properties_cache, &key, proplist, pool)); + } + else + { + /* return an empty prop list if the node doesn't have any props */ + proplist = apr_hash_make(pool); + } + + *proplist_p = proplist; + + return SVN_NO_ERROR; +} + + + +/* Fetch the list of change in revision REV in FS and return it in *CHANGES. + * Allocate the result in POOL. + */ +svn_error_t * +svn_fs_x__get_changes(apr_array_header_t **changes, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + apr_file_t *revision_file; + svn_boolean_t found; + fs_x_data_t *ffd = fs->fsap_data; + + /* try cache lookup first */ + + if (ffd->changes_container_cache && svn_fs_x__is_packed_rev(fs, rev)) + { + apr_off_t offset; + apr_uint32_t sub_item; + pair_cache_key_t key; + + SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, rev, NULL, + SVN_FS_X__ITEM_INDEX_CHANGES, pool)); + key.revision = svn_fs_x__packed_base_rev(fs, rev); + key.second = offset; + + SVN_ERR(svn_cache__get_partial((void **)changes, &found, + ffd->changes_container_cache, &key, + svn_fs_x__changes_get_list_func, + &sub_item, pool)); + } + else if (ffd->changes_cache) + { + SVN_ERR(svn_cache__get((void **) changes, &found, ffd->changes_cache, + &rev, pool)); + } + + if (!found) + { + /* read changes from revision file */ + + SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, pool)); + SVN_ERR(svn_fs_x__open_pack_or_rev_file(&revision_file, fs, rev, + pool)); + + /* 'block-read' will also provide us with the desired data */ + SVN_ERR(block_read((void **)changes, fs, + rev, SVN_FS_X__ITEM_INDEX_CHANGES, + revision_file, pool, pool)); + + SVN_ERR(svn_io_file_close(revision_file, pool)); + } + + SVN_ERR(dgb__log_access(fs, rev, SVN_FS_X__ITEM_INDEX_CHANGES, *changes, + SVN_FS_X__ITEM_TYPE_CHANGES, pool)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +block_read_windows(svn_fs_x__rep_header_t *rep_header, + svn_fs_t *fs, + apr_file_t *file, + svn_stream_t *stream, + svn_fs_x__p2l_entry_t* entry, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + rep_state_t rs = { 0 }; + svn_filesize_t fulltext_len; + + if (!ffd->txdelta_window_cache || !ffd->combined_window_cache) + return SVN_NO_ERROR; + + SVN_ERR(init_rep_state(&rs, rep_header, fs, file, stream, entry, pool)); + SVN_ERR(cache_windows(&fulltext_len, fs, &rs, pool)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +block_read_contents(svn_stringbuf_t **item, + svn_fs_t *fs, + apr_file_t *file, + svn_stream_t *stream, + svn_fs_x__p2l_entry_t* entry, + pair_cache_key_t *key, + apr_pool_t *pool) +{ + representation_cache_key_t header_key = { 0 }; + svn_fs_x__rep_header_t *rep_header; + + header_key.revision = (apr_int32_t)key->revision; + header_key.is_packed = svn_fs_x__is_packed_rev(fs, header_key.revision); + header_key.item_index = key->second; + + SVN_ERR(read_rep_header(&rep_header, fs, stream, &header_key, pool)); + SVN_ERR(block_read_windows(rep_header, fs, file, stream, entry, pool)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +auto_select_stream(svn_stream_t **stream, + svn_fs_t *fs, + apr_file_t *file, + svn_stream_t *file_stream, + svn_fs_x__p2l_entry_t* entry, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + + if (((entry->offset + entry->size) ^ entry->offset) >= ffd->block_size) + { + svn_stringbuf_t *text = svn_stringbuf_create_ensure(entry->size, pool); + text->len = entry->size; + text->data[text->len] = 0; + SVN_ERR(svn_io_file_read_full2(file, text->data, text->len, NULL, + NULL, pool)); + *stream = svn_stream_from_stringbuf(text, pool); + } + else + { + *stream = file_stream; + } + + return SVN_NO_ERROR; +} + +static svn_error_t * +block_read_changes(apr_array_header_t **changes, + svn_fs_t *fs, + apr_file_t *file, + svn_stream_t *file_stream, + svn_fs_x__p2l_entry_t* entry, + svn_boolean_t must_read, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_stream_t *stream; + if (!must_read && !ffd->changes_cache) + return SVN_NO_ERROR; + + /* we don't support containers, yet */ + SVN_ERR_ASSERT(entry->item_count == 1); + + /* already in cache? */ + if (!must_read && ffd->changes_cache) + { + svn_boolean_t is_cached = FALSE; + SVN_ERR(svn_cache__has_key(&is_cached, ffd->changes_cache, + &entry->items[0].revision, pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + SVN_ERR(auto_select_stream(&stream, fs, file, file_stream, entry, pool)); + + /* read changes from revision file */ + + SVN_ERR(svn_fs_x__read_changes(changes, stream, pool)); + + /* cache for future reference */ + + if (ffd->changes_cache) + SVN_ERR(svn_cache__set(ffd->changes_cache, &entry->items[0].revision, + *changes, pool)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +block_read_changes_container(apr_array_header_t **changes, + svn_fs_t *fs, + apr_file_t *file, + svn_stream_t *file_stream, + svn_fs_x__p2l_entry_t* entry, + apr_uint32_t sub_item, + svn_boolean_t must_read, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__changes_t *container; + pair_cache_key_t key; + svn_stream_t *stream; + + key.revision = svn_fs_x__packed_base_rev(fs, entry->items[0].revision); + key.second = entry->offset; + + /* already in cache? */ + if (!must_read && ffd->changes_container_cache) + { + svn_boolean_t is_cached = FALSE; + SVN_ERR(svn_cache__has_key(&is_cached, ffd->changes_container_cache, + &key, pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + SVN_ERR(auto_select_stream(&stream, fs, file, file_stream, entry, pool)); + + /* read changes from revision file */ + + SVN_ERR(svn_fs_x__read_changes_container(&container, stream, pool, pool)); + + /* extract requested data */ + + if (must_read) + SVN_ERR(svn_fs_x__changes_get_list(changes, container, sub_item, pool)); + + if (ffd->changes_container_cache) + SVN_ERR(svn_cache__set(ffd->changes_container_cache, &key, container, + pool)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +block_read_noderev(node_revision_t **noderev_p, + svn_fs_t *fs, + apr_file_t *file, + svn_stream_t *file_stream, + svn_fs_x__p2l_entry_t* entry, + pair_cache_key_t *key, + svn_boolean_t must_read, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_stream_t *stream; + if (!must_read && !ffd->node_revision_cache) + return SVN_NO_ERROR; + + /* we don't support containers, yet */ + SVN_ERR_ASSERT(entry->item_count == 1); + + /* already in cache? */ + if (!must_read && ffd->node_revision_cache) + { + svn_boolean_t is_cached = FALSE; + SVN_ERR(svn_cache__has_key(&is_cached, ffd->node_revision_cache, key, + pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + SVN_ERR(auto_select_stream(&stream, fs, file, file_stream, entry, pool)); + + /* read node rev from revision file */ + + SVN_ERR(svn_fs_x__read_noderev(noderev_p, stream, pool)); + + /* Workaround issue #4031: is-fresh-txn-root in revision files. */ + (*noderev_p)->is_fresh_txn_root = FALSE; + + if (ffd->node_revision_cache) + SVN_ERR(svn_cache__set(ffd->node_revision_cache, key, *noderev_p, pool)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +block_read_noderevs_container(node_revision_t **noderev_p, + svn_fs_t *fs, + apr_file_t *file, + svn_stream_t *file_stream, + svn_fs_x__p2l_entry_t* entry, + apr_uint32_t sub_item, + svn_boolean_t must_read, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__noderevs_t *container; + svn_stream_t *stream; + pair_cache_key_t key; + + key.revision = svn_fs_x__packed_base_rev(fs, entry->items[0].revision); + key.second = entry->offset; + + /* already in cache? */ + if (!must_read && ffd->noderevs_container_cache) + { + svn_boolean_t is_cached = FALSE; + SVN_ERR(svn_cache__has_key(&is_cached, ffd->noderevs_container_cache, + &key, pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + SVN_ERR(auto_select_stream(&stream, fs, file, file_stream, entry, pool)); + + /* read noderevs from revision file */ + + SVN_ERR(svn_fs_x__read_noderevs_container(&container, stream, pool, pool)); + + /* extract requested data */ + + if (must_read) + SVN_ERR(svn_fs_x__noderevs_get(noderev_p, container, sub_item, pool)); + + if (ffd->noderevs_container_cache) + SVN_ERR(svn_cache__set(ffd->noderevs_container_cache, &key, container, + pool)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +block_read_reps_container(svn_fs_x__rep_extractor_t **extractor, + svn_fs_t *fs, + apr_file_t *file, + svn_stream_t *file_stream, + svn_fs_x__p2l_entry_t* entry, + apr_uint32_t sub_item, + svn_boolean_t must_read, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__reps_t *container; + svn_stream_t *stream; + pair_cache_key_t key; + + key.revision = svn_fs_x__packed_base_rev(fs, entry->items[0].revision); + key.second = entry->offset; + + /* already in cache? */ + if (!must_read && ffd->reps_container_cache) + { + svn_boolean_t is_cached = FALSE; + SVN_ERR(svn_cache__has_key(&is_cached, ffd->reps_container_cache, + &key, pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + SVN_ERR(auto_select_stream(&stream, fs, file, file_stream, entry, pool)); + + /* read noderevs from revision file */ + + SVN_ERR(svn_fs_x__read_reps_container(&container, stream, pool, pool)); + + /* extract requested data */ + + if (must_read) + SVN_ERR(svn_fs_x__reps_get(extractor, fs, container, sub_item, pool)); + + if (ffd->noderevs_container_cache) + SVN_ERR(svn_cache__set(ffd->reps_container_cache, &key, container, + pool)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +block_read(void **result, + svn_fs_t *fs, + svn_revnum_t revision, + apr_uint64_t item_index, + apr_file_t *revision_file, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + apr_off_t offset, wanted_offset = 0; + apr_off_t block_start = 0; + apr_uint32_t wanted_sub_item = 0; + apr_array_header_t *entries; + int run_count = 0; + int i; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + svn_stream_t *stream = svn_stream_from_aprfile2(revision_file, TRUE, + scratch_pool); + + /* don't try this on transaction protorev files */ + SVN_ERR_ASSERT(SVN_IS_VALID_REVNUM(revision)); + + /* index lookup: find the OFFSET of the item we *must* read plus (in the + * "do-while" block) the list of items in the same block. */ + SVN_ERR(svn_fs_x__item_offset(&wanted_offset, &wanted_sub_item, fs, + revision, NULL, item_index, iterpool)); + + offset = wanted_offset; + do + { + SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, revision, offset, + scratch_pool)); + SVN_ERR(aligned_seek(fs, revision_file, &block_start, offset, iterpool)); + + /* read all items from the block */ + for (i = 0; i < entries->nelts; ++i) + { + svn_boolean_t is_result; + apr_pool_t *pool; + + svn_fs_x__p2l_entry_t* entry + = &APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t); + + /* skip empty sections */ + if (entry->type == SVN_FS_X__ITEM_TYPE_UNUSED) + continue; + + /* the item / container we were looking for? */ + is_result = result + && entry->offset == wanted_offset + && entry->item_count >= wanted_sub_item + && entry->items[wanted_sub_item].revision == revision + && entry->items[wanted_sub_item].number == item_index; + + /* select the pool that we want the item to be allocated in */ + pool = is_result ? result_pool : iterpool; + + /* handle all items that start within this block and are relatively + * small (i.e. < block size). Always read the item we need to return. + */ + if (is_result || ( entry->offset >= block_start + && entry->size < ffd->block_size)) + { + void *item = NULL; + pair_cache_key_t key = { 0 }; + key.revision = entry->items[0].revision; + key.second = entry->items[0].number; + + SVN_ERR(svn_io_file_seek(revision_file, SEEK_SET, + &entry->offset, iterpool)); + switch (entry->type) + { + case SVN_FS_X__ITEM_TYPE_FILE_REP: + case SVN_FS_X__ITEM_TYPE_DIR_REP: + case SVN_FS_X__ITEM_TYPE_FILE_PROPS: + case SVN_FS_X__ITEM_TYPE_DIR_PROPS: + SVN_ERR(block_read_contents((svn_stringbuf_t **)&item, + fs, revision_file, stream, + entry, &key, pool)); + break; + + case SVN_FS_X__ITEM_TYPE_NODEREV: + if (ffd->node_revision_cache || is_result) + SVN_ERR(block_read_noderev((node_revision_t **)&item, + fs, revision_file, stream, + entry, &key, is_result, + pool)); + break; + + case SVN_FS_X__ITEM_TYPE_CHANGES: + SVN_ERR(block_read_changes((apr_array_header_t **)&item, + fs, revision_file, stream, + entry, is_result, pool)); + break; + + case SVN_FS_X__ITEM_TYPE_CHANGES_CONT: + SVN_ERR(block_read_changes_container + ((apr_array_header_t **)&item, + fs, revision_file, stream, + entry, wanted_sub_item, + is_result, pool)); + break; + + case SVN_FS_X__ITEM_TYPE_NODEREVS_CONT: + SVN_ERR(block_read_noderevs_container + ((node_revision_t **)&item, + fs, revision_file, stream, + entry, wanted_sub_item, + is_result, pool)); + break; + + case SVN_FS_X__ITEM_TYPE_REPS_CONT: + SVN_ERR(block_read_reps_container + ((svn_fs_x__rep_extractor_t **)&item, + fs, revision_file, stream, + entry, wanted_sub_item, + is_result, pool)); + break; + + default: + break; + } + + if (is_result) + *result = item; + + /* if we crossed a block boundary, read the remainder of + * the last block as well */ + offset = entry->offset + entry->size; + if (offset > block_start + ffd->block_size) + ++run_count; + + svn_pool_clear(iterpool); + } + } + } + while(run_count++ == 1); /* can only be true once and only if a block + * boundary got crossed */ + + /* if the caller requested a result, we must have provided one by now */ + assert(!result || *result); + SVN_ERR(svn_stream_close(stream)); + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_fs_x/cached_data.h b/subversion/libsvn_fs_x/cached_data.h new file mode 100644 index 0000000..68b8b60 --- /dev/null +++ b/subversion/libsvn_fs_x/cached_data.h
@@ -0,0 +1,157 @@ +/* cached_data.h --- cached (read) access to FSX data + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS__CACHED_DATA_H +#define SVN_LIBSVN_FS__CACHED_DATA_H + +#include "svn_pools.h" +#include "svn_fs.h" + +#include "fs.h" +#include "index.h" + + + +/* Set *NODEREV_P to the node-revision for the node ID in FS. Do any + allocations in POOL. */ +svn_error_t * +svn_fs_x__get_node_revision(node_revision_t **noderev_p, + svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool); + +/* Set *ROOT_ID to the node-id for the root of revision REV in + filesystem FS. Do any allocations in POOL. */ +svn_error_t * +svn_fs_x__rev_get_root(svn_fs_id_t **root_id, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +/* Verify that representation REP in FS can be accessed. Successive calls + to this function should pass a non-NULL value to HINT. In that case, + many file open / close operations can be eliminated. + Do any allocations in POOL. */ +svn_error_t * +svn_fs_x__check_rep(representation_t *rep, + svn_fs_t *fs, + void **hint, + apr_pool_t *pool); + +/* Follow the representation delta chain in FS starting with REP. The + number of reps (including REP) in the chain will be returned in + *CHAIN_LENGTH. Do any allocations in POOL. */ +svn_error_t * +svn_fs_x__rep_chain_length(int *chain_length, + representation_t *rep, + svn_fs_t *fs, + apr_pool_t *pool); + +/* Set *CONTENTS to be a readable svn_stream_t that receives the text + representation REP as seen in filesystem FS. + Use POOL for temporary allocations. */ +svn_error_t * +svn_fs_x__get_contents(svn_stream_t **contents_p, + svn_fs_t *fs, + representation_t *rep, + apr_pool_t *pool); + +/* Determine on-disk and expanded sizes of the representation identified + * by ENTRY in FS and return the result in PACKED_LEN and EXPANDED_LEN, + * respectively. FILE must point to the start of the representation and + * STREAM must be a stream defined on top of FILE. + * Use POOL for allocations. + */ +svn_error_t * +svn_fs_x__get_representation_length(svn_filesize_t *packed_len, + svn_filesize_t *expanded_len, + svn_fs_t *fs, + apr_file_t *file, + svn_stream_t *stream, + svn_fs_x__p2l_entry_t* entry, + apr_pool_t *pool); + +/* Attempt to fetch the text representation of node-revision NODEREV as + seen in filesystem FS and pass it along with the BATON to the PROCESSOR. + Set *SUCCESS only of the data could be provided and the processing + had been called. + Use POOL for all allocations. + */ +svn_error_t * +svn_fs_x__try_process_file_contents(svn_boolean_t *success, + svn_fs_t *fs, + node_revision_t *noderev, + svn_fs_process_contents_func_t processor, + void* baton, + apr_pool_t *pool); + +/* Set *STREAM_P to a delta stream turning the contents of the file SOURCE into + the contents of the file TARGET, allocated in POOL. + If SOURCE is null, the empty string will be used. */ +svn_error_t * +svn_fs_x__get_file_delta_stream(svn_txdelta_stream_t **stream_p, + svn_fs_t *fs, + node_revision_t *source, + node_revision_t *target, + apr_pool_t *pool); + +/* Set *ENTRIES to an apr_hash_t of dirent structs that contain the + directory entries of node-revision NODEREV in filesystem FS. The + returned table (and its keys and values) is allocated in POOL, + which is also used for temporary allocations. */ +svn_error_t * +svn_fs_x__rep_contents_dir(apr_hash_t **entries_p, + svn_fs_t *fs, + node_revision_t *noderev, + apr_pool_t *pool); + +/* Set *DIRENT to the entry identified by NAME in the directory given + by NODEREV in filesystem FS. If no such entry exits, *DIRENT will + be NULL. The returned object is allocated in RESULT_POOL; SCRATCH_POOL + used for temporary allocations. */ +svn_error_t * +svn_fs_x__rep_contents_dir_entry(svn_fs_dirent_t **dirent, + svn_fs_t *fs, + node_revision_t *noderev, + const char *name, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + +/* Set *PROPLIST to be an apr_hash_t containing the property list of + node-revision NODEREV as seen in filesystem FS. Use POOL for + temporary allocations. */ +svn_error_t * +svn_fs_x__get_proplist(apr_hash_t **proplist, + svn_fs_t *fs, + node_revision_t *noderev, + apr_pool_t *pool); + +/* Fetch the list of change in revision REV in FS and return it in *CHANGES. + * Allocate the result in POOL. + */ +svn_error_t * +svn_fs_x__get_changes(apr_array_header_t **changes, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +#endif \ No newline at end of file
diff --git a/subversion/libsvn_fs_x/caching.c b/subversion/libsvn_fs_x/caching.c new file mode 100644 index 0000000..653f4f6 --- /dev/null +++ b/subversion/libsvn_fs_x/caching.c
@@ -0,0 +1,867 @@ +/* caching.c : in-memory caching + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "fs.h" +#include "fs_x.h" +#include "id.h" +#include "dag.h" +#include "tree.h" +#include "index.h" +#include "changes.h" +#include "noderevs.h" +#include "temp_serializer.h" +#include "reps.h" +#include "../libsvn_fs/fs-loader.h" + +#include "svn_config.h" +#include "svn_cache_config.h" + +#include "svn_private_config.h" +#include "svn_hash.h" +#include "svn_pools.h" + +#include "private/svn_debug.h" +#include "private/svn_subr_private.h" + +/* Take the ORIGINAL string and replace all occurrences of ":" without + * limiting the key space. Allocate the result in POOL. + */ +static const char * +normalize_key_part(const char *original, + apr_pool_t *pool) +{ + apr_size_t i; + apr_size_t len = strlen(original); + svn_stringbuf_t *normalized = svn_stringbuf_create_ensure(len, pool); + + for (i = 0; i < len; ++i) + { + char c = original[i]; + switch (c) + { + case ':': svn_stringbuf_appendbytes(normalized, "%_", 2); + break; + case '%': svn_stringbuf_appendbytes(normalized, "%%", 2); + break; + default : svn_stringbuf_appendbyte(normalized, c); + } + } + + return normalized->data; +} + +/* Return a memcache in *MEMCACHE_P for FS if it's configured to use + memcached, or NULL otherwise. Also, sets *FAIL_STOP to a boolean + indicating whether cache errors should be returned to the caller or + just passed to the FS warning handler. + + *CACHE_TXDELTAS, *CACHE_FULLTEXTS and *CACHE_REVPROPS flags will be set + according to FS->CONFIG. *CACHE_NAMESPACE receives the cache prefix + to use. + + Use FS->pool for allocating the memcache and CACHE_NAMESPACE, and POOL + for temporary allocations. */ +static svn_error_t * +read_config(svn_memcache_t **memcache_p, + svn_boolean_t *fail_stop, + const char **cache_namespace, + svn_boolean_t *cache_txdeltas, + svn_boolean_t *cache_fulltexts, + svn_boolean_t *cache_revprops, + svn_fs_t *fs, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + + SVN_ERR(svn_cache__make_memcache_from_config(memcache_p, ffd->config, + fs->pool)); + + /* No cache namespace by default. I.e. all FS instances share the + * cached data. If you specify different namespaces, the data will + * share / compete for the same cache memory but keys will not match + * across namespaces and, thus, cached data will not be shared between + * namespaces. + * + * Since the namespace will be concatenated with other elements to form + * the complete key prefix, we must make sure that the resulting string + * is unique and cannot be created by any other combination of elements. + */ + *cache_namespace + = normalize_key_part(svn_hash__get_cstring(fs->config, + SVN_FS_CONFIG_FSFS_CACHE_NS, + ""), + pool); + + /* don't cache text deltas by default. + * Once we reconstructed the fulltexts from the deltas, + * these deltas are rarely re-used. Therefore, only tools + * like svnadmin will activate this to speed up operations + * dump and verify. + */ + *cache_txdeltas + = svn_hash__get_bool(fs->config, + SVN_FS_CONFIG_FSFS_CACHE_DELTAS, + TRUE); + + /* by default, cache fulltexts. + * Most SVN tools care about reconstructed file content. + * Thus, this is a reasonable default. + * SVN admin tools may set that to FALSE because fulltexts + * won't be re-used rendering the cache less effective + * by squeezing wanted data out. + */ + *cache_fulltexts + = svn_hash__get_bool(fs->config, + SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS, + TRUE); + + /* don't cache revprops by default. + * Revprop caching significantly speeds up operations like + * svn ls -v. However, it requires synchronization that may + * not be available or efficient in the current server setup. + * + * If the caller chose option "2", enable revprop caching if + * the required API support is there to make it efficient. + */ + if (strcmp(svn_hash__get_cstring(fs->config, + SVN_FS_CONFIG_FSFS_CACHE_REVPROPS, + ""), "2")) + *cache_revprops + = svn_hash__get_bool(fs->config, + SVN_FS_CONFIG_FSFS_CACHE_REVPROPS, + FALSE); + else + *cache_revprops = svn_named_atomic__is_efficient(); + + return svn_config_get_bool(ffd->config, fail_stop, + CONFIG_SECTION_CACHES, CONFIG_OPTION_FAIL_STOP, + FALSE); +} + + +/* Implements svn_cache__error_handler_t + * This variant clears the error after logging it. + */ +static svn_error_t * +warn_and_continue_on_cache_errors(svn_error_t *err, + void *baton, + apr_pool_t *pool) +{ + svn_fs_t *fs = baton; + (fs->warning)(fs->warning_baton, err); + svn_error_clear(err); + + return SVN_NO_ERROR; +} + +/* Implements svn_cache__error_handler_t + * This variant logs the error and passes it on to the callers. + */ +static svn_error_t * +warn_and_fail_on_cache_errors(svn_error_t *err, + void *baton, + apr_pool_t *pool) +{ + svn_fs_t *fs = baton; + (fs->warning)(fs->warning_baton, err); + return err; +} + +#ifdef SVN_DEBUG_CACHE_DUMP_STATS +/* Baton to be used for the dump_cache_statistics() pool cleanup function, */ +struct dump_cache_baton_t +{ + /* the pool about to be cleaned up. Will be used for temp. allocations. */ + apr_pool_t *pool; + + /* the cache to dump the statistics for */ + svn_cache__t *cache; +}; + +/* APR pool cleanup handler that will printf the statistics of the + cache referenced by the baton in BATON_VOID. */ +static apr_status_t +dump_cache_statistics(void *baton_void) +{ + struct dump_cache_baton_t *baton = baton_void; + + apr_status_t result = APR_SUCCESS; + svn_cache__info_t info; + svn_string_t *text_stats; + apr_array_header_t *lines; + int i; + + svn_error_t *err = svn_cache__get_info(baton->cache, + &info, + TRUE, + baton->pool); + + /* skip unused caches */ + if (! err && (info.gets > 0 || info.sets > 0)) + { + text_stats = svn_cache__format_info(&info, TRUE, baton->pool); + lines = svn_cstring_split(text_stats->data, "\n", FALSE, baton->pool); + + for (i = 0; i < lines->nelts; ++i) + { + const char *line = APR_ARRAY_IDX(lines, i, const char *); +#ifdef SVN_DEBUG + SVN_DBG(("%s\n", line)); +#endif + } + } + + /* process error returns */ + if (err) + { + result = err->apr_err; + svn_error_clear(err); + } + + return result; +} + +static apr_status_t +dump_global_cache_statistics(void *baton_void) +{ + apr_pool_t *pool = baton_void; + + svn_cache__info_t *info = svn_cache__membuffer_get_global_info(pool); + svn_string_t *text_stats = svn_cache__format_info(info, FALSE, pool); + apr_array_header_t *lines = svn_cstring_split(text_stats->data, "\n", + FALSE, pool); + + int i; + for (i = 0; i < lines->nelts; ++i) + { + const char *line = APR_ARRAY_IDX(lines, i, const char *); +#ifdef SVN_DEBUG + SVN_DBG(("%s\n", line)); +#endif + } + + return APR_SUCCESS; +} + +#endif /* SVN_DEBUG_CACHE_DUMP_STATS */ + +/* This function sets / registers the required callbacks for a given + * not transaction-specific CACHE object in FS, if CACHE is not NULL. + * + * All these svn_cache__t instances shall be handled uniformly. Unless + * ERROR_HANDLER is NULL, register it for the given CACHE in FS. + */ +static svn_error_t * +init_callbacks(svn_cache__t *cache, + svn_fs_t *fs, + svn_cache__error_handler_t error_handler, + apr_pool_t *pool) +{ + if (cache != NULL) + { +#ifdef SVN_DEBUG_CACHE_DUMP_STATS + + /* schedule printing the access statistics upon pool cleanup, + * i.e. end of FSX session. + */ + struct dump_cache_baton_t *baton; + + baton = apr_palloc(pool, sizeof(*baton)); + baton->pool = pool; + baton->cache = cache; + + apr_pool_cleanup_register(pool, + baton, + dump_cache_statistics, + apr_pool_cleanup_null); +#endif + + if (error_handler) + SVN_ERR(svn_cache__set_error_handler(cache, + error_handler, + fs, + pool)); + + } + + return SVN_NO_ERROR; +} + +/* Sets *CACHE_P to cache instance based on provided options. + * Creates memcache if MEMCACHE is not NULL. Creates membuffer cache if + * MEMBUFFER is not NULL. Fallbacks to inprocess cache if MEMCACHE and + * MEMBUFFER are NULL and pages is non-zero. Sets *CACHE_P to NULL + * otherwise. Use the given PRIORITY class for the new cache. If it + * is 0, then use the default priority class. + * + * Unless NO_HANDLER is true, register an error handler that reports errors + * as warnings to the FS warning callback. + * + * Cache is allocated in POOL. + * */ +static svn_error_t * +create_cache(svn_cache__t **cache_p, + svn_memcache_t *memcache, + svn_membuffer_t *membuffer, + apr_int64_t pages, + apr_int64_t items_per_page, + svn_cache__serialize_func_t serializer, + svn_cache__deserialize_func_t deserializer, + apr_ssize_t klen, + const char *prefix, + apr_uint32_t priority, + svn_fs_t *fs, + svn_boolean_t no_handler, + apr_pool_t *pool) +{ + svn_cache__error_handler_t error_handler = no_handler + ? NULL + : warn_and_fail_on_cache_errors; + if (priority == 0) + priority = SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY; + + if (memcache) + { + SVN_ERR(svn_cache__create_memcache(cache_p, memcache, + serializer, deserializer, klen, + prefix, pool)); + error_handler = no_handler + ? NULL + : warn_and_continue_on_cache_errors; + } + else if (membuffer) + { + SVN_ERR(svn_cache__create_membuffer_cache( + cache_p, membuffer, serializer, deserializer, + klen, prefix, priority, FALSE, pool)); + } + else if (pages) + { + SVN_ERR(svn_cache__create_inprocess( + cache_p, serializer, deserializer, klen, pages, + items_per_page, FALSE, prefix, pool)); + } + else + { + *cache_p = NULL; + } + + SVN_ERR(init_callbacks(*cache_p, fs, error_handler, pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__initialize_caches(svn_fs_t *fs, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + const char *prefix = apr_pstrcat(pool, + "fsx:", fs->uuid, + "/", normalize_key_part(fs->path, pool), + ":", + (char *)NULL); + svn_memcache_t *memcache; + svn_membuffer_t *membuffer; + svn_boolean_t no_handler; + svn_boolean_t cache_txdeltas; + svn_boolean_t cache_fulltexts; + svn_boolean_t cache_revprops; + const char *cache_namespace; + + /* Evaluating the cache configuration. */ + SVN_ERR(read_config(&memcache, + &no_handler, + &cache_namespace, + &cache_txdeltas, + &cache_fulltexts, + &cache_revprops, + fs, + pool)); + + prefix = apr_pstrcat(pool, "ns:", cache_namespace, ":", prefix, NULL); + + membuffer = svn_cache__get_global_membuffer_cache(); + + /* General rules for assigning cache priorities: + * + * - Data that can be reconstructed from other elements has low prio + * (e.g. fulltexts, directories etc.) + * - Index data required to find any of the other data has high prio + * (e.g. noderevs, L2P and P2L index pages) + * - everthing else should use default prio + */ + +#ifdef SVN_DEBUG_CACHE_DUMP_STATS + + /* schedule printing the global access statistics upon pool cleanup, + * i.e. end of FSX session. + */ + if (membuffer) + apr_pool_cleanup_register(pool, + pool, + dump_global_cache_statistics, + apr_pool_cleanup_null); +#endif + + /* Make the cache for revision roots. For the vast majority of + * commands, this is only going to contain a few entries (svnadmin + * dump/verify is an exception here), so to reduce overhead let's + * try to keep it to just one page. I estimate each entry has about + * 72 bytes of overhead (svn_revnum_t key, svn_fs_id_t + + * id_private_t + 3 strings for value, and the cache_entry); the + * default pool size is 8192, so about a hundred should fit + * comfortably. */ + SVN_ERR(create_cache(&(ffd->rev_root_id_cache), + NULL, + membuffer, + 1, 100, + svn_fs_x__serialize_id, + svn_fs_x__deserialize_id, + sizeof(svn_revnum_t), + apr_pstrcat(pool, prefix, "RRI", (char *)NULL), + 0, + fs, + no_handler, + fs->pool)); + + /* Rough estimate: revision DAG nodes have size around 320 bytes, so + * let's put 16 on a page. */ + SVN_ERR(create_cache(&(ffd->rev_node_cache), + NULL, + membuffer, + 1024, 16, + svn_fs_x__dag_serialize, + svn_fs_x__dag_deserialize, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "DAG", (char *)NULL), + SVN_CACHE__MEMBUFFER_LOW_PRIORITY, + fs, + no_handler, + fs->pool)); + + /* 1st level DAG node cache */ + ffd->dag_node_cache = svn_fs_x__create_dag_cache(pool); + + /* Very rough estimate: 1K per directory. */ + SVN_ERR(create_cache(&(ffd->dir_cache), + NULL, + membuffer, + 1024, 8, + svn_fs_x__serialize_dir_entries, + svn_fs_x__deserialize_dir_entries, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "DIR", (char *)NULL), + SVN_CACHE__MEMBUFFER_LOW_PRIORITY, + fs, + no_handler, + fs->pool)); + + /* Only 16 bytes per entry (a revision number + the corresponding offset). + Since we want ~8k pages, that means 512 entries per page. */ + SVN_ERR(create_cache(&(ffd->packed_offset_cache), + NULL, + membuffer, + 32, 1, + svn_fs_x__serialize_manifest, + svn_fs_x__deserialize_manifest, + sizeof(svn_revnum_t), + apr_pstrcat(pool, prefix, "PACK-MANIFEST", + (char *)NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool)); + + /* initialize node revision cache, if caching has been enabled */ + SVN_ERR(create_cache(&(ffd->node_revision_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_node_revision, + svn_fs_x__deserialize_node_revision, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "NODEREVS", (char *)NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool)); + + /* initialize representation header cache, if caching has been enabled */ + SVN_ERR(create_cache(&(ffd->rep_header_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_rep_header, + svn_fs_x__deserialize_rep_header, + sizeof(representation_cache_key_t), + apr_pstrcat(pool, prefix, "REPHEADER", (char *)NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool)); + + /* initialize node change list cache, if caching has been enabled */ + SVN_ERR(create_cache(&(ffd->changes_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_changes, + svn_fs_x__deserialize_changes, + sizeof(svn_revnum_t), + apr_pstrcat(pool, prefix, "CHANGES", (char *)NULL), + 0, + fs, + no_handler, + fs->pool)); + + /* if enabled, cache fulltext and other derived information */ + if (cache_fulltexts) + { + SVN_ERR(create_cache(&(ffd->fulltext_cache), + memcache, + membuffer, + 0, 0, /* Do not use inprocess cache */ + /* Values are svn_stringbuf_t */ + NULL, NULL, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "TEXT", (char *)NULL), + SVN_CACHE__MEMBUFFER_LOW_PRIORITY, + fs, + no_handler, + fs->pool)); + + SVN_ERR(create_cache(&(ffd->properties_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_properties, + svn_fs_x__deserialize_properties, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "PROP", + (char *)NULL), + SVN_CACHE__MEMBUFFER_LOW_PRIORITY, + fs, + no_handler, + fs->pool)); + + SVN_ERR(create_cache(&(ffd->mergeinfo_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_mergeinfo, + svn_fs_x__deserialize_mergeinfo, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "MERGEINFO", + (char *)NULL), + 0, + fs, + no_handler, + fs->pool)); + + SVN_ERR(create_cache(&(ffd->mergeinfo_existence_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + /* Values are svn_stringbuf_t */ + NULL, NULL, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "HAS_MERGEINFO", + (char *)NULL), + 0, + fs, + no_handler, + fs->pool)); + } + else + { + ffd->fulltext_cache = NULL; + ffd->properties_cache = NULL; + ffd->mergeinfo_cache = NULL; + ffd->mergeinfo_existence_cache = NULL; + } + + /* initialize revprop cache, if full-text caching has been enabled */ + if (cache_revprops) + { + SVN_ERR(create_cache(&(ffd->revprop_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_properties, + svn_fs_x__deserialize_properties, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "REVPROP", + (char *)NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool)); + } + else + { + ffd->revprop_cache = NULL; + } + + /* if enabled, cache text deltas and their combinations */ + if (cache_txdeltas) + { + SVN_ERR(create_cache(&(ffd->txdelta_window_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_txdelta_window, + svn_fs_x__deserialize_txdelta_window, + sizeof(window_cache_key_t), + apr_pstrcat(pool, prefix, "TXDELTA_WINDOW", + (char *)NULL), + 0, + fs, + no_handler, + fs->pool)); + + SVN_ERR(create_cache(&(ffd->combined_window_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + /* Values are svn_stringbuf_t */ + NULL, NULL, + sizeof(window_cache_key_t), + apr_pstrcat(pool, prefix, "COMBINED_WINDOW", + (char *)NULL), + SVN_CACHE__MEMBUFFER_LOW_PRIORITY, + fs, + no_handler, + fs->pool)); + } + else + { + ffd->txdelta_window_cache = NULL; + ffd->combined_window_cache = NULL; + } + + SVN_ERR(create_cache(&(ffd->noderevs_container_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_noderevs_container, + svn_fs_x__deserialize_noderevs_container, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "NODEREVSCNT", + (char *)NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool)); + SVN_ERR(create_cache(&(ffd->changes_container_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_changes_container, + svn_fs_x__deserialize_changes_container, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "CHANGESCNT", + (char *)NULL), + 0, + fs, + no_handler, + fs->pool)); + SVN_ERR(create_cache(&(ffd->reps_container_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_reps_container, + svn_fs_x__deserialize_reps_container, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "REPSCNT", + (char *)NULL), + 0, + fs, + no_handler, + fs->pool)); + + SVN_ERR(create_cache(&(ffd->l2p_header_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_l2p_header, + svn_fs_x__deserialize_l2p_header, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "L2P_HEADER", + (char *)NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool)); + SVN_ERR(create_cache(&(ffd->l2p_page_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_l2p_page, + svn_fs_x__deserialize_l2p_page, + sizeof(svn_fs_x__page_cache_key_t), + apr_pstrcat(pool, prefix, "L2P_PAGE", + (char *)NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool)); + SVN_ERR(create_cache(&(ffd->p2l_header_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_p2l_header, + svn_fs_x__deserialize_p2l_header, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "P2L_HEADER", + (char *)NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool)); + SVN_ERR(create_cache(&(ffd->p2l_page_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_p2l_page, + svn_fs_x__deserialize_p2l_page, + sizeof(svn_fs_x__page_cache_key_t), + apr_pstrcat(pool, prefix, "P2L_PAGE", + (char *)NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool)); + + return SVN_NO_ERROR; +} + +/* Baton to be used for the remove_txn_cache() pool cleanup function, */ +struct txn_cleanup_baton_t +{ + /* the cache to reset */ + svn_cache__t *txn_cache; + + /* the position where to reset it */ + svn_cache__t **to_reset; +}; + +/* APR pool cleanup handler that will reset the cache pointer given in + BATON_VOID. */ +static apr_status_t +remove_txn_cache(void *baton_void) +{ + struct txn_cleanup_baton_t *baton = baton_void; + + /* be careful not to hurt performance by resetting newer txn's caches. */ + if (*baton->to_reset == baton->txn_cache) + { + /* This is equivalent to calling svn_fs_x__reset_txn_caches(). */ + *baton->to_reset = NULL; + } + + return APR_SUCCESS; +} + +/* This function sets / registers the required callbacks for a given + * transaction-specific *CACHE object, if CACHE is not NULL and a no-op + * otherwise. In particular, it will ensure that *CACHE gets reset to NULL + * upon POOL destruction latest. + */ +static void +init_txn_callbacks(svn_cache__t **cache, + apr_pool_t *pool) +{ + if (*cache != NULL) + { + struct txn_cleanup_baton_t *baton; + + baton = apr_palloc(pool, sizeof(*baton)); + baton->txn_cache = *cache; + baton->to_reset = cache; + + apr_pool_cleanup_register(pool, + baton, + remove_txn_cache, + apr_pool_cleanup_null); + } +} + +svn_error_t * +svn_fs_x__initialize_txn_caches(svn_fs_t *fs, + const char *txn_id, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + + /* Transaction content needs to be carefully prefixed to virtually + eliminate any chance for conflicts. The (repo, txn_id) pair + should be unique but if a transaction fails, it might be possible + to start a new transaction later that receives the same id. + Therefore, throw in a uuid as well - just to be sure. */ + const char *prefix = apr_pstrcat(pool, + "fsx:", fs->uuid, + "/", fs->path, + ":", txn_id, + ":", svn_uuid_generate(pool), ":", + (char *)NULL); + + /* We don't support caching for concurrent transactions in the SAME + * FSX session. Maybe, you forgot to clean POOL. */ + if (ffd->txn_dir_cache != NULL || ffd->concurrent_transactions) + { + ffd->txn_dir_cache = NULL; + ffd->concurrent_transactions = TRUE; + + return SVN_NO_ERROR; + } + + /* create a txn-local directory cache */ + SVN_ERR(create_cache(&ffd->txn_dir_cache, + NULL, + svn_cache__get_global_membuffer_cache(), + 1024, 8, + svn_fs_x__serialize_dir_entries, + svn_fs_x__deserialize_dir_entries, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "TXNDIR", + (char *)NULL), + 0, + fs, + TRUE, + pool)); + + /* reset the transaction-specific cache if the pool gets cleaned up. */ + init_txn_callbacks(&(ffd->txn_dir_cache), pool); + + return SVN_NO_ERROR; +} + +void +svn_fs_x__reset_txn_caches(svn_fs_t *fs) +{ + /* we can always just reset the caches. This may degrade performance but + * can never cause in incorrect behavior. */ + + fs_x_data_t *ffd = fs->fsap_data; + ffd->txn_dir_cache = NULL; +}
diff --git a/subversion/libsvn_fs_x/changes.c b/subversion/libsvn_fs_x/changes.c new file mode 100644 index 0000000..358688a --- /dev/null +++ b/subversion/libsvn_fs_x/changes.c
@@ -0,0 +1,579 @@ +/* changes.h --- FSX changed paths lists container + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "svn_private_config.h" + +#include "private/svn_packed_data.h" + +#include "changes.h" +#include "string_table.h" +#include "temp_serializer.h" + +/* These flags will be used with the FLAGS field in binary_change_t. + */ + +/* the change contains a text modification */ +#define CHANGE_TEXT_MOD 0x00001 + +/* the change contains a property modification */ +#define CHANGE_PROP_MOD 0x00002 + +/* the last part (rev_id) of node revision ID is a transaction ID */ +#define CHANGE_TXN_NODE 0x00004 + +/* (flags & CHANGE_NODE_MASK) >> CHANGE_NODE_SHIFT extracts the node type */ +#define CHANGE_NODE_SHIFT 0x00003 +#define CHANGE_NODE_MASK 0x00018 + +/* node types according to svn_node_kind_t */ +#define CHANGE_NODE_NONE 0x00000 +#define CHANGE_NODE_FILE 0x00008 +#define CHANGE_NODE_DIR 0x00010 +#define CHANGE_NODE_UNKNOWN 0x00018 + +/* (flags & CHANGE_KIND_MASK) >> CHANGE_KIND_SHIFT extracts the change type */ +#define CHANGE_KIND_SHIFT 0x00005 +#define CHANGE_KIND_MASK 0x000E0 + +/* node types according to svn_fs_path_change_kind_t */ +#define CHANGE_KIND_MODIFY 0x00000 +#define CHANGE_KIND_ADD 0x00020 +#define CHANGE_KIND_DELETE 0x00040 +#define CHANGE_KIND_REPLACE 0x00060 +#define CHANGE_KIND_RESET 0x00080 + +/* Our internal representation of a change */ +typedef struct binary_change_t +{ + /* define the kind of change and what specific information is present */ + int flags; + + /* Path of the change. */ + apr_size_t path; + + /* copy-from information. + * Not present if COPYFROM_REV is SVN_INVALID_REVNUM. */ + svn_revnum_t copyfrom_rev; + apr_size_t copyfrom_path; + + /* Relevant parts of the node revision ID of the change. + * Empty, if REV_ID is not "used". */ + svn_fs_x__id_part_t node_id; + svn_fs_x__id_part_t copy_id; + svn_fs_x__id_part_t rev_id; /* txn ID if CHANGE_TXN_NODE set in FLAGS */ + +} binary_change_t; + +/* The actual container object. Change lists are concatenated into CHANGES + * and and their begins and ends are stored in OFFSETS. + */ +struct svn_fs_x__changes_t +{ + /* The paths - either in 'builder' mode or finalized mode. + * The respective other pointer will be NULL. */ + string_table_builder_t *builder; + string_table_t *paths; + + /* All changes of all change lists concatenated. + * Array elements are binary_change_t.structs (not pointer!) */ + apr_array_header_t *changes; + + /* [Offsets[index] .. Offsets[index+1]) is the range in CHANGES that + * forms the contents of change list INDEX. */ + apr_array_header_t *offsets; +}; + +/* Create and return a new container object, allocated in POOL and with + * and initial capacity of INITIAL_COUNT changes. The PATH and BUILDER + * members must be initialized by the caller afterwards. + */ +static svn_fs_x__changes_t * +changes_create_body(apr_size_t initial_count, + apr_pool_t *pool) +{ + svn_fs_x__changes_t *changes = apr_pcalloc(pool, sizeof(*changes)); + + changes->changes = apr_array_make(pool, (int)initial_count, + sizeof(binary_change_t)); + changes->offsets = apr_array_make(pool, 16, sizeof(int)); + APR_ARRAY_PUSH(changes->offsets, int) = 0; + + return changes; +} + +svn_fs_x__changes_t * +svn_fs_x__changes_create(apr_size_t initial_count, + apr_pool_t *pool) +{ + svn_fs_x__changes_t *changes = changes_create_body(initial_count, pool); + changes->builder = svn_fs_x__string_table_builder_create(pool); + + return changes; +} + +/* Add CHANGE to the latest change list in CHANGES. + */ +static svn_error_t * +append_change(svn_fs_x__changes_t *changes, + change_t *change) +{ + binary_change_t binary_change = { 0 }; + svn_boolean_t is_txn_id; + svn_fs_path_change2_t *info; + + /* CHANGE must be sufficiently complete */ + SVN_ERR_ASSERT(change); + SVN_ERR_ASSERT(change->path.data); + + /* define the kind of change and what specific information is present */ + info = &change->info; + is_txn_id = info->node_rev_id && svn_fs_x__id_is_txn(info->node_rev_id); + binary_change.flags = (info->text_mod ? CHANGE_TEXT_MOD : 0) + | (info->prop_mod ? CHANGE_PROP_MOD : 0) + | (is_txn_id ? CHANGE_TXN_NODE : 0) + | ((int)info->change_kind << CHANGE_KIND_SHIFT) + | ((int)info->node_kind << CHANGE_NODE_SHIFT); + + /* Path of the change. */ + binary_change.path + = svn_fs_x__string_table_builder_add(changes->builder, + change->path.data, + change->path.len); + + /* copy-from information, if presence is indicated by FLAGS */ + if (SVN_IS_VALID_REVNUM(info->copyfrom_rev)) + { + binary_change.copyfrom_rev = info->copyfrom_rev; + binary_change.copyfrom_path + = svn_fs_x__string_table_builder_add(changes->builder, + info->copyfrom_path, + 0); + } + else + { + binary_change.copyfrom_rev = SVN_INVALID_REVNUM; + binary_change.copyfrom_path = 0; + } + + /* Relevant parts of the revision ID of the change. */ + if (info->node_rev_id) + { + binary_change.node_id = *svn_fs_x__id_node_id(info->node_rev_id); + binary_change.copy_id = *svn_fs_x__id_copy_id(info->node_rev_id); + binary_change.rev_id = is_txn_id + ? *svn_fs_x__id_txn_id(info->node_rev_id) + : *svn_fs_x__id_rev_item(info->node_rev_id); + } + else + { + svn_fs_x__id_txn_reset(&binary_change.rev_id); + } + + APR_ARRAY_PUSH(changes->changes, binary_change_t) = binary_change; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__changes_append_list(apr_size_t *list_index, + svn_fs_x__changes_t *changes, + apr_array_header_t *list) +{ + int i; + + /* CHANGES must be in 'builder' mode */ + SVN_ERR_ASSERT(changes->builder); + SVN_ERR_ASSERT(changes->paths == NULL); + + /* simply append the list and all changes */ + for (i = 0; i < list->nelts; ++i) + append_change(changes, APR_ARRAY_IDX(list, i, change_t *)); + + /* terminate the list by storing the next changes offset */ + APR_ARRAY_PUSH(changes->offsets, int) = changes->changes->nelts; + *list_index = (apr_size_t)(changes->offsets->nelts - 2); + + return SVN_NO_ERROR; +} + +apr_size_t +svn_fs_x__changes_estimate_size(const svn_fs_x__changes_t *changes) +{ + /* CHANGES must be in 'builder' mode */ + if (changes->builder == NULL) + return 0; + + /* string table code makes its own prediction, + * changes should be < 10 bytes each, + * some static overhead should be assumed */ + return svn_fs_x__string_table_builder_estimate_size(changes->builder) + + changes->changes->nelts * 10 + + 100; +} + +svn_error_t * +svn_fs_x__changes_get_list(apr_array_header_t **list, + const svn_fs_x__changes_t *changes, + apr_size_t idx, + apr_pool_t *pool) +{ + int first; + int last; + int i; + + /* CHANGES must be in 'finalized' mode */ + SVN_ERR_ASSERT(changes->builder == NULL); + SVN_ERR_ASSERT(changes->paths); + + /* validate index */ + if (idx + 1 >= (apr_size_t)changes->offsets->nelts) + return svn_error_createf(SVN_ERR_FS_CONTAINER_INDEX, NULL, + _("Changes list index %" APR_SIZE_T_FMT + " exceeds container size %d"), + idx, changes->offsets->nelts - 1); + + /* range of changes to return */ + first = APR_ARRAY_IDX(changes->offsets, (int)idx, int); + last = APR_ARRAY_IDX(changes->offsets, (int)idx + 1, int); + + /* construct result */ + *list = apr_array_make(pool, last - first, sizeof(change_t*)); + for (i = first; i < last; ++i) + { + const binary_change_t *binary_change + = &APR_ARRAY_IDX(changes->changes, i, binary_change_t); + + /* convert BINARY_CHANGE into a standard FSX change_t */ + change_t *change = apr_pcalloc(pool, sizeof(*change)); + svn_fs_path_change2_t *info = &change->info; + change->path.data = svn_fs_x__string_table_get(changes->paths, + binary_change->path, + &change->path.len, + pool); + + if (svn_fs_x__id_txn_used(&binary_change->rev_id)) + info->node_rev_id + = (binary_change->flags & CHANGE_TXN_NODE) + ? svn_fs_x__id_txn_create(&binary_change->node_id, + &binary_change->copy_id, + &binary_change->rev_id, + pool) + : svn_fs_x__id_rev_create(&binary_change->node_id, + &binary_change->copy_id, + &binary_change->rev_id, + pool); + + info->change_kind = (svn_fs_path_change_kind_t) + ((binary_change->flags & CHANGE_KIND_MASK) >> CHANGE_KIND_SHIFT); + info->text_mod = (binary_change->flags & CHANGE_TEXT_MOD) != 0; + info->prop_mod = (binary_change->flags & CHANGE_PROP_MOD) != 0; + info->node_kind = (svn_node_kind_t) + ((binary_change->flags & CHANGE_NODE_MASK) >> CHANGE_NODE_SHIFT); + + info->copyfrom_rev = binary_change->copyfrom_rev; + info->copyfrom_known = TRUE; + if (SVN_IS_VALID_REVNUM(binary_change->copyfrom_rev)) + info->copyfrom_path + = svn_fs_x__string_table_get(changes->paths, + binary_change->copyfrom_path, + NULL, + pool); + + /* add it to the result */ + APR_ARRAY_PUSH(*list, change_t*) = change; + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__write_changes_container(svn_stream_t *stream, + const svn_fs_x__changes_t *changes, + apr_pool_t *pool) +{ + int i; + + string_table_t *paths = changes->paths + ? changes->paths + : svn_fs_x__string_table_create(changes->builder, + pool); + + svn_packed__data_root_t *root = svn_packed__data_create_root(pool); + + /* one top-level stream for each array */ + svn_packed__int_stream_t *offsets_stream + = svn_packed__create_int_stream(root, TRUE, FALSE); + svn_packed__int_stream_t *changes_stream + = svn_packed__create_int_stream(root, FALSE, FALSE); + + /* structure the CHANGES_STREAM such we can extract much of the redundancy + * from the binary_change_t structs */ + svn_packed__create_int_substream(changes_stream, TRUE, FALSE); + svn_packed__create_int_substream(changes_stream, TRUE, FALSE); + svn_packed__create_int_substream(changes_stream, TRUE, TRUE); + svn_packed__create_int_substream(changes_stream, TRUE, FALSE); + svn_packed__create_int_substream(changes_stream, TRUE, TRUE); + svn_packed__create_int_substream(changes_stream, TRUE, FALSE); + svn_packed__create_int_substream(changes_stream, TRUE, TRUE); + svn_packed__create_int_substream(changes_stream, TRUE, FALSE); + svn_packed__create_int_substream(changes_stream, TRUE, TRUE); + svn_packed__create_int_substream(changes_stream, TRUE, FALSE); + + /* serialize offsets array */ + for (i = 0; i < changes->offsets->nelts; ++i) + svn_packed__add_uint(offsets_stream, + APR_ARRAY_IDX(changes->offsets, i, int)); + + /* serialize changes array */ + for (i = 0; i < changes->changes->nelts; ++i) + { + const binary_change_t *change + = &APR_ARRAY_IDX(changes->changes, i, binary_change_t); + + svn_packed__add_uint(changes_stream, change->flags); + svn_packed__add_uint(changes_stream, change->path); + + svn_packed__add_uint(changes_stream, change->copyfrom_rev); + svn_packed__add_uint(changes_stream, change->copyfrom_path); + + svn_packed__add_uint(changes_stream, change->node_id.revision); + svn_packed__add_uint(changes_stream, change->node_id.number); + svn_packed__add_uint(changes_stream, change->copy_id.revision); + svn_packed__add_uint(changes_stream, change->copy_id.number); + svn_packed__add_uint(changes_stream, change->rev_id.revision); + svn_packed__add_uint(changes_stream, change->rev_id.number); + } + + /* write to disk */ + SVN_ERR(svn_fs_x__write_string_table(stream, paths, pool)); + SVN_ERR(svn_packed__data_write(stream, root, pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__read_changes_container(svn_fs_x__changes_t **changes_p, + svn_stream_t *stream, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + apr_size_t i; + apr_size_t count; + + svn_fs_x__changes_t *changes = apr_pcalloc(result_pool, sizeof(*changes)); + + svn_packed__data_root_t *root; + svn_packed__int_stream_t *offsets_stream; + svn_packed__int_stream_t *changes_stream; + + /* read from disk */ + SVN_ERR(svn_fs_x__read_string_table(&changes->paths, stream, + result_pool, scratch_pool)); + + SVN_ERR(svn_packed__data_read(&root, stream, result_pool, scratch_pool)); + offsets_stream = svn_packed__first_int_stream(root); + changes_stream = svn_packed__next_int_stream(offsets_stream); + + /* read offsets array */ + count = svn_packed__int_count(offsets_stream); + changes->offsets = apr_array_make(result_pool, count, sizeof(int)); + for (i = 0; i < count; ++i) + APR_ARRAY_PUSH(changes->offsets, int) + = (int)svn_packed__get_uint(offsets_stream); + + /* read changes array */ + count + = svn_packed__int_count(svn_packed__first_int_substream(changes_stream)); + changes->changes + = apr_array_make(result_pool, count, sizeof(binary_change_t)); + for (i = 0; i < count; ++i) + { + binary_change_t change; + + change.flags = svn_packed__get_uint(changes_stream); + change.path = svn_packed__get_uint(changes_stream); + + change.copyfrom_rev = svn_packed__get_uint(changes_stream); + change.copyfrom_path = svn_packed__get_uint(changes_stream); + + change.node_id.revision = svn_packed__get_uint(changes_stream); + change.node_id.number = svn_packed__get_uint(changes_stream); + change.copy_id.revision = svn_packed__get_uint(changes_stream); + change.copy_id.number = svn_packed__get_uint(changes_stream); + change.rev_id.revision = svn_packed__get_uint(changes_stream); + change.rev_id.number = svn_packed__get_uint(changes_stream); + + APR_ARRAY_PUSH(changes->changes, binary_change_t) = change; + } + + *changes_p = changes; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__serialize_changes_container(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + svn_fs_x__changes_t *changes = in; + svn_stringbuf_t *serialized; + + /* make a guesstimate on the size of the serialized data. Erring on the + * low side will cause the serializer to re-alloc its buffer. */ + apr_size_t size + = changes->changes->elt_size * changes->changes->nelts + + changes->offsets->elt_size * changes->offsets->nelts + + 10 * changes->changes->elt_size + + 100; + + /* serialize array header and all its elements */ + svn_temp_serializer__context_t *context + = svn_temp_serializer__init(changes, sizeof(*changes), size, pool); + + /* serialize sub-structures */ + svn_fs_x__serialize_string_table(context, &changes->paths); + svn_fs_x__serialize_apr_array(context, &changes->changes); + svn_fs_x__serialize_apr_array(context, &changes->offsets); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_changes_container(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + svn_fs_x__changes_t *changes = (svn_fs_x__changes_t *)data; + + /* de-serialize sub-structures */ + svn_fs_x__deserialize_string_table(changes, &changes->paths); + svn_fs_x__deserialize_apr_array(changes, &changes->changes, pool); + svn_fs_x__deserialize_apr_array(changes, &changes->offsets, pool); + + /* done */ + *out = changes; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__changes_get_list_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool) +{ + int first; + int last; + int i; + apr_array_header_t *list; + + apr_uint32_t idx = *(apr_uint32_t *)baton; + const svn_fs_x__changes_t *container = data; + + /* resolve all the sub-container pointers we need */ + const string_table_t *paths + = svn_temp_deserializer__ptr(container, + (const void *const *)&container->paths); + const apr_array_header_t *serialized_offsets + = svn_temp_deserializer__ptr(container, + (const void *const *)&container->offsets); + const apr_array_header_t *serialized_changes + = svn_temp_deserializer__ptr(container, + (const void *const *)&container->changes); + const int *offsets + = svn_temp_deserializer__ptr(serialized_offsets, + (const void *const *)&serialized_offsets->elts); + const binary_change_t *changes + = svn_temp_deserializer__ptr(serialized_changes, + (const void *const *)&serialized_changes->elts); + + /* validate index */ + if (idx + 1 >= (apr_size_t)serialized_offsets->nelts) + return svn_error_createf(SVN_ERR_FS_CONTAINER_INDEX, NULL, + _("Changes list index %u exceeds container " + "size %d"), + (unsigned)idx, serialized_offsets->nelts - 1); + + /* range of changes to return */ + first = offsets[idx]; + last = offsets[idx+1]; + + /* construct result */ + list = apr_array_make(pool, last - first, sizeof(change_t*)); + + for (i = first; i < last; ++i) + { + const binary_change_t *binary_change = &changes[i]; + + /* convert BINARY_CHANGE into a standard FSX change_t */ + change_t *change = apr_pcalloc(pool, sizeof(*change)); + svn_fs_path_change2_t *info = &change->info; + change->path.data + = svn_fs_x__string_table_get_func(paths, binary_change->path, + &change->path.len, pool); + + if (svn_fs_x__id_txn_used(&binary_change->rev_id)) + info->node_rev_id + = (binary_change->flags & CHANGE_TXN_NODE) + ? svn_fs_x__id_txn_create(&binary_change->node_id, + &binary_change->copy_id, + &binary_change->rev_id, + pool) + : svn_fs_x__id_rev_create(&binary_change->node_id, + &binary_change->copy_id, + &binary_change->rev_id, + pool); + + info->change_kind = (svn_fs_path_change_kind_t) + ((binary_change->flags & CHANGE_KIND_MASK) >> CHANGE_KIND_SHIFT); + info->text_mod = (binary_change->flags & CHANGE_TEXT_MOD) != 0; + info->prop_mod = (binary_change->flags & CHANGE_PROP_MOD) != 0; + info->node_kind = (svn_node_kind_t) + ((binary_change->flags & CHANGE_NODE_MASK) >> CHANGE_NODE_SHIFT); + + info->copyfrom_rev = binary_change->copyfrom_rev; + info->copyfrom_known = TRUE; + if (SVN_IS_VALID_REVNUM(binary_change->copyfrom_rev)) + info->copyfrom_path + = svn_fs_x__string_table_get_func(paths, + binary_change->copyfrom_path, + NULL, + pool); + + /* add it to the result */ + APR_ARRAY_PUSH(list, change_t*) = change; + } + + *out = list; + + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_fs_x/changes.h b/subversion/libsvn_fs_x/changes.h new file mode 100644 index 0000000..5809e9c --- /dev/null +++ b/subversion/libsvn_fs_x/changes.h
@@ -0,0 +1,130 @@ +/* changes.h --- FSX changed paths lists container + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS__CHANGES_H +#define SVN_LIBSVN_FS__CHANGES_H + +#include "svn_io.h" +#include "fs.h" + +/* Entries in a revision's change list tend to be widely redundant (similar + * changes to similar paths). Even more so, change lists from a larger + * revision range also tend to overlap. + * + * In its serialized form, the svn_fs_x__changes_t container extracts most + * of that redundancy and the run-time representation is also much smaller + * than sum of the respective change_t* arrays. + * + * As with other containers, this one has two modes: 'construction', in + * which you may add data to it, and 'getter' in which there is only r/o + * access to the data. + */ + +/* An opaque collection of change lists (apr_array_header_t * of change_t*). + */ +typedef struct svn_fs_x__changes_t svn_fs_x__changes_t; + +/* Create and populate changes containers. */ + +/* Create and return a new changes container with an initial capacity of + * INITIAL_COUNT change_t objects. Allocate the result in POOL. + */ +svn_fs_x__changes_t * +svn_fs_x__changes_create(apr_size_t initial_count, + apr_pool_t *pool); + +/* Start a new change list CHANGES (implicitly terminating the previous one) + * and return its index in *LIST_INDEX. Append all changes from LIST to + * that new change list. + */ +svn_error_t * +svn_fs_x__changes_append_list(apr_size_t *list_index, + svn_fs_x__changes_t *changes, + apr_array_header_t *list); + +/* Return a rough estimate in bytes for the serialized representation + * of CHANGES. + */ +apr_size_t +svn_fs_x__changes_estimate_size(const svn_fs_x__changes_t *changes); + +/* Read changes containers. */ + +/* From CHANGES, extract the change list with the given IDX. Allocate + * the result in POOL and return it in *LIST. + */ +svn_error_t * +svn_fs_x__changes_get_list(apr_array_header_t **list, + const svn_fs_x__changes_t *changes, + apr_size_t idx, + apr_pool_t *pool); + +/* I/O interface. */ + +/* Write a serialized representation of CHANGES to STREAM. Use POOL for + * temporary allocations. + */ +svn_error_t * +svn_fs_x__write_changes_container(svn_stream_t *stream, + const svn_fs_x__changes_t *changes, + apr_pool_t *pool); + +/* Read a changes container from its serialized representation in STREAM. + * Allocate the result in RESULT_POOL and return it in *CHANGES_P. Use + * SCRATCH_POOL for temporary allocations. + */ +svn_error_t * +svn_fs_x__read_changes_container(svn_fs_x__changes_t **changes_p, + svn_stream_t *stream, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + +/* Implements #svn_cache__serialize_func_t for svn_fs_x__changes_t objects. + */ +svn_error_t * +svn_fs_x__serialize_changes_container(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/* Implements #svn_cache__deserialize_func_t for svn_fs_x__changes_t objects. + */ +svn_error_t * +svn_fs_x__deserialize_changes_container(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/* Implements svn_cache__partial_getter_func_t for svn_fs_x__changes_t, + * setting *OUT to the change list (apr_array_header_t *) selected by + * the apr_uint32_t index passed in as *BATON. This function is similar + * to svn_fs_x__changes_get_list but operates on the cache serialized + * representation of the container. + */ +svn_error_t * +svn_fs_x__changes_get_list_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool); + +#endif \ No newline at end of file
diff --git a/subversion/libsvn_fs_x/dag.c b/subversion/libsvn_fs_x/dag.c new file mode 100644 index 0000000..3f83809 --- /dev/null +++ b/subversion/libsvn_fs_x/dag.c
@@ -0,0 +1,1353 @@ +/* dag.c : DAG-like interface filesystem, private to libsvn_fs + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include <string.h> + +#include "svn_path.h" +#include "svn_error.h" +#include "svn_fs.h" +#include "svn_props.h" +#include "svn_pools.h" + +#include "dag.h" +#include "fs.h" +#include "fs_x.h" +#include "id.h" +#include "cached_data.h" +#include "transaction.h" + +#include "../libsvn_fs/fs-loader.h" + +#include "private/svn_fspath.h" +#include "svn_private_config.h" +#include "private/svn_temp_serializer.h" +#include "temp_serializer.h" + + +/* Initializing a filesystem. */ + +struct dag_node_t +{ + /* The filesystem this dag node came from. */ + svn_fs_t *fs; + + /* The node revision ID for this dag node, allocated in POOL. */ + svn_fs_id_t *id; + + /* In the special case that this node is the root of a transaction + that has not yet been modified, the node revision ID for this dag + node's predecessor; otherwise NULL. (Used in + svn_fs_node_created_rev.) */ + const svn_fs_id_t *fresh_root_predecessor_id; + + /* The node's type (file, dir, etc.) */ + svn_node_kind_t kind; + + /* The node's NODE-REVISION, or NULL if we haven't read it in yet. + This is allocated in this node's POOL. + + If you're willing to respect all the rules above, you can munge + this yourself, but you're probably better off just calling + `get_node_revision' and `set_node_revision', which take care of + things for you. */ + node_revision_t *node_revision; + + /* The pool to allocate NODE_REVISION in. */ + apr_pool_t *node_pool; + + /* the path at which this node was created. */ + const char *created_path; +}; + + + +/* Trivial helper/accessor functions. */ +svn_node_kind_t svn_fs_x__dag_node_kind(dag_node_t *node) +{ + return node->kind; +} + + +const svn_fs_id_t * +svn_fs_x__dag_get_id(const dag_node_t *node) +{ + return node->id; +} + + +const char * +svn_fs_x__dag_get_created_path(dag_node_t *node) +{ + return node->created_path; +} + + +svn_fs_t * +svn_fs_x__dag_get_fs(dag_node_t *node) +{ + return node->fs; +} + +void +svn_fs_x__dag_set_fs(dag_node_t *node, svn_fs_t *fs) +{ + node->fs = fs; +} + + +/* Dup NODEREV and all associated data into POOL. + Leaves the id and is_fresh_txn_root fields as zero bytes. */ +static node_revision_t * +copy_node_revision(node_revision_t *noderev, + apr_pool_t *pool) +{ + node_revision_t *nr = apr_pcalloc(pool, sizeof(*nr)); + nr->kind = noderev->kind; + if (noderev->predecessor_id) + nr->predecessor_id = svn_fs_x__id_copy(noderev->predecessor_id, pool); + nr->predecessor_count = noderev->predecessor_count; + if (noderev->copyfrom_path) + nr->copyfrom_path = apr_pstrdup(pool, noderev->copyfrom_path); + nr->copyfrom_rev = noderev->copyfrom_rev; + nr->copyroot_path = apr_pstrdup(pool, noderev->copyroot_path); + nr->copyroot_rev = noderev->copyroot_rev; + nr->data_rep = svn_fs_x__rep_copy(noderev->data_rep, pool); + nr->prop_rep = svn_fs_x__rep_copy(noderev->prop_rep, pool); + nr->mergeinfo_count = noderev->mergeinfo_count; + nr->has_mergeinfo = noderev->has_mergeinfo; + + if (noderev->created_path) + nr->created_path = apr_pstrdup(pool, noderev->created_path); + return nr; +} + + +/* Set *NODEREV_P to the cached node-revision for NODE. + If the node-revision was not already cached in NODE, read it in, + allocating the cache in NODE->NODE_POOL. + + If you plan to change the contents of NODE, be careful! We're + handing you a pointer directly to our cached node-revision, not + your own copy. If you change it as part of some operation, but + then some Berkeley DB function deadlocks or gets an error, you'll + need to back out your changes, or else the cache will reflect + changes that never got committed. It's probably best not to change + the structure at all. */ +static svn_error_t * +get_node_revision(node_revision_t **noderev_p, + dag_node_t *node) +{ + /* If we've already got a copy, there's no need to read it in. */ + if (! node->node_revision) + { + node_revision_t *noderev; + + SVN_ERR(svn_fs_x__get_node_revision(&noderev, node->fs, + node->id, node->node_pool)); + node->node_revision = noderev; + } + + /* Now NODE->node_revision is set. */ + *noderev_p = node->node_revision; + return SVN_NO_ERROR; +} + + +svn_boolean_t svn_fs_x__dag_check_mutable(const dag_node_t *node) +{ + return svn_fs_x__id_is_txn(svn_fs_x__dag_get_id(node)); +} + + +svn_error_t * +svn_fs_x__dag_get_node(dag_node_t **node, + svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool) +{ + dag_node_t *new_node; + node_revision_t *noderev; + + /* Construct the node. */ + new_node = apr_pcalloc(pool, sizeof(*new_node)); + new_node->fs = fs; + new_node->id = svn_fs_x__id_copy(id, pool); + + /* Grab the contents so we can inspect the node's kind and created path. */ + new_node->node_pool = pool; + SVN_ERR(get_node_revision(&noderev, new_node)); + + /* Initialize the KIND and CREATED_PATH attributes */ + new_node->kind = noderev->kind; + new_node->created_path = apr_pstrdup(pool, noderev->created_path); + + if (noderev->is_fresh_txn_root) + new_node->fresh_root_predecessor_id = noderev->predecessor_id; + else + new_node->fresh_root_predecessor_id = NULL; + + /* Return a fresh new node */ + *node = new_node; + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__dag_get_revision(svn_revnum_t *rev, + dag_node_t *node, + apr_pool_t *pool) +{ + /* In the special case that this is an unmodified transaction root, + we need to actually get the revision of the noderev's predecessor + (the revision root); see Issue #2608. */ + const svn_fs_id_t *correct_id = node->fresh_root_predecessor_id + ? node->fresh_root_predecessor_id : node->id; + + /* Look up the committed revision from the Node-ID. */ + *rev = svn_fs_x__id_rev(correct_id); + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__dag_get_predecessor_id(const svn_fs_id_t **id_p, + dag_node_t *node) +{ + node_revision_t *noderev; + + SVN_ERR(get_node_revision(&noderev, node)); + *id_p = noderev->predecessor_id; + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__dag_get_predecessor_count(int *count, + dag_node_t *node) +{ + node_revision_t *noderev; + + SVN_ERR(get_node_revision(&noderev, node)); + *count = noderev->predecessor_count; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__dag_get_mergeinfo_count(apr_int64_t *count, + dag_node_t *node) +{ + node_revision_t *noderev; + + SVN_ERR(get_node_revision(&noderev, node)); + *count = noderev->mergeinfo_count; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__dag_has_mergeinfo(svn_boolean_t *has_mergeinfo, + dag_node_t *node) +{ + node_revision_t *noderev; + + SVN_ERR(get_node_revision(&noderev, node)); + *has_mergeinfo = noderev->has_mergeinfo; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__dag_has_descendants_with_mergeinfo(svn_boolean_t *do_they, + dag_node_t *node) +{ + node_revision_t *noderev; + + if (node->kind != svn_node_dir) + { + *do_they = FALSE; + return SVN_NO_ERROR; + } + + SVN_ERR(get_node_revision(&noderev, node)); + if (noderev->mergeinfo_count > 1) + *do_they = TRUE; + else if (noderev->mergeinfo_count == 1 && !noderev->has_mergeinfo) + *do_they = TRUE; + else + *do_they = FALSE; + return SVN_NO_ERROR; +} + + +/*** Directory node functions ***/ + +/* Some of these are helpers for functions outside this section. */ + +/* Set *ID_P to the node-id for entry NAME in PARENT. If no such + entry, set *ID_P to NULL but do not error. The node-id is + allocated in POOL. */ +static svn_error_t * +dir_entry_id_from_node(const svn_fs_id_t **id_p, + dag_node_t *parent, + const char *name, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + svn_fs_dirent_t *dirent; + + SVN_ERR(svn_fs_x__dag_dir_entry(&dirent, parent, name, result_pool, + scratch_pool)); + *id_p = dirent ? dirent->id : NULL; + + return SVN_NO_ERROR; +} + + +/* Add or set in PARENT a directory entry NAME pointing to ID. + Allocations are done in POOL. + + Assumptions: + - PARENT is a mutable directory. + - ID does not refer to an ancestor of parent + - NAME is a single path component +*/ +static svn_error_t * +set_entry(dag_node_t *parent, + const char *name, + const svn_fs_id_t *id, + svn_node_kind_t kind, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + node_revision_t *parent_noderev; + + /* Get the parent's node-revision. */ + SVN_ERR(get_node_revision(&parent_noderev, parent)); + + /* Set the new entry. */ + return svn_fs_x__set_entry(parent->fs, txn_id, parent_noderev, name, id, + kind, pool); +} + + +/* Make a new entry named NAME in PARENT. If IS_DIR is true, then the + node revision the new entry points to will be a directory, else it + will be a file. The new node will be allocated in POOL. PARENT + must be mutable, and must not have an entry named NAME. + + Use POOL for all allocations, except caching the node_revision in PARENT. + */ +static svn_error_t * +make_entry(dag_node_t **child_p, + dag_node_t *parent, + const char *parent_path, + const char *name, + svn_boolean_t is_dir, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + const svn_fs_id_t *new_node_id; + node_revision_t new_noderev, *parent_noderev; + + /* Make sure that NAME is a single path component. */ + if (! svn_path_is_single_path_component(name)) + return svn_error_createf + (SVN_ERR_FS_NOT_SINGLE_PATH_COMPONENT, NULL, + _("Attempted to create a node with an illegal name '%s'"), name); + + /* Make sure that parent is a directory */ + if (parent->kind != svn_node_dir) + return svn_error_create + (SVN_ERR_FS_NOT_DIRECTORY, NULL, + _("Attempted to create entry in non-directory parent")); + + /* Check that the parent is mutable. */ + if (! svn_fs_x__dag_check_mutable(parent)) + return svn_error_createf + (SVN_ERR_FS_NOT_MUTABLE, NULL, + _("Attempted to clone child of non-mutable node")); + + /* Create the new node's NODE-REVISION */ + memset(&new_noderev, 0, sizeof(new_noderev)); + new_noderev.kind = is_dir ? svn_node_dir : svn_node_file; + new_noderev.created_path = svn_fspath__join(parent_path, name, pool); + + SVN_ERR(get_node_revision(&parent_noderev, parent)); + new_noderev.copyroot_path = apr_pstrdup(pool, + parent_noderev->copyroot_path); + new_noderev.copyroot_rev = parent_noderev->copyroot_rev; + new_noderev.copyfrom_rev = SVN_INVALID_REVNUM; + new_noderev.copyfrom_path = NULL; + + SVN_ERR(svn_fs_x__create_node + (&new_node_id, svn_fs_x__dag_get_fs(parent), &new_noderev, + svn_fs_x__id_copy_id(svn_fs_x__dag_get_id(parent)), + txn_id, pool)); + + /* Create a new dag_node_t for our new node */ + SVN_ERR(svn_fs_x__dag_get_node(child_p, svn_fs_x__dag_get_fs(parent), + new_node_id, pool)); + + /* We can safely call set_entry because we already know that + PARENT is mutable, and we just created CHILD, so we know it has + no ancestors (therefore, PARENT cannot be an ancestor of CHILD) */ + return set_entry(parent, name, svn_fs_x__dag_get_id(*child_p), + new_noderev.kind, txn_id, pool); +} + + +svn_error_t * +svn_fs_x__dag_dir_entries(apr_hash_t **entries, + dag_node_t *node, + apr_pool_t *pool) +{ + node_revision_t *noderev; + + SVN_ERR(get_node_revision(&noderev, node)); + + if (noderev->kind != svn_node_dir) + return svn_error_create(SVN_ERR_FS_NOT_DIRECTORY, NULL, + _("Can't get entries of non-directory")); + + return svn_fs_x__rep_contents_dir(entries, node->fs, noderev, pool); +} + +svn_error_t * +svn_fs_x__dag_dir_entry(svn_fs_dirent_t **dirent, + dag_node_t *node, + const char* name, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + node_revision_t *noderev; + SVN_ERR(get_node_revision(&noderev, node)); + + if (noderev->kind != svn_node_dir) + return svn_error_create(SVN_ERR_FS_NOT_DIRECTORY, NULL, + _("Can't get entries of non-directory")); + + /* Get a dirent hash for this directory. */ + return svn_fs_x__rep_contents_dir_entry(dirent, node->fs, noderev, name, + result_pool, scratch_pool); +} + + +svn_error_t * +svn_fs_x__dag_set_entry(dag_node_t *node, + const char *entry_name, + const svn_fs_id_t *id, + svn_node_kind_t kind, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + /* Check it's a directory. */ + if (node->kind != svn_node_dir) + return svn_error_create + (SVN_ERR_FS_NOT_DIRECTORY, NULL, + _("Attempted to set entry in non-directory node")); + + /* Check it's mutable. */ + if (! svn_fs_x__dag_check_mutable(node)) + return svn_error_create + (SVN_ERR_FS_NOT_MUTABLE, NULL, + _("Attempted to set entry in immutable node")); + + return set_entry(node, entry_name, id, kind, txn_id, pool); +} + + + +/*** Proplists. ***/ + +svn_error_t * +svn_fs_x__dag_get_proplist(apr_hash_t **proplist_p, + dag_node_t *node, + apr_pool_t *pool) +{ + node_revision_t *noderev; + apr_hash_t *proplist = NULL; + + SVN_ERR(get_node_revision(&noderev, node)); + + SVN_ERR(svn_fs_x__get_proplist(&proplist, node->fs, noderev, pool)); + + *proplist_p = proplist; + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__dag_set_proplist(dag_node_t *node, + apr_hash_t *proplist, + apr_pool_t *pool) +{ + node_revision_t *noderev; + + /* Sanity check: this node better be mutable! */ + if (! svn_fs_x__dag_check_mutable(node)) + { + svn_string_t *idstr = svn_fs_x__id_unparse(node->id, pool); + return svn_error_createf + (SVN_ERR_FS_NOT_MUTABLE, NULL, + "Can't set proplist on *immutable* node-revision %s", + idstr->data); + } + + /* Go get a fresh NODE-REVISION for this node. */ + SVN_ERR(get_node_revision(&noderev, node)); + + /* Set the new proplist. */ + return svn_fs_x__set_proplist(node->fs, noderev, proplist, pool); +} + + +svn_error_t * +svn_fs_x__dag_increment_mergeinfo_count(dag_node_t *node, + apr_int64_t increment, + apr_pool_t *pool) +{ + node_revision_t *noderev; + + /* Sanity check: this node better be mutable! */ + if (! svn_fs_x__dag_check_mutable(node)) + { + svn_string_t *idstr = svn_fs_x__id_unparse(node->id, pool); + return svn_error_createf + (SVN_ERR_FS_NOT_MUTABLE, NULL, + "Can't increment mergeinfo count on *immutable* node-revision %s", + idstr->data); + } + + if (increment == 0) + return SVN_NO_ERROR; + + /* Go get a fresh NODE-REVISION for this node. */ + SVN_ERR(get_node_revision(&noderev, node)); + + noderev->mergeinfo_count += increment; + if (noderev->mergeinfo_count < 0) + { + svn_string_t *idstr = svn_fs_x__id_unparse(node->id, pool); + return svn_error_createf + (SVN_ERR_FS_CORRUPT, NULL, + apr_psprintf(pool, + _("Can't increment mergeinfo count on node-revision %%s " + "to negative value %%%s"), + APR_INT64_T_FMT), + idstr->data, noderev->mergeinfo_count); + } + if (noderev->mergeinfo_count > 1 && noderev->kind == svn_node_file) + { + svn_string_t *idstr = svn_fs_x__id_unparse(node->id, pool); + return svn_error_createf + (SVN_ERR_FS_CORRUPT, NULL, + apr_psprintf(pool, + _("Can't increment mergeinfo count on *file* " + "node-revision %%s to %%%s (> 1)"), + APR_INT64_T_FMT), + idstr->data, noderev->mergeinfo_count); + } + + /* Flush it out. */ + return svn_fs_x__put_node_revision(node->fs, noderev->id, + noderev, FALSE, pool); +} + +svn_error_t * +svn_fs_x__dag_set_has_mergeinfo(dag_node_t *node, + svn_boolean_t has_mergeinfo, + apr_pool_t *pool) +{ + node_revision_t *noderev; + + /* Sanity check: this node better be mutable! */ + if (! svn_fs_x__dag_check_mutable(node)) + { + svn_string_t *idstr = svn_fs_x__id_unparse(node->id, pool); + return svn_error_createf + (SVN_ERR_FS_NOT_MUTABLE, NULL, + "Can't set mergeinfo flag on *immutable* node-revision %s", + idstr->data); + } + + /* Go get a fresh NODE-REVISION for this node. */ + SVN_ERR(get_node_revision(&noderev, node)); + + noderev->has_mergeinfo = has_mergeinfo; + + /* Flush it out. */ + return svn_fs_x__put_node_revision(node->fs, noderev->id, + noderev, FALSE, pool); +} + + +/*** Roots. ***/ + +svn_error_t * +svn_fs_x__dag_revision_root(dag_node_t **node_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + dag_node_t *new_node; + + /* Construct the node. */ + new_node = apr_pcalloc(pool, sizeof(*new_node)); + new_node->fs = fs; + SVN_ERR(svn_fs_x__rev_get_root(&new_node->id, fs, rev, pool)); + + /* Grab the contents so we can inspect the node's kind and created path. */ + new_node->node_pool = pool; + + /* Initialize the KIND and CREATED_PATH attributes */ + new_node->kind = svn_node_dir; + new_node->created_path = "/"; + new_node->fresh_root_predecessor_id = NULL; + + /* Return a fresh new node */ + *node_p = new_node; + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__dag_txn_root(dag_node_t **node_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + const svn_fs_id_t *root_id, *ignored; + + SVN_ERR(svn_fs_x__get_txn_ids(&root_id, &ignored, fs, txn_id, pool)); + return svn_fs_x__dag_get_node(node_p, fs, root_id, pool); +} + + +svn_error_t * +svn_fs_x__dag_txn_base_root(dag_node_t **node_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + const svn_fs_id_t *base_root_id, *ignored; + + SVN_ERR(svn_fs_x__get_txn_ids(&ignored, &base_root_id, fs, txn_id, pool)); + return svn_fs_x__dag_get_node(node_p, fs, base_root_id, pool); +} + + +svn_error_t * +svn_fs_x__dag_clone_child(dag_node_t **child_p, + dag_node_t *parent, + const char *parent_path, + const char *name, + const svn_fs_x__id_part_t *copy_id, + const svn_fs_x__id_part_t *txn_id, + svn_boolean_t is_parent_copyroot, + apr_pool_t *pool) +{ + dag_node_t *cur_entry; /* parent's current entry named NAME */ + const svn_fs_id_t *new_node_id; /* node id we'll put into NEW_NODE */ + svn_fs_t *fs = svn_fs_x__dag_get_fs(parent); + apr_pool_t *subpool = svn_pool_create(pool); + + /* First check that the parent is mutable. */ + if (! svn_fs_x__dag_check_mutable(parent)) + return svn_error_createf + (SVN_ERR_FS_NOT_MUTABLE, NULL, + "Attempted to clone child of non-mutable node"); + + /* Make sure that NAME is a single path component. */ + if (! svn_path_is_single_path_component(name)) + return svn_error_createf + (SVN_ERR_FS_NOT_SINGLE_PATH_COMPONENT, NULL, + "Attempted to make a child clone with an illegal name '%s'", name); + + /* Find the node named NAME in PARENT's entries list if it exists. */ + SVN_ERR(svn_fs_x__dag_open(&cur_entry, parent, name, pool, subpool)); + + /* Check for mutability in the node we found. If it's mutable, we + don't need to clone it. */ + if (svn_fs_x__dag_check_mutable(cur_entry)) + { + /* This has already been cloned */ + new_node_id = cur_entry->id; + } + else + { + node_revision_t *noderev, *parent_noderev; + + /* Go get a fresh NODE-REVISION for current child node. */ + SVN_ERR(get_node_revision(&noderev, cur_entry)); + + if (is_parent_copyroot) + { + SVN_ERR(get_node_revision(&parent_noderev, parent)); + noderev->copyroot_rev = parent_noderev->copyroot_rev; + noderev->copyroot_path = apr_pstrdup(pool, + parent_noderev->copyroot_path); + } + + noderev->copyfrom_path = NULL; + noderev->copyfrom_rev = SVN_INVALID_REVNUM; + + noderev->predecessor_id = svn_fs_x__id_copy(cur_entry->id, pool); + if (noderev->predecessor_count != -1) + noderev->predecessor_count++; + noderev->created_path = svn_fspath__join(parent_path, name, pool); + + SVN_ERR(svn_fs_x__create_successor(&new_node_id, fs, cur_entry->id, + noderev, copy_id, txn_id, pool)); + + /* Replace the ID in the parent's ENTRY list with the ID which + refers to the mutable clone of this child. */ + SVN_ERR(set_entry(parent, name, new_node_id, noderev->kind, txn_id, + pool)); + } + + /* Initialize the youngster. */ + svn_pool_destroy(subpool); + return svn_fs_x__dag_get_node(child_p, fs, new_node_id, pool); +} + + + +svn_error_t * +svn_fs_x__dag_clone_root(dag_node_t **root_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + const svn_fs_id_t *base_root_id, *root_id; + + /* Get the node ID's of the root directories of the transaction and + its base revision. */ + SVN_ERR(svn_fs_x__get_txn_ids(&root_id, &base_root_id, fs, txn_id, pool)); + + /* Oh, give me a clone... + (If they're the same, we haven't cloned the transaction's root + directory yet.) */ + SVN_ERR_ASSERT(!svn_fs_x__id_eq(root_id, base_root_id)); + + /* + * (Sung to the tune of "Home, Home on the Range", with thanks to + * Randall Garrett and Isaac Asimov.) + */ + + /* One way or another, root_id now identifies a cloned root node. */ + return svn_fs_x__dag_get_node(root_p, fs, root_id, pool); +} + + +svn_error_t * +svn_fs_x__dag_delete(dag_node_t *parent, + const char *name, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + node_revision_t *parent_noderev; + svn_fs_t *fs = parent->fs; + svn_fs_dirent_t *dirent; + svn_fs_id_t *id; + apr_pool_t *subpool; + + /* Make sure parent is a directory. */ + if (parent->kind != svn_node_dir) + return svn_error_createf + (SVN_ERR_FS_NOT_DIRECTORY, NULL, + "Attempted to delete entry '%s' from *non*-directory node", name); + + /* Make sure parent is mutable. */ + if (! svn_fs_x__dag_check_mutable(parent)) + return svn_error_createf + (SVN_ERR_FS_NOT_MUTABLE, NULL, + "Attempted to delete entry '%s' from immutable directory node", name); + + /* Make sure that NAME is a single path component. */ + if (! svn_path_is_single_path_component(name)) + return svn_error_createf + (SVN_ERR_FS_NOT_SINGLE_PATH_COMPONENT, NULL, + "Attempted to delete a node with an illegal name '%s'", name); + + /* Get a fresh NODE-REVISION for the parent node. */ + SVN_ERR(get_node_revision(&parent_noderev, parent)); + + subpool = svn_pool_create(pool); + + /* Search this directory for a dirent with that NAME. */ + SVN_ERR(svn_fs_x__rep_contents_dir_entry(&dirent, fs, parent_noderev, + name, subpool, subpool)); + + /* If we never found ID in ENTRIES (perhaps because there are no + ENTRIES, perhaps because ID just isn't in the existing ENTRIES + ... it doesn't matter), return an error. */ + if (! dirent) + return svn_error_createf + (SVN_ERR_FS_NO_SUCH_ENTRY, NULL, + "Delete failed--directory has no entry '%s'", name); + + /* Copy the ID out of the subpool and release the rest of the + directory listing. */ + id = svn_fs_x__id_copy(dirent->id, pool); + svn_pool_destroy(subpool); + + /* If mutable, remove it and any mutable children from db. */ + SVN_ERR(svn_fs_x__dag_delete_if_mutable(parent->fs, id, pool)); + + /* Remove this entry from its parent's entries list. */ + return svn_fs_x__set_entry(parent->fs, txn_id, parent_noderev, name, + NULL, svn_node_unknown, pool); +} + + +svn_error_t * +svn_fs_x__dag_remove_node(svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool) +{ + dag_node_t *node; + + /* Fetch the node. */ + SVN_ERR(svn_fs_x__dag_get_node(&node, fs, id, pool)); + + /* If immutable, do nothing and return immediately. */ + if (! svn_fs_x__dag_check_mutable(node)) + return svn_error_createf(SVN_ERR_FS_NOT_MUTABLE, NULL, + "Attempted removal of immutable node"); + + /* Delete the node revision. */ + return svn_fs_x__delete_node_revision(fs, id, pool); +} + + +svn_error_t * +svn_fs_x__dag_delete_if_mutable(svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool) +{ + dag_node_t *node; + + /* Get the node. */ + SVN_ERR(svn_fs_x__dag_get_node(&node, fs, id, pool)); + + /* If immutable, do nothing and return immediately. */ + if (! svn_fs_x__dag_check_mutable(node)) + return SVN_NO_ERROR; + + /* Else it's mutable. Recurse on directories... */ + if (node->kind == svn_node_dir) + { + apr_hash_t *entries; + apr_hash_index_t *hi; + + /* Loop over hash entries */ + SVN_ERR(svn_fs_x__dag_dir_entries(&entries, node, pool)); + if (entries) + { + for (hi = apr_hash_first(pool, entries); + hi; + hi = apr_hash_next(hi)) + { + svn_fs_dirent_t *dirent = svn__apr_hash_index_val(hi); + + SVN_ERR(svn_fs_x__dag_delete_if_mutable(fs, dirent->id, pool)); + } + } + } + + /* ... then delete the node itself, after deleting any mutable + representations and strings it points to. */ + return svn_fs_x__dag_remove_node(fs, id, pool); +} + +svn_error_t * +svn_fs_x__dag_make_file(dag_node_t **child_p, + dag_node_t *parent, + const char *parent_path, + const char *name, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + /* Call our little helper function */ + return make_entry(child_p, parent, parent_path, name, FALSE, txn_id, pool); +} + + +svn_error_t * +svn_fs_x__dag_make_dir(dag_node_t **child_p, + dag_node_t *parent, + const char *parent_path, + const char *name, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + /* Call our little helper function */ + return make_entry(child_p, parent, parent_path, name, TRUE, txn_id, pool); +} + + +svn_error_t * +svn_fs_x__dag_get_contents(svn_stream_t **contents_p, + dag_node_t *file, + apr_pool_t *pool) +{ + node_revision_t *noderev; + svn_stream_t *contents; + + /* Make sure our node is a file. */ + if (file->kind != svn_node_file) + return svn_error_createf + (SVN_ERR_FS_NOT_FILE, NULL, + "Attempted to get textual contents of a *non*-file node"); + + /* Go get a fresh node-revision for FILE. */ + SVN_ERR(get_node_revision(&noderev, file)); + + /* Get a stream to the contents. */ + SVN_ERR(svn_fs_x__get_contents(&contents, file->fs, + noderev->data_rep, pool)); + + *contents_p = contents; + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__dag_get_file_delta_stream(svn_txdelta_stream_t **stream_p, + dag_node_t *source, + dag_node_t *target, + apr_pool_t *pool) +{ + node_revision_t *src_noderev; + node_revision_t *tgt_noderev; + + /* Make sure our nodes are files. */ + if ((source && source->kind != svn_node_file) + || target->kind != svn_node_file) + return svn_error_createf + (SVN_ERR_FS_NOT_FILE, NULL, + "Attempted to get textual contents of a *non*-file node"); + + /* Go get fresh node-revisions for the nodes. */ + if (source) + SVN_ERR(get_node_revision(&src_noderev, source)); + else + src_noderev = NULL; + SVN_ERR(get_node_revision(&tgt_noderev, target)); + + /* Get the delta stream. */ + return svn_fs_x__get_file_delta_stream(stream_p, target->fs, + src_noderev, tgt_noderev, pool); +} + + +svn_error_t * +svn_fs_x__dag_try_process_file_contents(svn_boolean_t *success, + dag_node_t *node, + svn_fs_process_contents_func_t processor, + void* baton, + apr_pool_t *pool) +{ + node_revision_t *noderev; + + /* Go get fresh node-revisions for the nodes. */ + SVN_ERR(get_node_revision(&noderev, node)); + + return svn_fs_x__try_process_file_contents(success, node->fs, + noderev, + processor, baton, pool); +} + + +svn_error_t * +svn_fs_x__dag_file_length(svn_filesize_t *length, + dag_node_t *file, + apr_pool_t *pool) +{ + node_revision_t *noderev; + + /* Make sure our node is a file. */ + if (file->kind != svn_node_file) + return svn_error_createf + (SVN_ERR_FS_NOT_FILE, NULL, + "Attempted to get length of a *non*-file node"); + + /* Go get a fresh node-revision for FILE, and . */ + SVN_ERR(get_node_revision(&noderev, file)); + + return svn_fs_x__file_length(length, noderev, pool); +} + + +svn_error_t * +svn_fs_x__dag_file_checksum(svn_checksum_t **checksum, + dag_node_t *file, + svn_checksum_kind_t kind, + apr_pool_t *pool) +{ + node_revision_t *noderev; + + if (file->kind != svn_node_file) + return svn_error_createf + (SVN_ERR_FS_NOT_FILE, NULL, + "Attempted to get checksum of a *non*-file node"); + + SVN_ERR(get_node_revision(&noderev, file)); + + return svn_fs_x__file_checksum(checksum, noderev, kind, pool); +} + + +svn_error_t * +svn_fs_x__dag_get_edit_stream(svn_stream_t **contents, + dag_node_t *file, + apr_pool_t *pool) +{ + node_revision_t *noderev; + svn_stream_t *ws; + + /* Make sure our node is a file. */ + if (file->kind != svn_node_file) + return svn_error_createf + (SVN_ERR_FS_NOT_FILE, NULL, + "Attempted to set textual contents of a *non*-file node"); + + /* Make sure our node is mutable. */ + if (! svn_fs_x__dag_check_mutable(file)) + return svn_error_createf + (SVN_ERR_FS_NOT_MUTABLE, NULL, + "Attempted to set textual contents of an immutable node"); + + /* Get the node revision. */ + SVN_ERR(get_node_revision(&noderev, file)); + + SVN_ERR(svn_fs_x__set_contents(&ws, file->fs, noderev, pool)); + + *contents = ws; + + return SVN_NO_ERROR; +} + + + +svn_error_t * +svn_fs_x__dag_finalize_edits(dag_node_t *file, + const svn_checksum_t *checksum, + apr_pool_t *pool) +{ + if (checksum) + { + svn_checksum_t *file_checksum; + + SVN_ERR(svn_fs_x__dag_file_checksum(&file_checksum, file, + checksum->kind, pool)); + if (!svn_checksum_match(checksum, file_checksum)) + return svn_checksum_mismatch_err(checksum, file_checksum, pool, + _("Checksum mismatch for '%s'"), + file->created_path); + } + + return SVN_NO_ERROR; +} + + +dag_node_t * +svn_fs_x__dag_dup(const dag_node_t *node, + apr_pool_t *pool) +{ + /* Allocate our new node. */ + dag_node_t *new_node = apr_pcalloc(pool, sizeof(*new_node)); + + new_node->fs = node->fs; + new_node->id = svn_fs_x__id_copy(node->id, pool); + new_node->kind = node->kind; + new_node->created_path = apr_pstrdup(pool, node->created_path); + + /* Only copy cached node_revision_t for immutable nodes. */ + if (node->node_revision && !svn_fs_x__dag_check_mutable(node)) + { + new_node->node_revision = copy_node_revision(node->node_revision, pool); + new_node->node_revision->id = + svn_fs_x__id_copy(node->node_revision->id, pool); + new_node->node_revision->is_fresh_txn_root = + node->node_revision->is_fresh_txn_root; + } + new_node->node_pool = pool; + + return new_node; +} + +svn_error_t * +svn_fs_x__dag_serialize(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + dag_node_t *node = in; + svn_stringbuf_t *serialized; + + /* create an serialization context and serialize the dag node as root */ + svn_temp_serializer__context_t *context = + svn_temp_serializer__init(node, + sizeof(*node), + 1024 - SVN_TEMP_SERIALIZER__OVERHEAD, + pool); + + /* for mutable nodes, we will _never_ cache the noderev */ + if (node->node_revision && !svn_fs_x__dag_check_mutable(node)) + svn_fs_x__noderev_serialize(context, &node->node_revision); + else + svn_temp_serializer__set_null(context, + (const void * const *)&node->node_revision); + + /* The deserializer will use its own pool. */ + svn_temp_serializer__set_null(context, + (const void * const *)&node->node_pool); + + /* serialize other sub-structures */ + svn_fs_x__id_serialize(context, (const svn_fs_id_t **)&node->id); + svn_fs_x__id_serialize(context, &node->fresh_root_predecessor_id); + svn_temp_serializer__add_string(context, &node->created_path); + + /* return serialized data */ + serialized = svn_temp_serializer__get(context); + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__dag_deserialize(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + dag_node_t *node = (dag_node_t *)data; + if (data_len == 0) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Empty noderev in cache")); + + /* Copy the _full_ buffer as it also contains the sub-structures. */ + node->fs = NULL; + + /* fixup all references to sub-structures */ + svn_fs_x__id_deserialize(node, &node->id); + svn_fs_x__id_deserialize(node, + (svn_fs_id_t **)&node->fresh_root_predecessor_id); + svn_fs_x__noderev_deserialize(node, &node->node_revision); + node->node_pool = pool; + + svn_temp_deserializer__resolve(node, (void**)&node->created_path); + + /* return result */ + *out = node; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__dag_open(dag_node_t **child_p, + dag_node_t *parent, + const char *name, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + const svn_fs_id_t *node_id; + + /* Ensure that NAME exists in PARENT's entry list. */ + SVN_ERR(dir_entry_id_from_node(&node_id, parent, name, + scratch_pool, scratch_pool)); + if (! node_id) + return svn_error_createf + (SVN_ERR_FS_NOT_FOUND, NULL, + "Attempted to open non-existent child node '%s'", name); + + /* Make sure that NAME is a single path component. */ + if (! svn_path_is_single_path_component(name)) + return svn_error_createf + (SVN_ERR_FS_NOT_SINGLE_PATH_COMPONENT, NULL, + "Attempted to open node with an illegal name '%s'", name); + + /* Now get the node that was requested. */ + return svn_fs_x__dag_get_node(child_p, svn_fs_x__dag_get_fs(parent), + node_id, result_pool); +} + + +svn_error_t * +svn_fs_x__dag_copy(dag_node_t *to_node, + const char *entry, + dag_node_t *from_node, + svn_boolean_t preserve_history, + svn_revnum_t from_rev, + const char *from_path, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + const svn_fs_id_t *id; + + if (preserve_history) + { + node_revision_t *from_noderev, *to_noderev; + svn_fs_x__id_part_t copy_id; + const svn_fs_id_t *src_id = svn_fs_x__dag_get_id(from_node); + svn_fs_t *fs = svn_fs_x__dag_get_fs(from_node); + + /* Make a copy of the original node revision. */ + SVN_ERR(get_node_revision(&from_noderev, from_node)); + to_noderev = copy_node_revision(from_noderev, pool); + + /* Reserve a copy ID for this new copy. */ + SVN_ERR(svn_fs_x__reserve_copy_id(©_id, fs, txn_id, pool)); + + /* Create a successor with its predecessor pointing at the copy + source. */ + to_noderev->predecessor_id = svn_fs_x__id_copy(src_id, pool); + if (to_noderev->predecessor_count != -1) + to_noderev->predecessor_count++; + to_noderev->created_path = + svn_fspath__join(svn_fs_x__dag_get_created_path(to_node), entry, + pool); + to_noderev->copyfrom_path = apr_pstrdup(pool, from_path); + to_noderev->copyfrom_rev = from_rev; + + /* Set the copyroot equal to our own id. */ + to_noderev->copyroot_path = NULL; + + SVN_ERR(svn_fs_x__create_successor(&id, fs, src_id, to_noderev, + ©_id, txn_id, pool)); + + } + else /* don't preserve history */ + { + id = svn_fs_x__dag_get_id(from_node); + } + + /* Set the entry in to_node to the new id. */ + return svn_fs_x__dag_set_entry(to_node, entry, id, from_node->kind, + txn_id, pool); +} + + + +/*** Comparison. ***/ + +svn_error_t * +svn_fs_x__dag_things_different(svn_boolean_t *props_changed, + svn_boolean_t *contents_changed, + dag_node_t *node1, + dag_node_t *node2) +{ + node_revision_t *noderev1, *noderev2; + + /* If we have no place to store our results, don't bother doing + anything. */ + if (! props_changed && ! contents_changed) + return SVN_NO_ERROR; + + /* The node revision skels for these two nodes. */ + SVN_ERR(get_node_revision(&noderev1, node1)); + SVN_ERR(get_node_revision(&noderev2, node2)); + + /* Compare property keys. */ + if (props_changed != NULL) + *props_changed = (! svn_fs_x__noderev_same_rep_key(noderev1->prop_rep, + noderev2->prop_rep)); + + /* Compare contents keys. */ + if (contents_changed != NULL) + *contents_changed = + (! svn_fs_x__noderev_same_rep_key(noderev1->data_rep, + noderev2->data_rep)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__dag_get_copyroot(svn_revnum_t *rev, + const char **path, + dag_node_t *node) +{ + node_revision_t *noderev; + + /* Go get a fresh node-revision for NODE. */ + SVN_ERR(get_node_revision(&noderev, node)); + + *rev = noderev->copyroot_rev; + *path = noderev->copyroot_path; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__dag_get_copyfrom_rev(svn_revnum_t *rev, + dag_node_t *node) +{ + node_revision_t *noderev; + + /* Go get a fresh node-revision for NODE. */ + SVN_ERR(get_node_revision(&noderev, node)); + + *rev = noderev->copyfrom_rev; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__dag_get_copyfrom_path(const char **path, + dag_node_t *node) +{ + node_revision_t *noderev; + + /* Go get a fresh node-revision for NODE. */ + SVN_ERR(get_node_revision(&noderev, node)); + + *path = noderev->copyfrom_path; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__dag_update_ancestry(dag_node_t *target, + dag_node_t *source, + apr_pool_t *pool) +{ + node_revision_t *source_noderev, *target_noderev; + + if (! svn_fs_x__dag_check_mutable(target)) + return svn_error_createf + (SVN_ERR_FS_NOT_MUTABLE, NULL, + _("Attempted to update ancestry of non-mutable node")); + + SVN_ERR(get_node_revision(&source_noderev, source)); + SVN_ERR(get_node_revision(&target_noderev, target)); + + target_noderev->predecessor_id = source->id; + target_noderev->predecessor_count = source_noderev->predecessor_count; + if (target_noderev->predecessor_count != -1) + target_noderev->predecessor_count++; + + return svn_fs_x__put_node_revision(target->fs, target->id, target_noderev, + FALSE, pool); +}
diff --git a/subversion/libsvn_fs_x/dag.h b/subversion/libsvn_fs_x/dag.h new file mode 100644 index 0000000..539a44d --- /dev/null +++ b/subversion/libsvn_fs_x/dag.h
@@ -0,0 +1,585 @@ +/* dag.h : DAG-like interface filesystem, private to libsvn_fs + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS_DAG_H +#define SVN_LIBSVN_FS_DAG_H + +#include "svn_fs.h" +#include "svn_delta.h" +#include "private/svn_cache.h" + +#include "id.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + + +/* The interface in this file provides all the essential filesystem + operations, but exposes the filesystem's DAG structure. This makes + it simpler to implement than the public interface, since a client + of this interface has to understand and cope with shared structure + directly as it appears in the database. However, it's still a + self-consistent set of invariants to maintain, making it + (hopefully) a useful interface boundary. + + In other words: + + - The dag_node_t interface exposes the internal DAG structure of + the filesystem, while the svn_fs.h interface does any cloning + necessary to make the filesystem look like a tree. + + - The dag_node_t interface exposes the existence of copy nodes, + whereas the svn_fs.h handles them transparently. + + - dag_node_t's must be explicitly cloned, whereas the svn_fs.h + operations make clones implicitly. + + - Callers of the dag_node_t interface use Berkeley DB transactions + to ensure consistency between operations, while callers of the + svn_fs.h interface use Subversion transactions. */ + + +/* Generic DAG node stuff. */ + +typedef struct dag_node_t dag_node_t; + +/* Fill *NODE with a dag_node_t representing node revision ID in FS, + allocating in POOL. */ +svn_error_t * +svn_fs_x__dag_get_node(dag_node_t **node, + svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool); + + +/* Return a new dag_node_t object referring to the same node as NODE, + allocated in POOL. If you're trying to build a structure in a + pool that wants to refer to dag nodes that may have been allocated + elsewhere, you can call this function and avoid inter-pool pointers. */ +dag_node_t * +svn_fs_x__dag_dup(const dag_node_t *node, + apr_pool_t *pool); + +/* Serialize a DAG node, except don't try to preserve the 'fs' member. + Implements svn_cache__serialize_func_t */ +svn_error_t * +svn_fs_x__dag_serialize(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/* Deserialize a DAG node, leaving the 'fs' member as NULL. + Implements svn_cache__deserialize_func_t */ +svn_error_t * +svn_fs_x__dag_deserialize(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/* Return the filesystem containing NODE. */ +svn_fs_t *svn_fs_x__dag_get_fs(dag_node_t *node); + +/* Changes the filesystem containing NODE to FS. (Used when pulling + nodes out of a shared cache, say.) */ +void svn_fs_x__dag_set_fs(dag_node_t *node, svn_fs_t *fs); + + +/* Set *REV to NODE's revision number, allocating in POOL. If NODE + has never been committed as part of a revision, set *REV to + SVN_INVALID_REVNUM. */ +svn_error_t *svn_fs_x__dag_get_revision(svn_revnum_t *rev, + dag_node_t *node, + apr_pool_t *pool); + + +/* Return the node revision ID of NODE. The value returned is shared + with NODE, and will be deallocated when NODE is. */ +const svn_fs_id_t *svn_fs_x__dag_get_id(const dag_node_t *node); + + +/* Return the created path of NODE. The value returned is shared + with NODE, and will be deallocated when NODE is. */ +const char *svn_fs_x__dag_get_created_path(dag_node_t *node); + + +/* Set *ID_P to the node revision ID of NODE's immediate predecessor, + or NULL if NODE has no predecessor. + */ +svn_error_t *svn_fs_x__dag_get_predecessor_id(const svn_fs_id_t **id_p, + dag_node_t *node); + + +/* Set *COUNT to the number of predecessors NODE has (recursively), or + -1 if not known. + */ +/* ### This function is currently only used by 'verify'. */ +svn_error_t *svn_fs_x__dag_get_predecessor_count(int *count, + dag_node_t *node); + +/* Set *COUNT to the number of node under NODE (inclusive) with + svn:mergeinfo properties. + */ +svn_error_t *svn_fs_x__dag_get_mergeinfo_count(apr_int64_t *count, + dag_node_t *node); + +/* Set *DO_THEY to a flag indicating whether or not NODE is a + directory with at least one descendant (not including itself) with + svn:mergeinfo. + */ +svn_error_t * +svn_fs_x__dag_has_descendants_with_mergeinfo(svn_boolean_t *do_they, + dag_node_t *node); + +/* Set *HAS_MERGEINFO to a flag indicating whether or not NODE itself + has svn:mergeinfo set on it. + */ +svn_error_t * +svn_fs_x__dag_has_mergeinfo(svn_boolean_t *has_mergeinfo, + dag_node_t *node); + +/* Return non-zero IFF NODE is currently mutable. */ +svn_boolean_t svn_fs_x__dag_check_mutable(const dag_node_t *node); + +/* Return the node kind of NODE. */ +svn_node_kind_t svn_fs_x__dag_node_kind(dag_node_t *node); + +/* Set *PROPLIST_P to a PROPLIST hash representing the entire property + list of NODE, allocating from POOL. The hash has const char * + names (the property names) and svn_string_t * values (the property + values). + + If properties do not exist on NODE, *PROPLIST_P will be set to + NULL. + + Use POOL for all allocations. + */ +svn_error_t *svn_fs_x__dag_get_proplist(apr_hash_t **proplist_p, + dag_node_t *node, + apr_pool_t *pool); + +/* Set the property list of NODE to PROPLIST, allocating from POOL. + The node being changed must be mutable. + + Use POOL for all allocations. + */ +svn_error_t *svn_fs_x__dag_set_proplist(dag_node_t *node, + apr_hash_t *proplist, + apr_pool_t *pool); + +/* Increment the mergeinfo_count field on NODE by INCREMENT. The node + being changed must be mutable. + + Use POOL for all allocations. + */ +svn_error_t *svn_fs_x__dag_increment_mergeinfo_count(dag_node_t *node, + apr_int64_t increment, + apr_pool_t *pool); + +/* Set the has-mergeinfo flag on NODE to HAS_MERGEINFO. The node + being changed must be mutable. + + Use POOL for all allocations. + */ +svn_error_t *svn_fs_x__dag_set_has_mergeinfo(dag_node_t *node, + svn_boolean_t has_mergeinfo, + apr_pool_t *pool); + + + +/* Revision and transaction roots. */ + + +/* Open the root of revision REV of filesystem FS, allocating from + POOL. Set *NODE_P to the new node. */ +svn_error_t *svn_fs_x__dag_revision_root(dag_node_t **node_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + + +/* Set *NODE_P to the root of transaction TXN_ID in FS, allocating + from POOL. + + Note that the root node of TXN_ID is not necessarily mutable. If + no changes have been made in the transaction, then it may share its + root directory with its base revision. To get a mutable root node + for a transaction, call svn_fs_x__dag_clone_root. */ +svn_error_t *svn_fs_x__dag_txn_root(dag_node_t **node_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + + +/* Set *NODE_P to the base root of transaction TXN_ID in FS, + allocating from POOL. Allocate the node in TRAIL->pool. */ +svn_error_t *svn_fs_x__dag_txn_base_root(dag_node_t **node_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + + +/* Clone the root directory of TXN_ID in FS, and update the + `transactions' table entry to point to it, unless this has been + done already. In either case, set *ROOT_P to a reference to the + root directory clone. Allocate *ROOT_P in POOL. */ +svn_error_t *svn_fs_x__dag_clone_root(dag_node_t **root_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + + + +/* Directories. */ + + +/* Open the node named NAME in the directory PARENT. Set *CHILD_P to + the new node, allocated in RESULT_POOL. NAME must be a single path + component; it cannot be a slash-separated directory path. + */ +svn_error_t * +svn_fs_x__dag_open(dag_node_t **child_p, + dag_node_t *parent, + const char *name, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + + +/* Set *ENTRIES_P to a hash table of NODE's entries. The keys of the + table are entry names, and the values are svn_fs_dirent_t's. The + returned table (and its keys and values) is allocated in POOL, + which is also used for temporary allocations. */ +svn_error_t *svn_fs_x__dag_dir_entries(apr_hash_t **entries_p, + dag_node_t *node, + apr_pool_t *pool); + +/* Fetches the NODE's entries and returns a copy of the entry selected + by the key value given in NAME and set *DIRENT to a copy of that + entry. If such entry was found, the copy will be allocated in + RESULT_POOL. Temporary data will be used in SCRATCH_POOL. + Otherwise, the *DIRENT will be set to NULL. + */ +/* ### This function is currently only called from dag.c. */ +svn_error_t * svn_fs_x__dag_dir_entry(svn_fs_dirent_t **dirent, + dag_node_t *node, + const char* name, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + +/* Set ENTRY_NAME in NODE to point to ID (with kind KIND), allocating + from POOL. NODE must be a mutable directory. ID can refer to a + mutable or immutable node. If ENTRY_NAME does not exist, it will + be created. TXN_ID is the Subversion transaction under which this + occurs. + + Use POOL for all allocations, including to cache the node_revision in + NODE. + */ +svn_error_t *svn_fs_x__dag_set_entry(dag_node_t *node, + const char *entry_name, + const svn_fs_id_t *id, + svn_node_kind_t kind, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + + +/* Make a new mutable clone of the node named NAME in PARENT, and + adjust PARENT's directory entry to point to it, unless NAME in + PARENT already refers to a mutable node. In either case, set + *CHILD_P to a reference to the new node, allocated in POOL. PARENT + must be mutable. NAME must be a single path component; it cannot + be a slash-separated directory path. PARENT_PATH must be the + canonicalized absolute path of the parent directory. + + COPY_ID, if non-NULL, is a key into the `copies' table, and + indicates that this new node is being created as the result of a + copy operation, and specifically which operation that was. + + PATH is the canonicalized absolute path at which this node is being + created. + + TXN_ID is the Subversion transaction under which this occurs. + + Use POOL for all allocations. + */ +svn_error_t *svn_fs_x__dag_clone_child(dag_node_t **child_p, + dag_node_t *parent, + const char *parent_path, + const char *name, + const svn_fs_x__id_part_t *copy_id, + const svn_fs_x__id_part_t *txn_id, + svn_boolean_t is_parent_copyroot, + apr_pool_t *pool); + + +/* Delete the directory entry named NAME from PARENT, allocating from + POOL. PARENT must be mutable. NAME must be a single path + component; it cannot be a slash-separated directory path. If the + node being deleted is a mutable directory, remove all mutable nodes + reachable from it. TXN_ID is the Subversion transaction under + which this occurs. + + If return SVN_ERR_FS_NO_SUCH_ENTRY, then there is no entry NAME in + PARENT. + + Use POOL for all allocations. + */ +svn_error_t *svn_fs_x__dag_delete(dag_node_t *parent, + const char *name, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + + +/* Delete the node revision assigned to node ID from FS's `nodes' + table, allocating from POOL. Also delete any mutable + representations and strings associated with that node revision. ID + may refer to a file or directory, which must be mutable. + + NOTE: If ID represents a directory, and that directory has mutable + children, you risk orphaning those children by leaving them + dangling, disconnected from all DAG trees. It is assumed that + callers of this interface know what in the world they are doing. */ +svn_error_t *svn_fs_x__dag_remove_node(svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool); + + +/* Delete all mutable node revisions reachable from node ID, including + ID itself, from FS's `nodes' table, allocating from POOL. Also + delete any mutable representations and strings associated with that + node revision. ID may refer to a file or directory, which may be + mutable or immutable. */ +svn_error_t *svn_fs_x__dag_delete_if_mutable(svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool); + + +/* Create a new mutable directory named NAME in PARENT. Set *CHILD_P + to a reference to the new node, allocated in POOL. The new + directory has no contents, and no properties. PARENT must be + mutable. NAME must be a single path component; it cannot be a + slash-separated directory path. PARENT_PATH must be the + canonicalized absolute path of the parent directory. PARENT must + not currently have an entry named NAME. TXN_ID is the Subversion + transaction under which this occurs. + + Use POOL for all allocations. + */ +svn_error_t *svn_fs_x__dag_make_dir(dag_node_t **child_p, + dag_node_t *parent, + const char *parent_path, + const char *name, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + + + +/* Files. */ + + +/* Set *CONTENTS to a readable generic stream which yields the + contents of FILE. Allocate the stream in POOL. + + If FILE is not a file, return SVN_ERR_FS_NOT_FILE. + + Use POOL for all allocations. + */ +svn_error_t *svn_fs_x__dag_get_contents(svn_stream_t **contents, + dag_node_t *file, + apr_pool_t *pool); + +/* Attempt to fetch the contents of NODE and pass it along with the BATON + to the PROCESSOR. Set *SUCCESS only of the data could be provided + and the processor had been called. + + Use POOL for all allocations. + */ +svn_error_t * +svn_fs_x__dag_try_process_file_contents(svn_boolean_t *success, + dag_node_t *node, + svn_fs_process_contents_func_t processor, + void* baton, + apr_pool_t *pool); + + +/* Set *STREAM_P to a delta stream that will turn the contents of SOURCE into + the contents of TARGET, allocated in POOL. If SOURCE is null, the empty + string will be used. + + Use POOL for all allocations. + */ +svn_error_t * +svn_fs_x__dag_get_file_delta_stream(svn_txdelta_stream_t **stream_p, + dag_node_t *source, + dag_node_t *target, + apr_pool_t *pool); + +/* Return a generic writable stream in *CONTENTS with which to set the + contents of FILE. Allocate the stream in POOL. + + Any previous edits on the file will be deleted, and a new edit + stream will be constructed. + + Use POOL for all allocations. + */ +svn_error_t *svn_fs_x__dag_get_edit_stream(svn_stream_t **contents, + dag_node_t *file, + apr_pool_t *pool); + + +/* Signify the completion of edits to FILE made using the stream + returned by svn_fs_x__dag_get_edit_stream, allocating from POOL. + + If CHECKSUM is non-null, it must match the checksum for FILE's + contents (note: this is not recalculated, the recorded checksum is + used), else the error SVN_ERR_CHECKSUM_MISMATCH is returned. + + This operation is a no-op if no edits are present. + + Use POOL for all allocations, including to cache the node_revision in + FILE. + */ +svn_error_t *svn_fs_x__dag_finalize_edits(dag_node_t *file, + const svn_checksum_t *checksum, + apr_pool_t *pool); + + +/* Set *LENGTH to the length of the contents of FILE. + + Use POOL for all allocations. + */ +svn_error_t *svn_fs_x__dag_file_length(svn_filesize_t *length, + dag_node_t *file, + apr_pool_t *pool); + +/* Put the recorded checksum of type KIND for FILE into CHECKSUM, allocating + from POOL. + + If no stored checksum is available, do not calculate the checksum, + just put NULL into CHECKSUM. + + Use POOL for all allocations. + */ +svn_error_t * +svn_fs_x__dag_file_checksum(svn_checksum_t **checksum, + dag_node_t *file, + svn_checksum_kind_t kind, + apr_pool_t *pool); + +/* Create a new mutable file named NAME in PARENT. Set *CHILD_P to a + reference to the new node, allocated in POOL. The new file's + contents are the empty string, and it has no properties. PARENT + must be mutable. NAME must be a single path component; it cannot + be a slash-separated directory path. PARENT_PATH must be the + canonicalized absolute path of the parent directory. TXN_ID is the + Subversion transaction under which this occurs. + + Use POOL for all allocations. + */ +svn_error_t *svn_fs_x__dag_make_file(dag_node_t **child_p, + dag_node_t *parent, + const char *parent_path, + const char *name, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + + + +/* Copies */ + +/* Make ENTRY in TO_NODE be a copy of FROM_NODE, allocating from POOL. + TO_NODE must be mutable. TXN_ID is the Subversion transaction + under which this occurs. + + If PRESERVE_HISTORY is true, the new node will record that it was + copied from FROM_PATH in FROM_REV; therefore, FROM_NODE should be + the node found at FROM_PATH in FROM_REV, although this is not + checked. FROM_PATH should be canonicalized before being passed + here. + + If PRESERVE_HISTORY is false, FROM_PATH and FROM_REV are ignored. + + Use POOL for all allocations. + */ +svn_error_t *svn_fs_x__dag_copy(dag_node_t *to_node, + const char *entry, + dag_node_t *from_node, + svn_boolean_t preserve_history, + svn_revnum_t from_rev, + const char *from_path, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + + +/* Comparison */ + +/* Find out what is the same between two nodes. + + If PROPS_CHANGED is non-null, set *PROPS_CHANGED to 1 if the two + nodes have different property lists, or to 0 if same. + + If CONTENTS_CHANGED is non-null, set *CONTENTS_CHANGED to 1 if the + two nodes have different contents, or to 0 if same. For files, + file contents are compared; for directories, the entries lists are + compared. If one is a file and the other is a directory, the one's + contents will be compared to the other's entries list. (Not + terribly useful, I suppose, but that's the caller's business.) + + ### todo: This function only compares rep keys at the moment. This + may leave us with a slight chance of a false positive, though I + don't really see how that would happen in practice. Nevertheless, + it should probably be fixed. + */ +svn_error_t *svn_fs_x__dag_things_different(svn_boolean_t *props_changed, + svn_boolean_t *contents_changed, + dag_node_t *node1, + dag_node_t *node2); + + +/* Set *REV and *PATH to the copyroot revision and path of node NODE, or + to SVN_INVALID_REVNUM and NULL if no copyroot exists. + */ +svn_error_t *svn_fs_x__dag_get_copyroot(svn_revnum_t *rev, + const char **path, + dag_node_t *node); + +/* Set *REV to the copyfrom revision associated with NODE. + */ +svn_error_t *svn_fs_x__dag_get_copyfrom_rev(svn_revnum_t *rev, + dag_node_t *node); + +/* Set *PATH to the copyfrom path associated with NODE. + */ +svn_error_t *svn_fs_x__dag_get_copyfrom_path(const char **path, + dag_node_t *node); + +/* Update *TARGET so that SOURCE is it's predecessor. + */ +svn_error_t * +svn_fs_x__dag_update_ancestry(dag_node_t *target, + dag_node_t *source, + apr_pool_t *pool); +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* SVN_LIBSVN_FS_DAG_H */
diff --git a/subversion/libsvn_fs_x/fs.c b/subversion/libsvn_fs_x/fs.c new file mode 100644 index 0000000..707243c --- /dev/null +++ b/subversion/libsvn_fs_x/fs.c
@@ -0,0 +1,497 @@ +/* fs.c --- creating, opening and closing filesystems + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> + +#include <apr_general.h> +#include <apr_pools.h> +#include <apr_file_io.h> +#include <apr_thread_mutex.h> + +#include "svn_fs.h" +#include "svn_delta.h" +#include "svn_version.h" +#include "svn_pools.h" +#include "fs.h" +#include "fs_x.h" +#include "pack.h" +#include "recovery.h" +#include "hotcopy.h" +#include "verify.h" +#include "tree.h" +#include "lock.h" +#include "id.h" +#include "revprops.h" +#include "rep-cache.h" +#include "transaction.h" +#include "svn_private_config.h" +#include "private/svn_fs_util.h" + +#include "../libsvn_fs/fs-loader.h" + +/* A prefix for the pool userdata variables used to hold + per-filesystem shared data. See fs_serialized_init. */ +#define SVN_FSX_SHARED_USERDATA_PREFIX "svn-fsx-shared-" + + + +static svn_error_t * +x_serialized_init(svn_fs_t *fs, apr_pool_t *common_pool, apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + const char *key; + void *val; + fs_x_shared_data_t *ffsd; + apr_status_t status; + + /* Note that we are allocating a small amount of long-lived data for + each separate repository opened during the lifetime of the + svn_fs_initialize pool. It's unlikely that anyone will notice + the modest expenditure; the alternative is to allocate each structure + in a subpool, add a reference-count, and add a serialized deconstructor + to the FS vtable. That's more machinery than it's worth. + + Using the uuid to obtain the lock creates a corner case if a + caller uses svn_fs_set_uuid on the repository in a process where + other threads might be using the same repository through another + FS object. The only real-world consumer of svn_fs_set_uuid is + "svnadmin load", so this is a low-priority problem, and we don't + know of a better way of associating such data with the + repository. */ + + SVN_ERR_ASSERT(fs->uuid); + key = apr_pstrcat(pool, SVN_FSX_SHARED_USERDATA_PREFIX, fs->uuid, + (char *) NULL); + status = apr_pool_userdata_get(&val, key, common_pool); + if (status) + return svn_error_wrap_apr(status, _("Can't fetch FSX shared data")); + ffsd = val; + + if (!ffsd) + { + ffsd = apr_pcalloc(common_pool, sizeof(*ffsd)); + ffsd->common_pool = common_pool; + + /* POSIX fcntl locks are per-process, so we need a mutex for + intra-process synchronization when grabbing the repository write + lock. */ + SVN_ERR(svn_mutex__init(&ffsd->fs_write_lock, + SVN_FS_X__USE_LOCK_MUTEX, common_pool)); + + /* ... not to mention locking the txn-current file. */ + SVN_ERR(svn_mutex__init(&ffsd->txn_current_lock, + SVN_FS_X__USE_LOCK_MUTEX, common_pool)); + + SVN_ERR(svn_mutex__init(&ffsd->txn_list_lock, + SVN_FS_X__USE_LOCK_MUTEX, common_pool)); + + key = apr_pstrdup(common_pool, key); + status = apr_pool_userdata_set(ffsd, key, NULL, common_pool); + if (status) + return svn_error_wrap_apr(status, _("Can't store FSX shared data")); + } + + ffd->shared = ffsd; + + return SVN_NO_ERROR; +} + + + +/* This function is provided for Subversion 1.0.x compatibility. It + has no effect for fsx backed Subversion filesystems. It conforms + to the fs_library_vtable_t.bdb_set_errcall() API. */ +static svn_error_t * +x_set_errcall(svn_fs_t *fs, + void (*db_errcall_fcn)(const char *errpfx, char *msg)) +{ + + return SVN_NO_ERROR; +} + +struct x_freeze_baton_t { + svn_fs_t *fs; + svn_fs_freeze_func_t freeze_func; + void *freeze_baton; +}; + +static svn_error_t * +x_freeze_body(void *baton, + apr_pool_t *pool) +{ + struct x_freeze_baton_t *b = baton; + svn_boolean_t exists; + + SVN_ERR(svn_fs_x__exists_rep_cache(&exists, b->fs, pool)); + if (exists) + SVN_ERR(svn_fs_x__lock_rep_cache(b->fs, pool)); + + SVN_ERR(b->freeze_func(b->freeze_baton, pool)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +x_freeze(svn_fs_t *fs, + svn_fs_freeze_func_t freeze_func, + void *freeze_baton, + apr_pool_t *pool) +{ + struct x_freeze_baton_t b; + + b.fs = fs; + b.freeze_func = freeze_func; + b.freeze_baton = freeze_baton; + + SVN_ERR(svn_fs__check_fs(fs, TRUE)); + SVN_ERR(svn_fs_x__with_write_lock(fs, x_freeze_body, &b, pool)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +x_info(const void **fsx_info, + svn_fs_t *fs, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_fs_fsx_info_t *info = apr_palloc(result_pool, sizeof(*info)); + info->fs_type = SVN_FS_TYPE_FSX; + info->shard_size = ffd->max_files_per_dir; + info->min_unpacked_rev = ffd->min_unpacked_rev; + *fsx_info = info; + return SVN_NO_ERROR; +} + + + +/* The vtable associated with a specific open filesystem. */ +static fs_vtable_t fs_vtable = { + svn_fs_x__youngest_rev, + svn_fs_x__revision_prop, + svn_fs_x__revision_proplist, + svn_fs_x__change_rev_prop, + svn_fs_x__set_uuid, + svn_fs_x__revision_root, + svn_fs_x__begin_txn, + svn_fs_x__open_txn, + svn_fs_x__purge_txn, + svn_fs_x__list_transactions, + svn_fs_x__deltify, + svn_fs_x__lock, + svn_fs_x__generate_lock_token, + svn_fs_x__unlock, + svn_fs_x__get_lock, + svn_fs_x__get_locks, + svn_fs_x__info_format, + svn_fs_x__info_config_files, + x_info, + svn_fs_x__verify_root, + x_freeze, + x_set_errcall +}; + + +/* Creating a new filesystem. */ + +/* Set up vtable and fsap_data fields in FS. */ +static svn_error_t * +initialize_fs_struct(svn_fs_t *fs) +{ + fs_x_data_t *ffd = apr_pcalloc(fs->pool, sizeof(*ffd)); + fs->vtable = &fs_vtable; + fs->fsap_data = ffd; + return SVN_NO_ERROR; +} + +/* This implements the fs_library_vtable_t.create() API. Create a new + fsx-backed Subversion filesystem at path PATH and link it into + *FS. Perform temporary allocations in POOL, and fs-global allocations + in COMMON_POOL. */ +static svn_error_t * +x_create(svn_fs_t *fs, const char *path, apr_pool_t *pool, + apr_pool_t *common_pool) +{ + SVN_ERR(svn_fs__check_fs(fs, FALSE)); + + SVN_ERR(initialize_fs_struct(fs)); + + SVN_ERR(svn_fs_x__create(fs, path, pool)); + + SVN_ERR(svn_fs_x__initialize_caches(fs, pool)); + return x_serialized_init(fs, common_pool, pool); +} + + + +/* Gaining access to an existing filesystem. */ + +/* This implements the fs_library_vtable_t.open() API. Open an FSX + Subversion filesystem located at PATH, set *FS to point to the + correct vtable for the filesystem. Use POOL for any temporary + allocations, and COMMON_POOL for fs-global allocations. */ +static svn_error_t * +x_open(svn_fs_t *fs, const char *path, apr_pool_t *pool, + apr_pool_t *common_pool) +{ + SVN_ERR(initialize_fs_struct(fs)); + + SVN_ERR(svn_fs_x__open(fs, path, pool)); + + SVN_ERR(svn_fs_x__initialize_caches(fs, pool)); + return x_serialized_init(fs, common_pool, pool); +} + + + +/* This implements the fs_library_vtable_t.open_for_recovery() API. */ +static svn_error_t * +x_open_for_recovery(svn_fs_t *fs, + const char *path, + apr_pool_t *pool, apr_pool_t *common_pool) +{ + /* Recovery for FSX is currently limited to recreating the 'current' + file from the latest revision. */ + + /* The only thing we have to watch out for is that the 'current' file + might not exist. So we'll try to create it here unconditionally, + and just ignore any errors that might indicate that it's already + present. (We'll need it to exist later anyway as a source for the + new file's permissions). */ + + /* Use a partly-filled fs pointer first to create 'current'. This will fail + if 'current' already exists, but we don't care about that. */ + fs->path = apr_pstrdup(fs->pool, path); + svn_error_clear(svn_io_file_create(svn_fs_x__path_current(fs, pool), + "0 1 1\n", pool)); + + /* Now open the filesystem properly by calling the vtable method directly. */ + return x_open(fs, path, pool, common_pool); +} + + + +/* This implements the fs_library_vtable_t.upgrade_fs() API. */ +static svn_error_t * +x_upgrade(svn_fs_t *fs, + const char *path, + svn_fs_upgrade_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool, + apr_pool_t *common_pool) +{ + SVN_ERR(svn_fs__check_fs(fs, FALSE)); + SVN_ERR(initialize_fs_struct(fs)); + SVN_ERR(svn_fs_x__open(fs, path, pool)); + SVN_ERR(svn_fs_x__initialize_caches(fs, pool)); + SVN_ERR(x_serialized_init(fs, common_pool, pool)); + return svn_fs_x__upgrade(fs, notify_func, notify_baton, + cancel_func, cancel_baton, pool); +} + +static svn_error_t * +x_verify(svn_fs_t *fs, const char *path, + svn_revnum_t start, + svn_revnum_t end, + svn_fs_progress_notify_func_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool, + apr_pool_t *common_pool) +{ + SVN_ERR(svn_fs__check_fs(fs, FALSE)); + SVN_ERR(initialize_fs_struct(fs)); + SVN_ERR(svn_fs_x__open(fs, path, pool)); + SVN_ERR(svn_fs_x__initialize_caches(fs, pool)); + SVN_ERR(x_serialized_init(fs, common_pool, pool)); + return svn_fs_x__verify(fs, start, end, notify_func, notify_baton, + cancel_func, cancel_baton, pool); +} + +static svn_error_t * +x_pack(svn_fs_t *fs, + const char *path, + svn_fs_pack_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool, + apr_pool_t *common_pool) +{ + SVN_ERR(svn_fs__check_fs(fs, FALSE)); + SVN_ERR(initialize_fs_struct(fs)); + SVN_ERR(svn_fs_x__open(fs, path, pool)); + SVN_ERR(svn_fs_x__initialize_caches(fs, pool)); + SVN_ERR(x_serialized_init(fs, common_pool, pool)); + return svn_fs_x__pack(fs, notify_func, notify_baton, + cancel_func, cancel_baton, pool); +} + + + + +/* This implements the fs_library_vtable_t.hotcopy() API. Copy a + possibly live Subversion filesystem SRC_FS from SRC_PATH to a + DST_FS at DEST_PATH. If INCREMENTAL is TRUE, make an effort not to + re-copy data which already exists in DST_FS. + The CLEAN_LOGS argument is ignored and included for Subversion + 1.0.x compatibility. Perform all temporary allocations in POOL. */ +static svn_error_t * +x_hotcopy(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + const char *src_path, + const char *dst_path, + svn_boolean_t clean_logs, + svn_boolean_t incremental, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + SVN_ERR(svn_fs__check_fs(src_fs, FALSE)); + SVN_ERR(initialize_fs_struct(src_fs)); + SVN_ERR(svn_fs_x__open(src_fs, src_path, pool)); + SVN_ERR(svn_fs_x__initialize_caches(src_fs, pool)); + SVN_ERR(x_serialized_init(src_fs, pool, pool)); + + SVN_ERR(svn_fs__check_fs(dst_fs, FALSE)); + SVN_ERR(initialize_fs_struct(dst_fs)); + /* In INCREMENTAL mode, svn_fs_x__hotcopy() will open DST_FS. + Otherwise, it's not an FS yet --- possibly just an empty dir --- so + can't be opened. + */ + return svn_fs_x__hotcopy(src_fs, dst_fs, src_path, dst_path, + incremental, cancel_func, cancel_baton, pool); +} + + + +/* This function is included for Subversion 1.0.x compatibility. It + has no effect for fsx backed Subversion filesystems. It conforms + to the fs_library_vtable_t.bdb_logfiles() API. */ +static svn_error_t * +x_logfiles(apr_array_header_t **logfiles, + const char *path, + svn_boolean_t only_unused, + apr_pool_t *pool) +{ + /* A no-op for FSX. */ + *logfiles = apr_array_make(pool, 0, sizeof(const char *)); + + return SVN_NO_ERROR; +} + + + + + +/* Delete the filesystem located at path PATH. Perform any temporary + allocations in POOL. */ +static svn_error_t * +x_delete_fs(const char *path, + apr_pool_t *pool) +{ + /* Remove everything. */ + return svn_io_remove_dir2(path, FALSE, NULL, NULL, pool); +} + +static const svn_version_t * +x_version(void) +{ + SVN_VERSION_BODY; +} + +static const char * +x_get_description(void) +{ + return _("Module for working with an experimental (FSX) repository."); +} + +static svn_error_t * +x_set_svn_fs_open(svn_fs_t *fs, + svn_error_t *(*svn_fs_open_)(svn_fs_t **, + const char *, + apr_hash_t *, + apr_pool_t *)) +{ + fs_x_data_t *ffd = fs->fsap_data; + ffd->svn_fs_open_ = svn_fs_open_; + return SVN_NO_ERROR; +} + +static void * +x_info_dup(const void *fsx_info_void, + apr_pool_t *result_pool) +{ + /* All fields are either ints or static strings. */ + const svn_fs_fsx_info_t *fsx_info = fsx_info_void; + return apr_pmemdup(result_pool, fsx_info, sizeof(*fsx_info)); +} + + +/* Base FS library vtable, used by the FS loader library. */ + +static fs_library_vtable_t library_vtable = { + x_version, + x_create, + x_open, + x_open_for_recovery, + x_upgrade, + x_verify, + x_delete_fs, + x_hotcopy, + x_get_description, + svn_fs_x__recover, + x_pack, + x_logfiles, + NULL /* parse_id */, + x_set_svn_fs_open, + x_info_dup +}; + +svn_error_t * +svn_fs_x__init(const svn_version_t *loader_version, + fs_library_vtable_t **vtable, apr_pool_t* common_pool) +{ + static const svn_version_checklist_t checklist[] = + { + { "svn_subr", svn_subr_version }, + { "svn_delta", svn_delta_version }, + { NULL, NULL } + }; + + /* Simplified version check to make sure we can safely use the + VTABLE parameter. The FS loader does a more exhaustive check. */ + if (loader_version->major != SVN_VER_MAJOR) + return svn_error_createf(SVN_ERR_VERSION_MISMATCH, NULL, + _("Unsupported FS loader version (%d) for fsx"), + loader_version->major); + SVN_ERR(svn_ver_check_list2(x_version(), checklist, svn_ver_equal)); + + *vtable = &library_vtable; + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_fs_x/fs.h b/subversion/libsvn_fs_x/fs.h new file mode 100644 index 0000000..6e317eb --- /dev/null +++ b/subversion/libsvn_fs_x/fs.h
@@ -0,0 +1,540 @@ +/* fs.h : interface to Subversion filesystem, private to libsvn_fs + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS_X_H +#define SVN_LIBSVN_FS_X_H + +#include <apr_pools.h> +#include <apr_hash.h> +#include <apr_network_io.h> +#include <apr_md5.h> +#include <apr_sha1.h> + +#include "svn_fs.h" +#include "svn_config.h" +#include "private/svn_atomic.h" +#include "private/svn_cache.h" +#include "private/svn_fs_private.h" +#include "private/svn_sqlite.h" +#include "private/svn_mutex.h" +#include "private/svn_named_atomic.h" + +#include "id.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + + +/*** The filesystem structure. ***/ + +/* Following are defines that specify the textual elements of the + native filesystem directories and revision files. */ + +/* Names of special files in the fs_x filesystem. */ +#define PATH_FORMAT "format" /* Contains format number */ +#define PATH_UUID "uuid" /* Contains UUID */ +#define PATH_CURRENT "current" /* Youngest revision */ +#define PATH_LOCK_FILE "write-lock" /* Revision lock file */ +#define PATH_REVS_DIR "revs" /* Directory of revisions */ +#define PATH_REVPROPS_DIR "revprops" /* Directory of revprops */ +#define PATH_TXNS_DIR "transactions" /* Directory of transactions */ +#define PATH_NODE_ORIGINS_DIR "node-origins" /* Lazy node-origin cache */ +#define PATH_TXN_PROTOS_DIR "txn-protorevs" /* Directory of proto-revs */ +#define PATH_TXN_CURRENT "txn-current" /* File with next txn key */ +#define PATH_TXN_CURRENT_LOCK "txn-current-lock" /* Lock for txn-current */ +#define PATH_LOCKS_DIR "locks" /* Directory of locks */ +#define PATH_MIN_UNPACKED_REV "min-unpacked-rev" /* Oldest revision which + has not been packed. */ +#define PATH_REVPROP_GENERATION "revprop-generation" + /* Current revprop generation*/ +#define PATH_MANIFEST "manifest" /* Manifest file name */ +#define PATH_PACKED "pack" /* Packed revision data file */ +#define PATH_EXT_PACKED_SHARD ".pack" /* Extension for packed + shards */ +#define PATH_EXT_L2P_INDEX ".l2p" /* extension of the log- + to-phys index */ +#define PATH_EXT_P2L_INDEX ".p2l" /* extension of the phys- + to-log index */ +/* If you change this, look at tests/svn_test_fs.c(maybe_install_fsx_conf) */ +#define PATH_CONFIG "fsx.conf" /* Configuration */ + +/* Names of special files and file extensions for transactions */ +#define PATH_CHANGES "changes" /* Records changes made so far */ +#define PATH_TXN_PROPS "props" /* Transaction properties */ +#define PATH_NEXT_IDS "next-ids" /* Next temporary ID assignments */ +#define PATH_PREFIX_NODE "node." /* Prefix for node filename */ +#define PATH_EXT_TXN ".txn" /* Extension of txn dir */ +#define PATH_EXT_CHILDREN ".children" /* Extension for dir contents */ +#define PATH_EXT_PROPS ".props" /* Extension for node props */ +#define PATH_EXT_REV ".rev" /* Extension of protorev file */ +#define PATH_EXT_REV_LOCK ".rev-lock" /* Extension of protorev lock file */ +#define PATH_TXN_ITEM_INDEX "itemidx" /* File containing the current item + index number */ +#define PATH_INDEX "index" /* name of index files w/o ext */ + +/* Names of files in legacy FS formats */ +#define PATH_REV "rev" /* Proto rev file */ +#define PATH_REV_LOCK "rev-lock" /* Proto rev (write) lock file */ + +/* Names of sections and options in fsx.conf. */ +#define CONFIG_SECTION_CACHES "caches" +#define CONFIG_OPTION_FAIL_STOP "fail-stop" +#define CONFIG_SECTION_REP_SHARING "rep-sharing" +#define CONFIG_OPTION_ENABLE_REP_SHARING "enable-rep-sharing" +#define CONFIG_SECTION_DELTIFICATION "deltification" +#define CONFIG_OPTION_MAX_DELTIFICATION_WALK "max-deltification-walk" +#define CONFIG_OPTION_MAX_LINEAR_DELTIFICATION "max-linear-deltification" +#define CONFIG_SECTION_PACKED_REVPROPS "packed-revprops" +#define CONFIG_OPTION_REVPROP_PACK_SIZE "revprop-pack-size" +#define CONFIG_OPTION_COMPRESS_PACKED_REVPROPS "compress-packed-revprops" +#define CONFIG_SECTION_IO "io" +#define CONFIG_OPTION_BLOCK_SIZE "block-size" +#define CONFIG_OPTION_L2P_PAGE_SIZE "l2p-page-size" +#define CONFIG_OPTION_P2L_PAGE_SIZE "p2l-page-size" + +/* The format number of this filesystem. + This is independent of the repository format number, and + independent of any other FS back ends. */ +#define SVN_FS_X__FORMAT_NUMBER 1 + +/* Private FSX-specific data shared between all svn_txn_t objects that + relate to a particular transaction in a filesystem (as identified + by transaction id and filesystem UUID). Objects of this type are + allocated in their own subpool of the common pool. */ +typedef struct fs_x_shared_txn_data_t +{ + /* The next transaction in the list, or NULL if there is no following + transaction. */ + struct fs_x_shared_txn_data_t *next; + + /* ID of this transaction. */ + svn_fs_x__id_part_t txn_id; + + /* Whether the transaction's prototype revision file is locked for + writing by any thread in this process (including the current + thread; recursive locks are not permitted). This is effectively + a non-recursive mutex. */ + svn_boolean_t being_written; + + /* The pool in which this object has been allocated; a subpool of the + common pool. */ + apr_pool_t *pool; +} fs_x_shared_txn_data_t; + +/* On most operating systems apr implements file locks per process, not + per file. On Windows apr implements the locking as per file handle + locks, so we don't have to add our own mutex for just in-process + synchronization. */ +/* Compare ../libsvn_subr/named_atomic.c:USE_THREAD_MUTEX */ +#if APR_HAS_THREADS && !defined(WIN32) +#define SVN_FS_X__USE_LOCK_MUTEX 1 +#else +#define SVN_FS_X__USE_LOCK_MUTEX 0 +#endif + +/* Private FSX-specific data shared between all svn_fs_t objects that + relate to a particular filesystem, as identified by filesystem UUID. + Objects of this type are allocated in the common pool. */ +typedef struct fs_x_shared_data_t +{ + /* A list of shared transaction objects for each transaction that is + currently active, or NULL if none are. All access to this list, + including the contents of the objects stored in it, is synchronised + under TXN_LIST_LOCK. */ + fs_x_shared_txn_data_t *txns; + + /* A free transaction object, or NULL if there is no free object. + Access to this object is synchronised under TXN_LIST_LOCK. */ + fs_x_shared_txn_data_t *free_txn; + + /* A lock for intra-process synchronization when accessing the TXNS list. */ + svn_mutex__t *txn_list_lock; + + /* A lock for intra-process synchronization when grabbing the + repository write lock. */ + svn_mutex__t *fs_write_lock; + + /* A lock for intra-process synchronization when locking the + txn-current file. */ + svn_mutex__t *txn_current_lock; + + /* The common pool, under which this object is allocated, subpools + of which are used to allocate the transaction objects. */ + apr_pool_t *common_pool; +} fs_x_shared_data_t; + +/* Data structure for the 1st level DAG node cache. */ +typedef struct fs_x_dag_cache_t fs_x_dag_cache_t; + +/* Key type for all caches that use revision + offset / counter as key. + + NOTE: always initialize this using calloc() or '= {0};'! This is used + as a cahe key and the padding bytes on 32 bit archs should be zero for + cache effectiveness. */ +typedef struct pair_cache_key_t +{ + svn_revnum_t revision; + + apr_uint64_t second; +} pair_cache_key_t; + +/* Key type that identifies a represenation / rep header. */ +typedef struct representation_cache_key_t +{ + /* Revision that contains the representation */ + apr_uint32_t revision; + + /* Packed or non-packed representation? */ + svn_boolean_t is_packed; + + /* Item index of the representation */ + apr_uint64_t item_index; +} representation_cache_key_t; + +/* Key type that identifies a txdelta window. */ +typedef struct window_cache_key_t +{ + /* Revision that contains the representation */ + apr_uint32_t revision; + + /* Window number within that representation */ + int chunk_index; + + /* Item index of the representation */ + apr_uint64_t item_index; +} window_cache_key_t; + +/* Private (non-shared) FSX-specific data for each svn_fs_t object. + Any caches in here may be NULL. */ +typedef struct fs_x_data_t +{ + /* The format number of this FS. */ + int format; + + /* The maximum number of files to store per directory. */ + int max_files_per_dir; + + /* Rev / pack file read granularity. */ + apr_int64_t block_size; + + /* Capacity in entries of log-to-phys index pages */ + apr_int64_t l2p_page_size; + + /* Rev / pack file granularity covered by phys-to-log index pages */ + apr_int64_t p2l_page_size; + + /* The revision that was youngest, last time we checked. */ + svn_revnum_t youngest_rev_cache; + + /* The fsx.conf file, parsed. Allocated in FS->pool. */ + svn_config_t *config; + + /* Caches of immutable data. (Note that if these are created with + svn_cache__create_memcache, the data can be shared between + multiple svn_fs_t's for the same filesystem.) */ + + /* A cache of revision root IDs, mapping from (svn_revnum_t *) to + (svn_fs_id_t *). (Not threadsafe.) */ + svn_cache__t *rev_root_id_cache; + + /* Caches native dag_node_t* instances and acts as a 1st level cache */ + fs_x_dag_cache_t *dag_node_cache; + + /* DAG node cache for immutable nodes. Maps (revision, fspath) + to (dag_node_t *). This is the 2nd level cache for DAG nodes. */ + svn_cache__t *rev_node_cache; + + /* A cache of the contents of immutable directories; maps from + unparsed FS ID to a apr_hash_t * mapping (const char *) dirent + names to (svn_fs_dirent_t *). */ + svn_cache__t *dir_cache; + + /* Fulltext cache; currently only used with memcached. Maps from + rep key (revision/offset) to svn_string_t. */ + svn_cache__t *fulltext_cache; + + /* Access object to the atomics namespace used by revprop caching. + Will be NULL until the first access. */ + svn_atomic_namespace__t *revprop_namespace; + + /* Access object to the revprop "generation". Will be NULL until + the first access. */ + svn_named_atomic__t *revprop_generation; + + /* Access object to the revprop update timeout. Will be NULL until + the first access. */ + svn_named_atomic__t *revprop_timeout; + + /* Revision property cache. Maps from (rev,generation) to apr_hash_t. */ + svn_cache__t *revprop_cache; + + /* Node properties cache. Maps from rep key to apr_hash_t. */ + svn_cache__t *properties_cache; + + /* Pack manifest cache; a cache mapping (svn_revnum_t) shard number to + a manifest; and a manifest is a mapping from (svn_revnum_t) revision + number offset within a shard to (apr_off_t) byte-offset in the + respective pack file. */ + svn_cache__t *packed_offset_cache; + + /* Cache for txdelta_window_t objects; the key is window_cache_key_t */ + svn_cache__t *txdelta_window_cache; + + /* Cache for combined windows as svn_stringbuf_t objects; + the key is window_cache_key_t */ + svn_cache__t *combined_window_cache; + + /* Cache for svn_fs_x__rep_header_t objects; + * the key is (revision, item index) */ + svn_cache__t *node_revision_cache; + + /* Cache for noderevs_t containers; + the key is a (pack file revision, file offset) pair */ + svn_cache__t *noderevs_container_cache; + + /* Cache for change lists as APR arrays of change_t * objects; the key + is the revision */ + svn_cache__t *changes_cache; + + /* Cache for change_list_t containers; + the key is a (pack file revision, file offset) pair */ + svn_cache__t *changes_container_cache; + + /* Cache for star-delta / representation containers; + the key is a (pack file revision, file offset) pair */ + svn_cache__t *reps_container_cache; + + /* Cache for svn_fs_x__rep_header_t objects; the key is a + (revision, item index) pair */ + svn_cache__t *rep_header_cache; + + /* Cache for svn_mergeinfo_t objects; the key is a combination of + revision, inheritance flags and path. */ + svn_cache__t *mergeinfo_cache; + + /* Cache for presence of svn_mergeinfo_t on a noderev; the key is a + combination of revision, inheritance flags and path; value is "1" + if the node has mergeinfo, "0" if it doesn't. */ + svn_cache__t *mergeinfo_existence_cache; + + /* Cache for l2p_header_t objects; the key is (revision, is-packed). + Will be NULL for pre-format7 repos */ + svn_cache__t *l2p_header_cache; + + /* Cache for l2p_page_t objects; the key is svn_fs_x__page_cache_key_t. + Will be NULL for pre-format7 repos */ + svn_cache__t *l2p_page_cache; + + /* Cache for p2l_header_t objects; the key is (revision, is-packed). + Will be NULL for pre-format7 repos */ + svn_cache__t *p2l_header_cache; + + /* Cache for apr_array_header_t objects containing svn_fs_x__p2l_entry_t + elements; the key is svn_fs_x__page_cache_key_t. + Will be NULL for pre-format7 repos */ + svn_cache__t *p2l_page_cache; + + /* TRUE while the we hold a lock on the write lock file. */ + svn_boolean_t has_write_lock; + + /* If set, there are or have been more than one concurrent transaction */ + svn_boolean_t concurrent_transactions; + + /* Temporary cache for changed directories yet to be committed; maps from + unparsed FS ID to ###x. NULL outside transactions. */ + svn_cache__t *txn_dir_cache; + + /* Data shared between all svn_fs_t objects for a given filesystem. */ + fs_x_shared_data_t *shared; + + /* The sqlite database used for rep caching. */ + svn_sqlite__db_t *rep_cache_db; + + /* Thread-safe boolean */ + svn_atomic_t rep_cache_db_opened; + + /* The oldest revision not in a pack file. It also applies to revprops + * if revprop packing has been enabled by the FSX format version. */ + svn_revnum_t min_unpacked_rev; + + /* Whether rep-sharing is supported by the filesystem + * and allowed by the configuration. */ + svn_boolean_t rep_sharing_allowed; + + /* File size limit in bytes up to which multiple revprops shall be packed + * into a single file. */ + apr_int64_t revprop_pack_size; + + /* Whether packed revprop files shall be compressed. */ + svn_boolean_t compress_packed_revprops; + + /* Restart deltification histories after each multiple of this value */ + apr_int64_t max_deltification_walk; + + /* Maximum number of length of the linear part at the top of the + * deltification history after which skip deltas will be used. */ + apr_int64_t max_linear_deltification; + + /* Pointer to svn_fs_open. */ + svn_error_t *(*svn_fs_open_)(svn_fs_t **, const char *, apr_hash_t *, + apr_pool_t *); +} fs_x_data_t; + + +/*** Filesystem Transaction ***/ +typedef struct transaction_t +{ + /* property list (const char * name, svn_string_t * value). + may be NULL if there are no properties. */ + apr_hash_t *proplist; + + /* node revision id of the root node. */ + const svn_fs_id_t *root_id; + + /* node revision id of the node which is the root of the revision + upon which this txn is base. (unfinished only) */ + const svn_fs_id_t *base_id; + + /* copies list (const char * copy_ids), or NULL if there have been + no copies in this transaction. */ + apr_array_header_t *copies; + +} transaction_t; + + +/*** Representation ***/ +/* If you add fields to this, check to see if you need to change + * svn_fs_x__rep_copy. */ +typedef struct representation_t +{ + /* Checksums digests for the contents produced by this representation. + This checksum is for the contents the rep shows to consumers, + regardless of how the rep stores the data under the hood. It is + independent of the storage (fulltext, delta, whatever). + + If has_sha1 is FALSE, then for compatibility behave as though this + checksum matches the expected checksum. + + The md5 checksum is always filled, unless this is rep which was + retrieved from the rep-cache. The sha1 checksum is only computed on + a write, for use with rep-sharing. */ + svn_boolean_t has_sha1; + unsigned char sha1_digest[APR_SHA1_DIGESTSIZE]; + unsigned char md5_digest[APR_MD5_DIGESTSIZE]; + + /* Revision where this representation is located. */ + svn_revnum_t revision; + + /* Item index with the the revision. */ + apr_uint64_t item_index; + + /* The size of the representation in bytes as seen in the revision + file. */ + svn_filesize_t size; + + /* The size of the fulltext of the representation. */ + svn_filesize_t expanded_size; + + /* Is this representation a transaction? */ + svn_fs_x__id_part_t txn_id; + + /* For rep-sharing, we need a way of uniquifying node-revs which share the + same representation (see svn_fs_x__noderev_same_rep_key() ). So, we + store the original txn of the node rev (not the rep!), along with some + intra-node uniqification content. */ + struct + { + svn_fs_x__id_part_t txn_id; + apr_uint64_t number; + } uniquifier; +} representation_t; + + +/*** Node-Revision ***/ +/* If you add fields to this, check to see if you need to change + * copy_node_revision in dag.c. */ +typedef struct node_revision_t +{ + /* node kind */ + svn_node_kind_t kind; + + /* The node-id for this node-rev. */ + const svn_fs_id_t *id; + + /* predecessor node revision id, or NULL if there is no predecessor + for this node revision */ + const svn_fs_id_t *predecessor_id; + + /* If this node-rev is a copy, where was it copied from? */ + const char *copyfrom_path; + svn_revnum_t copyfrom_rev; + + /* Helper for history tracing, root of the parent tree from whence + this node-rev was copied. */ + svn_revnum_t copyroot_rev; + const char *copyroot_path; + + /* number of predecessors this node revision has (recursively), or + -1 if not known (for backward compatibility). */ + int predecessor_count; + + /* representation key for this node's properties. may be NULL if + there are no properties. */ + representation_t *prop_rep; + + /* representation for this node's data. may be NULL if there is + no data. */ + representation_t *data_rep; + + /* path at which this node first came into existence. */ + const char *created_path; + + /* is this the unmodified root of a transaction? */ + svn_boolean_t is_fresh_txn_root; + + /* Number of nodes with svn:mergeinfo properties that are + descendants of this node (including it itself) */ + apr_int64_t mergeinfo_count; + + /* Does this node itself have svn:mergeinfo? */ + svn_boolean_t has_mergeinfo; + +} node_revision_t; + + +/*** Change ***/ +typedef struct change_t +{ + /* Path of the change. */ + svn_string_t path; + + /* API compatible change description */ + svn_fs_path_change2_t info; +} change_t; + + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* SVN_LIBSVN_FS_X_H */
diff --git a/subversion/libsvn_fs_x/fs_x.c b/subversion/libsvn_fs_x/fs_x.c new file mode 100644 index 0000000..70e3684 --- /dev/null +++ b/subversion/libsvn_fs_x/fs_x.c
@@ -0,0 +1,1167 @@ +/* fs_x.c --- filesystem operations specific to fs_x + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "fs_x.h" + +#include <apr_uuid.h> + +#include "svn_hash.h" +#include "svn_props.h" +#include "svn_time.h" +#include "svn_dirent_uri.h" +#include "svn_version.h" + +#include "cached_data.h" +#include "id.h" +#include "rep-cache.h" +#include "revprops.h" +#include "transaction.h" +#include "tree.h" +#include "util.h" +#include "index.h" + +#include "private/svn_fs_util.h" +#include "private/svn_string_private.h" +#include "private/svn_subr_private.h" +#include "../libsvn_fs/fs-loader.h" + +#include "svn_private_config.h" + +/* The default maximum number of files per directory to store in the + rev and revprops directory. The number below is somewhat arbitrary, + and can be overridden by defining the macro while compiling; the + figure of 1000 is reasonable for VFAT filesystems, which are by far + the worst performers in this area. */ +#ifndef SVN_FS_X_DEFAULT_MAX_FILES_PER_DIR +#define SVN_FS_X_DEFAULT_MAX_FILES_PER_DIR 1000 +#endif + +/* Begin deltification after a node history exceeded this this limit. + Useful values are 4 to 64 with 16 being a good compromise between + computational overhead and repository size savings. + Should be a power of 2. + Values < 2 will result in standard skip-delta behavior. */ +#define SVN_FS_X_MAX_LINEAR_DELTIFICATION 16 + +/* Finding a deltification base takes operations proportional to the + number of changes being skipped. To prevent exploding runtime + during commits, limit the deltification range to this value. + Should be a power of 2 minus one. + Values < 1 disable deltification. */ +#define SVN_FS_X_MAX_DELTIFICATION_WALK 1023 + + + + +/* Check that BUF, a nul-terminated buffer of text from format file PATH, + contains only digits at OFFSET and beyond, raising an error if not. + + Uses POOL for temporary allocation. */ +static svn_error_t * +check_format_file_buffer_numeric(const char *buf, apr_off_t offset, + const char *path, apr_pool_t *pool) +{ + return svn_fs_x__check_file_buffer_numeric(buf, offset, path, "Format", + pool); +} + +static svn_error_t * +read_format(int *pformat, int *max_files_per_dir, + const char *path, apr_pool_t *pool) +{ + svn_stream_t *stream; + svn_stringbuf_t *content; + svn_stringbuf_t *buf; + svn_boolean_t eos = FALSE; + + SVN_ERR(svn_stringbuf_from_file2(&content, path, pool)); + stream = svn_stream_from_stringbuf(content, pool); + SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, pool)); + if (buf->len == 0 && eos) + { + /* Return a more useful error message. */ + return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL, + _("Can't read first line of format file '%s'"), + svn_dirent_local_style(path, pool)); + } + + /* Check that the first line contains only digits. */ + SVN_ERR(check_format_file_buffer_numeric(buf->data, 0, path, pool)); + SVN_ERR(svn_cstring_atoi(pformat, buf->data)); + + /* Read any options. */ + SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, pool)); + if (!eos && strncmp(buf->data, "layout sharded ", 15) == 0) + { + /* Check that the argument is numeric. */ + SVN_ERR(check_format_file_buffer_numeric(buf->data, 15, path, pool)); + SVN_ERR(svn_cstring_atoi(max_files_per_dir, buf->data + 15)); + } + else + return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL, + _("'%s' contains invalid filesystem format option '%s'"), + svn_dirent_local_style(path, pool), buf->data); + + return SVN_NO_ERROR; +} + +/* Write the format number and maximum number of files per directory + to a new format file in PATH, possibly expecting to overwrite a + previously existing file. + + Use POOL for temporary allocation. */ +svn_error_t * +svn_fs_x__write_format(svn_fs_t *fs, + svn_boolean_t overwrite, + apr_pool_t *pool) +{ + svn_stringbuf_t *sb; + const char *path = svn_fs_x__path_format(fs, pool); + fs_x_data_t *ffd = fs->fsap_data; + + SVN_ERR_ASSERT(1 <= ffd->format && ffd->format <= SVN_FS_X__FORMAT_NUMBER); + + sb = svn_stringbuf_createf(pool, "%d\n", ffd->format); + svn_stringbuf_appendcstr(sb, apr_psprintf(pool, "layout sharded %d\n", + ffd->max_files_per_dir)); + + /* svn_io_write_version_file() does a load of magic to allow it to + replace version files that already exist. We only need to do + that when we're allowed to overwrite an existing file. */ + if (! overwrite) + { + /* Create the file */ + SVN_ERR(svn_io_file_create(path, sb->data, pool)); + } + else + { + SVN_ERR(svn_io_write_atomic(path, sb->data, sb->len, + NULL /* copy_perms_path */, pool)); + } + + /* And set the perms to make it read only */ + return svn_io_set_file_read_only(path, FALSE, pool); +} + +/* Return the error SVN_ERR_FS_UNSUPPORTED_FORMAT if FS's format + number is not the same as a format number supported by this + Subversion. */ +static svn_error_t * +check_format(int format) +{ + /* We support all formats from 1-current simultaneously */ + if (1 <= format && format <= SVN_FS_X__FORMAT_NUMBER) + return SVN_NO_ERROR; + + return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, + _("Expected FS format between '1' and '%d'; found format '%d'"), + SVN_FS_X__FORMAT_NUMBER, format); +} + +/* Find the youngest revision in a repository at path FS_PATH and + return it in *YOUNGEST_P. Perform temporary allocations in + POOL. */ +static svn_error_t * +get_youngest(svn_revnum_t *youngest_p, + const char *fs_path, + apr_pool_t *pool) +{ + svn_stringbuf_t *buf; + SVN_ERR(svn_fs_x__read_content(&buf, + svn_dirent_join(fs_path, PATH_CURRENT, pool), + pool)); + + *youngest_p = SVN_STR_TO_REV(buf->data); + + return SVN_NO_ERROR; +} + + +/* Read the configuration information of the file system at FS_PATH + * and set the respective values in FFD. Use POOL for allocations. + */ +static svn_error_t * +read_config(fs_x_data_t *ffd, + const char *fs_path, + apr_pool_t *pool) +{ + SVN_ERR(svn_config_read3(&ffd->config, + svn_dirent_join(fs_path, PATH_CONFIG, pool), + FALSE, FALSE, FALSE, pool)); + + /* Initialize ffd->rep_sharing_allowed. */ + SVN_ERR(svn_config_get_bool(ffd->config, &ffd->rep_sharing_allowed, + CONFIG_SECTION_REP_SHARING, + CONFIG_OPTION_ENABLE_REP_SHARING, TRUE)); + + /* Initialize deltification settings in ffd. */ + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->max_deltification_walk, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_MAX_DELTIFICATION_WALK, + SVN_FS_X_MAX_DELTIFICATION_WALK)); + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->max_linear_deltification, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_MAX_LINEAR_DELTIFICATION, + SVN_FS_X_MAX_LINEAR_DELTIFICATION)); + + /* Initialize revprop packing settings in ffd. */ + SVN_ERR(svn_config_get_bool(ffd->config, &ffd->compress_packed_revprops, + CONFIG_SECTION_PACKED_REVPROPS, + CONFIG_OPTION_COMPRESS_PACKED_REVPROPS, + TRUE)); + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->revprop_pack_size, + CONFIG_SECTION_PACKED_REVPROPS, + CONFIG_OPTION_REVPROP_PACK_SIZE, + ffd->compress_packed_revprops + ? 0x100 + : 0x40)); + + ffd->revprop_pack_size *= 1024; + + /* I/O settings in ffd. */ + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->block_size, + CONFIG_SECTION_IO, + CONFIG_OPTION_BLOCK_SIZE, + 64)); + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->l2p_page_size, + CONFIG_SECTION_IO, + CONFIG_OPTION_L2P_PAGE_SIZE, + 0x2000)); + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->p2l_page_size, + CONFIG_SECTION_IO, + CONFIG_OPTION_P2L_PAGE_SIZE, + 64)); + + ffd->block_size *= 0x400; + ffd->p2l_page_size *= 0x400; + + return SVN_NO_ERROR; +} + +static svn_error_t * +write_config(svn_fs_t *fs, + apr_pool_t *pool) +{ +#define NL APR_EOL_STR + static const char * const fsx_conf_contents = +"### This file controls the configuration of the FSX filesystem." NL +"" NL +"[" SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS "]" NL +"### These options name memcached servers used to cache internal FSX" NL +"### data. See http://www.danga.com/memcached/ for more information on" NL +"### memcached. To use memcached with FSX, run one or more memcached" NL +"### servers, and specify each of them as an option like so:" NL +"# first-server = 127.0.0.1:11211" NL +"# remote-memcached = mymemcached.corp.example.com:11212" NL +"### The option name is ignored; the value is of the form HOST:PORT." NL +"### memcached servers can be shared between multiple repositories;" NL +"### however, if you do this, you *must* ensure that repositories have" NL +"### distinct UUIDs and paths, or else cached data from one repository" NL +"### might be used by another accidentally. Note also that memcached has" NL +"### no authentication for reads or writes, so you must ensure that your" NL +"### memcached servers are only accessible by trusted users." NL +"" NL +"[" CONFIG_SECTION_CACHES "]" NL +"### When a cache-related error occurs, normally Subversion ignores it" NL +"### and continues, logging an error if the server is appropriately" NL +"### configured (and ignoring it with file:// access). To make" NL +"### Subversion never ignore cache errors, uncomment this line." NL +"# " CONFIG_OPTION_FAIL_STOP " = true" NL +"" NL +"[" CONFIG_SECTION_REP_SHARING "]" NL +"### To conserve space, the filesystem can optionally avoid storing" NL +"### duplicate representations. This comes at a slight cost in" NL +"### performance, as maintaining a database of shared representations can" NL +"### increase commit times. The space savings are dependent upon the size" NL +"### of the repository, the number of objects it contains and the amount of" NL +"### duplication between them, usually a function of the branching and" NL +"### merging process." NL +"###" NL +"### The following parameter enables rep-sharing in the repository. It can" NL +"### be switched on and off at will, but for best space-saving results" NL +"### should be enabled consistently over the life of the repository." NL +"### 'svnadmin verify' will check the rep-cache regardless of this setting." NL +"### rep-sharing is enabled by default." NL +"# " CONFIG_OPTION_ENABLE_REP_SHARING " = true" NL +"" NL +"[" CONFIG_SECTION_DELTIFICATION "]" NL +"### To conserve space, the filesystem stores data as differences against" NL +"### existing representations. This comes at a slight cost in performance," NL +"### as calculating differences can increase commit times. Reading data" NL +"### will also create higher CPU load and the data will be fragmented." NL +"### Since deltification tends to save significant amounts of disk space," NL +"### the overall I/O load can actually be lower." NL +"###" NL +"### The options in this section allow for tuning the deltification" NL +"### strategy. Their effects on data size and server performance may vary" NL +"### from one repository to another." NL +"###" NL +"### During commit, the server may need to walk the whole change history of" NL +"### of a given node to find a suitable deltification base. This linear" NL +"### process can impact commit times, svnadmin load and similar operations." NL +"### This setting limits the depth of the deltification history. If the" NL +"### threshold has been reached, the node will be stored as fulltext and a" NL +"### new deltification history begins." NL +"### Note, this is unrelated to svn log." NL +"### Very large values rarely provide significant additional savings but" NL +"### can impact performance greatly - in particular if directory" NL +"### deltification has been activated. Very small values may be useful in" NL +"### repositories that are dominated by large, changing binaries." NL +"### Should be a power of two minus 1. A value of 0 will effectively" NL +"### disable deltification." NL +"### For 1.9, the default value is 1023." NL +"# " CONFIG_OPTION_MAX_DELTIFICATION_WALK " = 1023" NL +"###" NL +"### The skip-delta scheme used by FSX tends to repeatably store redundant" NL +"### delta information where a simple delta against the latest version is" NL +"### often smaller. By default, 1.9+ will therefore use skip deltas only" NL +"### after the linear chain of deltas has grown beyond the threshold" NL +"### specified by this setting." NL +"### Values up to 64 can result in some reduction in repository size for" NL +"### the cost of quickly increasing I/O and CPU costs. Similarly, smaller" NL +"### numbers can reduce those costs at the cost of more disk space. For" NL +"### rarely read repositories or those containing larger binaries, this may" NL +"### present a better trade-off." NL +"### Should be a power of two. A value of 1 or smaller will cause the" NL +"### exclusive use of skip-deltas." NL +"### For 1.8, the default value is 16." NL +"# " CONFIG_OPTION_MAX_LINEAR_DELTIFICATION " = 16" NL +"" NL +"[" CONFIG_SECTION_PACKED_REVPROPS "]" NL +"### This parameter controls the size (in kBytes) of packed revprop files." NL +"### Revprops of consecutive revisions will be concatenated into a single" NL +"### file up to but not exceeding the threshold given here. However, each" NL +"### pack file may be much smaller and revprops of a single revision may be" NL +"### much larger than the limit set here. The threshold will be applied" NL +"### before optional compression takes place." NL +"### Large values will reduce disk space usage at the expense of increased" NL +"### latency and CPU usage reading and changing individual revprops. They" NL +"### become an advantage when revprop caching has been enabled because a" NL +"### lot of data can be read in one go. Values smaller than 4 kByte will" NL +"### not improve latency any further and quickly render revprop packing" NL +"### ineffective." NL +"### revprop-pack-size is 64 kBytes by default for non-compressed revprop" NL +"### pack files and 256 kBytes when compression has been enabled." NL +"# " CONFIG_OPTION_REVPROP_PACK_SIZE " = 64" NL +"###" NL +"### To save disk space, packed revprop files may be compressed. Standard" NL +"### revprops tend to allow for very effective compression. Reading and" NL +"### even more so writing, become significantly more CPU intensive. With" NL +"### revprop caching enabled, the overhead can be offset by reduced I/O" NL +"### unless you often modify revprops after packing." NL +"### Compressing packed revprops is enabled by default." NL +"# " CONFIG_OPTION_COMPRESS_PACKED_REVPROPS " = true" NL +"" NL +"[" CONFIG_SECTION_IO "]" NL +"### Parameters in this section control the data access granularity in" NL +"### format 7 repositories and later. The defaults should translate into" NL +"### decent performance over a wide range of setups." NL +"###" NL +"### When a specific piece of information needs to be read from disk, a" NL +"### data block is being read at once and its contents are being cached." NL +"### If the repository is being stored on a RAID, the block size should" NL +"### be either 50% or 100% of RAID block size / granularity. Also, your" NL +"### file system (clusters) should be properly aligned and sized. In that" NL +"### setup, each access will hit only one disk (minimizes I/O load) but" NL +"### uses all the data provided by the disk in a single access." NL +"### For SSD-based storage systems, slightly lower values around 16 kB" NL +"### may improve latency while still maximizing throughput." NL +"### Can be changed at any time but must be a power of 2." NL +"### block-size is 64 kBytes by default." NL +"# " CONFIG_OPTION_BLOCK_SIZE " = 64" NL +"###" NL +"### The log-to-phys index maps data item numbers to offsets within the" NL +"### rev or pack file. A revision typically contains 2 .. 5 such items" NL +"### per changed path. For each revision, at least one page is being" NL +"### allocated in the l2p index with unused parts resulting in no wasted" NL +"### space." NL +"### Changing this parameter only affects larger revisions with thousands" NL +"### of changed paths. A smaller value means that more pages need to be" NL +"### allocated for such revisions, increasing the size of the page table" NL +"### meaning it takes longer to read that table (once). Access to each" NL +"### page is then faster because less data has to read. So, if you have" NL +"### several extremely large revisions (approaching 1 mio changes), think" NL +"### about increasing this setting. Reducing the value will rarely result" NL +"### in a net speedup." NL +"### This is an expert setting. Any non-zero value is possible." NL +"### l2p-page-size is 8192 entries by default." NL +"# " CONFIG_OPTION_L2P_PAGE_SIZE " = 8192" NL +"###" NL +"### The phys-to-log index maps positions within the rev or pack file to" NL +"### to data items, i.e. describes what piece of information is being" NL +"### stored at that particular offset. The index describes the rev file" NL +"### in chunks (pages) and keeps a global list of all those pages. Large" NL +"### pages mean a shorter page table but a larger per-page description of" NL +"### data items in it. The latency sweetspot depends on the change size" NL +"### distribution but is relatively wide." NL +"### If the repository contains very large files, i.e. individual changes" NL +"### of tens of MB each, increasing the page size will shorten the index" NL +"### file at the expense of a slightly increased latency in sections with" NL +"### smaller changes." NL +"### For practical reasons, this should match block-size. Differing" NL +"### values are perfectly legal but may result in some processing overhead." NL +"### Must be a power of 2." NL +"### p2l-page-size is 64 kBytes by default." NL +"# " CONFIG_OPTION_P2L_PAGE_SIZE " = 64" NL +; +#undef NL + return svn_io_file_create(svn_dirent_join(fs->path, PATH_CONFIG, pool), + fsx_conf_contents, pool); +} + +svn_error_t * +svn_fs_x__open(svn_fs_t *fs, const char *path, apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + apr_file_t *uuid_file; + int format, max_files_per_dir; + char buf[APR_UUID_FORMATTED_LENGTH + 2]; + apr_size_t limit; + + fs->path = apr_pstrdup(fs->pool, path); + + /* Read the FS format number. */ + SVN_ERR(read_format(&format, &max_files_per_dir, + svn_fs_x__path_format(fs, pool), pool)); + SVN_ERR(check_format(format)); + + /* Now we've got a format number no matter what. */ + ffd->format = format; + ffd->max_files_per_dir = max_files_per_dir; + + /* Read in and cache the repository uuid. */ + SVN_ERR(svn_io_file_open(&uuid_file, svn_fs_x__path_uuid(fs, pool), + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool)); + + limit = sizeof(buf); + SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit, pool)); + fs->uuid = apr_pstrdup(fs->pool, buf); + + SVN_ERR(svn_io_file_close(uuid_file, pool)); + + /* Read the min unpacked revision. */ + SVN_ERR(svn_fs_x__update_min_unpacked_rev(fs, pool)); + + /* Read the configuration file. */ + SVN_ERR(read_config(ffd, fs->path, pool)); + + return get_youngest(&(ffd->youngest_rev_cache), path, pool); +} + +/* Baton type bridging svn_fs_x__upgrade and upgrade_body carrying + * parameters over between them. */ +struct upgrade_baton_t +{ + svn_fs_t *fs; + svn_fs_upgrade_notify_t notify_func; + void *notify_baton; + svn_cancel_func_t cancel_func; + void *cancel_baton; +}; + +static svn_error_t * +upgrade_body(void *baton, apr_pool_t *pool) +{ + struct upgrade_baton_t *upgrade_baton = baton; + svn_fs_t *fs = upgrade_baton->fs; + int format, max_files_per_dir; + const char *format_path = svn_fs_x__path_format(fs, pool); + + /* Read the FS format number and max-files-per-dir setting. */ + SVN_ERR(read_format(&format, &max_files_per_dir, format_path, pool)); + SVN_ERR(check_format(format)); + + /* If we're already up-to-date, there's nothing else to be done here. */ + if (format == SVN_FS_X__FORMAT_NUMBER) + return SVN_NO_ERROR; + + /* Done */ + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__upgrade(svn_fs_t *fs, + svn_fs_upgrade_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + struct upgrade_baton_t baton; + baton.fs = fs; + baton.notify_func = notify_func; + baton.notify_baton = notify_baton; + baton.cancel_func = cancel_func; + baton.cancel_baton = cancel_baton; + + return svn_fs_x__with_write_lock(fs, upgrade_body, (void *)&baton, pool); +} + + +svn_error_t * +svn_fs_x__youngest_rev(svn_revnum_t *youngest_p, + svn_fs_t *fs, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + + SVN_ERR(get_youngest(youngest_p, fs->path, pool)); + ffd->youngest_rev_cache = *youngest_p; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__ensure_revision_exists(svn_revnum_t rev, + svn_fs_t *fs, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + + if (! SVN_IS_VALID_REVNUM(rev)) + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("Invalid revision number '%ld'"), rev); + + + /* Did the revision exist the last time we checked the current + file? */ + if (rev <= ffd->youngest_rev_cache) + return SVN_NO_ERROR; + + SVN_ERR(get_youngest(&(ffd->youngest_rev_cache), fs->path, pool)); + + /* Check again. */ + if (rev <= ffd->youngest_rev_cache) + return SVN_NO_ERROR; + + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("No such revision %ld"), rev); + return SVN_NO_ERROR; +} + +/* Open the correct revision file for REV. If the filesystem FS has + been packed, *FILE will be set to the packed file; otherwise, set *FILE + to the revision file for REV. Return SVN_ERR_FS_NO_SUCH_REVISION if the + file doesn't exist. + + TODO: Consider returning an indication of whether this is a packed rev + file, so the caller need not rely on is_packed_rev() which in turn + relies on the cached FFD->min_unpacked_rev value not having changed + since the rev file was opened. + + Use POOL for allocations. */ +svn_error_t * +svn_fs_x__open_pack_or_rev_file(apr_file_t **file, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + svn_error_t *err; + svn_boolean_t retry = FALSE; + + do + { + const char *path = svn_fs_x__path_rev_absolute(fs, rev, pool); + + /* open the revision file in buffered r/o mode */ + err = svn_io_file_open(file, path, + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool); + if (err && APR_STATUS_IS_ENOENT(err->apr_err)) + { + /* Could not open the file. This may happen if the + * file once existed but got packed later. */ + svn_error_clear(err); + + /* if that was our 2nd attempt, leave it at that. */ + if (retry) + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("No such revision %ld"), rev); + + /* We failed for the first time. Refresh cache & retry. */ + SVN_ERR(svn_fs_x__update_min_unpacked_rev(fs, pool)); + + retry = TRUE; + } + else + { + retry = FALSE; + } + } + while (retry); + + return svn_error_trace(err); +} + + +svn_error_t * +svn_fs_x__revision_proplist(apr_hash_t **proplist_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + SVN_ERR(svn_fs_x__get_revision_proplist(proplist_p, fs, rev, pool)); + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__file_length(svn_filesize_t *length, + node_revision_t *noderev, + apr_pool_t *pool) +{ + if (noderev->data_rep) + *length = noderev->data_rep->expanded_size; + else + *length = 0; + + return SVN_NO_ERROR; +} + +svn_boolean_t +svn_fs_x__noderev_same_rep_key(representation_t *a, + representation_t *b) +{ + if (a == b) + return TRUE; + + if (a == NULL || b == NULL) + return FALSE; + + if (a->item_index != b->item_index) + return FALSE; + + if (a->revision != b->revision) + return FALSE; + + return memcmp(&a->uniquifier, &b->uniquifier, sizeof(a->uniquifier)) == 0; +} + +svn_error_t * +svn_fs_x__file_checksum(svn_checksum_t **checksum, + node_revision_t *noderev, + svn_checksum_kind_t kind, + apr_pool_t *pool) +{ + *checksum = NULL; + + if (noderev->data_rep) + { + svn_checksum_t temp; + temp.kind = kind; + + switch(kind) + { + case svn_checksum_md5: + temp.digest = noderev->data_rep->md5_digest; + break; + + case svn_checksum_sha1: + if (! noderev->data_rep->has_sha1) + return SVN_NO_ERROR; + + temp.digest = noderev->data_rep->sha1_digest; + break; + + default: + return SVN_NO_ERROR; + } + + *checksum = svn_checksum_dup(&temp, pool); + } + + return SVN_NO_ERROR; +} + +representation_t * +svn_fs_x__rep_copy(representation_t *rep, + apr_pool_t *pool) +{ + representation_t *rep_new; + + if (rep == NULL) + return NULL; + + rep_new = apr_palloc(pool, sizeof(*rep_new)); + + memcpy(rep_new, rep, sizeof(*rep_new)); + + return rep_new; +} + + +/* Write out the zeroth revision for filesystem FS. */ +static svn_error_t * +write_revision_zero(svn_fs_t *fs) +{ + const char *path_revision_zero = svn_fs_x__path_rev(fs, 0, fs->pool); + apr_hash_t *proplist; + svn_string_t date; + const char *path; + + /* Write out a rev file for revision 0. */ + SVN_ERR(svn_io_file_create_binary + (path_revision_zero, +/* "DELTA\nSVN\4END\nENDREP\n"*/ + "DELTA\nSVN\1" /* txdelta v1 */ + "\0\0\4\2\5" /* sview offset, sview len, + tview len, instr len, newlen */ + "\1\x84" /* 1 instr byte, new 4 bytes */ + "\4END\n" /* 4 new bytes, E, N, D, \n */ + "ENDREP\n" + "id: 0.0.r0/2\n" + "type: dir\n" + "count: 0\n" + "text: 0 3 16 4 " + "2d2977d1c96f487abe4a1e202dd03b4e\n" + "cpath: /\n" + "\n\n", + 0x78, fs->pool)); + + SVN_ERR(svn_io_set_file_read_only(path_revision_zero, FALSE, fs->pool)); + + path = svn_fs_x__path_l2p_index(fs, 0, fs->pool); + SVN_ERR(svn_io_file_create_binary + (path, + "\0\1\x80\x40\1\1" /* rev 0, single page */ + "\5\4" /* page size: bytes, count */ + "\0" /* 0 container offsets in list */ + "\0\x78\x1e\1", /* phys offsets + 1 */ + 13, + fs->pool)); + SVN_ERR(svn_io_set_file_read_only(path, FALSE, fs->pool)); + + path = svn_fs_x__path_p2l_index(fs, 0, fs->pool); + SVN_ERR(svn_io_file_create_binary + (path, + "\0" /* start rev */ + "\x80\x80\4\1\x11" /* 64k pages, 1 page using 17 bytes */ + "\0" /* offset entry 0 page 1 */ + "\x1d\x11\0\6" /* len, type + 16 * count, (rev, 2*item)* */ + "\x5a\x15\0\4" + "\1\x16\0\2" + "\x88\xff\3\0", /* last entry fills up 64k page */ + 23, + fs->pool)); + SVN_ERR(svn_io_set_file_read_only(path, FALSE, fs->pool)); + + /* Set a date on revision 0. */ + date.data = svn_time_to_cstring(apr_time_now(), fs->pool); + date.len = strlen(date.data); + proplist = apr_hash_make(fs->pool); + svn_hash_sets(proplist, SVN_PROP_REVISION_DATE, &date); + return svn_fs_x__set_revision_proplist(fs, 0, proplist, fs->pool); +} + +svn_error_t * +svn_fs_x__create(svn_fs_t *fs, + const char *path, + apr_pool_t *pool) +{ + int format = SVN_FS_X__FORMAT_NUMBER; + fs_x_data_t *ffd = fs->fsap_data; + + fs->path = apr_pstrdup(pool, path); + /* See if compatibility with older versions was explicitly requested. */ + if (fs->config) + { + svn_version_t *compatible_version; + const char *compatible; + compatible = svn_hash_gets(fs->config, SVN_FS_CONFIG_COMPATIBLE_VERSION); + if (compatible) + SVN_ERR(svn_version__parse_version_string(&compatible_version, + compatible, pool)); + if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_4_COMPATIBLE) + || svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_5_COMPATIBLE) + || svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_6_COMPATIBLE) + || svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_8_COMPATIBLE) + || (compatible && compatible_version->major == SVN_VER_MAJOR + && compatible_version->minor <= 8)) + return svn_error_create(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, + _("FSX is not compatible with Subversion prior to 1.9")); + } + ffd->format = format; + ffd->max_files_per_dir = SVN_FS_X_DEFAULT_MAX_FILES_PER_DIR; + + /* Create the revision data directories. */ + SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_rev_shard(fs, 0, pool), + pool)); + + /* Create the revprops directory. */ + SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_revprops_shard(fs, 0, + pool), + pool)); + + /* Create the transaction directory. */ + SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path, PATH_TXNS_DIR, + pool), + pool)); + + /* Create the protorevs directory. */ + SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path, PATH_TXN_PROTOS_DIR, + pool), + pool)); + + /* Create the 'current' file. */ + SVN_ERR(svn_io_file_create(svn_fs_x__path_current(fs, pool), "0\n", pool)); + SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_lock(fs, pool), pool)); + SVN_ERR(svn_fs_x__set_uuid(fs, NULL, pool)); + + SVN_ERR(write_revision_zero(fs)); + + SVN_ERR(write_config(fs, pool)); + + SVN_ERR(read_config(ffd, fs->path, pool)); + + /* Create the min unpacked rev file. */ + SVN_ERR(svn_io_file_create(svn_fs_x__path_min_unpacked_rev(fs, pool), + "0\n", pool)); + + /* Create the txn-current file if the repository supports + the transaction sequence file. */ + SVN_ERR(svn_io_file_create(svn_fs_x__path_txn_current(fs, pool), + "0\n", pool)); + SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_txn_current_lock(fs, pool), + pool)); + + /* This filesystem is ready. Stamp it with a format number. */ + SVN_ERR(svn_fs_x__write_format(fs, FALSE, pool)); + + ffd->youngest_rev_cache = 0; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__set_uuid(svn_fs_t *fs, + const char *uuid, + apr_pool_t *pool) +{ + char *my_uuid; + apr_size_t my_uuid_len; + const char *uuid_path = svn_fs_x__path_uuid(fs, pool); + + if (! uuid) + uuid = svn_uuid_generate(pool); + + /* Make sure we have a copy in FS->POOL, and append a newline. */ + my_uuid = apr_pstrcat(fs->pool, uuid, "\n", (char *)NULL); + my_uuid_len = strlen(my_uuid); + + /* We use the permissions of the 'current' file, because the 'uuid' + file does not exist during repository creation. */ + SVN_ERR(svn_io_write_atomic(uuid_path, my_uuid, my_uuid_len, + svn_fs_x__path_current(fs, pool) /* perms */, + pool)); + + /* Remove the newline we added, and stash the UUID. */ + my_uuid[my_uuid_len - 1] = '\0'; + fs->uuid = my_uuid; + + return SVN_NO_ERROR; +} + +/** Node origin lazy cache. */ + +/* If directory PATH does not exist, create it and give it the same + permissions as FS_path.*/ +svn_error_t * +svn_fs_x__ensure_dir_exists(const char *path, + const char *fs_path, + apr_pool_t *pool) +{ + svn_error_t *err = svn_io_dir_make(path, APR_OS_DEFAULT, pool); + if (err && APR_STATUS_IS_EEXIST(err->apr_err)) + { + svn_error_clear(err); + return SVN_NO_ERROR; + } + SVN_ERR(err); + + /* We successfully created a new directory. Dup the permissions + from FS->path. */ + return svn_io_copy_perms(fs_path, path, pool); +} + +/* Set *NODE_ORIGINS to a hash mapping 'const char *' node IDs to + 'svn_string_t *' node revision IDs. Use POOL for allocations. */ +static svn_error_t * +get_node_origins_from_file(svn_fs_t *fs, + apr_hash_t **node_origins, + const char *node_origins_file, + apr_pool_t *pool) +{ + apr_file_t *fd; + svn_error_t *err; + svn_stream_t *stream; + + *node_origins = NULL; + err = svn_io_file_open(&fd, node_origins_file, + APR_READ, APR_OS_DEFAULT, pool); + if (err && APR_STATUS_IS_ENOENT(err->apr_err)) + { + svn_error_clear(err); + return SVN_NO_ERROR; + } + SVN_ERR(err); + + stream = svn_stream_from_aprfile2(fd, FALSE, pool); + *node_origins = apr_hash_make(pool); + SVN_ERR(svn_hash_read2(*node_origins, stream, SVN_HASH_TERMINATOR, pool)); + return svn_stream_close(stream); +} + +svn_error_t * +svn_fs_x__get_node_origin(const svn_fs_id_t **origin_id, + svn_fs_t *fs, + const svn_fs_x__id_part_t *node_id, + apr_pool_t *pool) +{ + apr_hash_t *node_origins; + + *origin_id = NULL; + SVN_ERR(get_node_origins_from_file(fs, &node_origins, + svn_fs_x__path_node_origin(fs, node_id, pool), + pool)); + if (node_origins) + { + char node_id_ptr[SVN_INT64_BUFFER_SIZE]; + apr_size_t len = svn__ui64tobase36(node_id_ptr, node_id->number); + svn_string_t *origin_id_str + = apr_hash_get(node_origins, node_id_ptr, len); + + if (origin_id_str) + *origin_id = svn_fs_x__id_parse(origin_id_str->data, + origin_id_str->len, pool); + } + return SVN_NO_ERROR; +} + + +/* Helper for svn_fs_x__set_node_origin. Takes a NODE_ID/NODE_REV_ID + pair and adds it to the NODE_ORIGINS_PATH file. */ +static svn_error_t * +set_node_origins_for_file(svn_fs_t *fs, + const char *node_origins_path, + const svn_fs_x__id_part_t *node_id, + svn_string_t *node_rev_id, + apr_pool_t *pool) +{ + const char *path_tmp; + svn_stream_t *stream; + apr_hash_t *origins_hash; + svn_string_t *old_node_rev_id; + + /* the hash serialization functions require strings as keys */ + char node_id_ptr[SVN_INT64_BUFFER_SIZE]; + apr_size_t len = svn__ui64tobase36(node_id_ptr, node_id->number); + + SVN_ERR(svn_fs_x__ensure_dir_exists(svn_dirent_join(fs->path, + PATH_NODE_ORIGINS_DIR, + pool), + fs->path, pool)); + + /* Read the previously existing origins (if any), and merge our + update with it. */ + SVN_ERR(get_node_origins_from_file(fs, &origins_hash, + node_origins_path, pool)); + if (! origins_hash) + origins_hash = apr_hash_make(pool); + + old_node_rev_id = apr_hash_get(origins_hash, node_id_ptr, len); + + if (old_node_rev_id && !svn_string_compare(node_rev_id, old_node_rev_id)) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Node origin for '%s' exists with a different " + "value (%s) than what we were about to store " + "(%s)"), + node_id_ptr, old_node_rev_id->data, + node_rev_id->data); + + apr_hash_set(origins_hash, node_id_ptr, len, node_rev_id); + + /* Sure, there's a race condition here. Two processes could be + trying to add different cache elements to the same file at the + same time, and the entries added by the first one to write will + be lost. But this is just a cache of reconstructible data, so + we'll accept this problem in return for not having to deal with + locking overhead. */ + + /* Create a temporary file, write out our hash, and close the file. */ + SVN_ERR(svn_stream_open_unique(&stream, &path_tmp, + svn_dirent_dirname(node_origins_path, pool), + svn_io_file_del_none, pool, pool)); + SVN_ERR(svn_hash_write2(origins_hash, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_stream_close(stream)); + + /* Rename the temp file as the real destination */ + return svn_io_file_rename(path_tmp, node_origins_path, pool); +} + + +svn_error_t * +svn_fs_x__set_node_origin(svn_fs_t *fs, + const svn_fs_x__id_part_t *node_id, + const svn_fs_id_t *node_rev_id, + apr_pool_t *pool) +{ + svn_error_t *err; + const char *filename = svn_fs_x__path_node_origin(fs, node_id, pool); + + err = set_node_origins_for_file(fs, filename, + node_id, + svn_fs_x__id_unparse(node_rev_id, pool), + pool); + if (err && APR_STATUS_IS_EACCES(err->apr_err)) + { + /* It's just a cache; stop trying if I can't write. */ + svn_error_clear(err); + err = NULL; + } + return svn_error_trace(err); +} + + +/*** Revisions ***/ + +svn_error_t * +svn_fs_x__revision_prop(svn_string_t **value_p, + svn_fs_t *fs, + svn_revnum_t rev, + const char *propname, + apr_pool_t *pool) +{ + apr_hash_t *table; + + SVN_ERR(svn_fs__check_fs(fs, TRUE)); + SVN_ERR(svn_fs_x__revision_proplist(&table, fs, rev, pool)); + + *value_p = svn_hash_gets(table, propname); + + return SVN_NO_ERROR; +} + + +/* Baton used for change_rev_prop_body below. */ +struct change_rev_prop_baton { + svn_fs_t *fs; + svn_revnum_t rev; + const char *name; + const svn_string_t *const *old_value_p; + const svn_string_t *value; +}; + +/* The work-horse for svn_fs_x__change_rev_prop, called with the FS + write lock. This implements the svn_fs_x__with_write_lock() + 'body' callback type. BATON is a 'struct change_rev_prop_baton *'. */ +static svn_error_t * +change_rev_prop_body(void *baton, apr_pool_t *pool) +{ + struct change_rev_prop_baton *cb = baton; + apr_hash_t *table; + + SVN_ERR(svn_fs_x__revision_proplist(&table, cb->fs, cb->rev, pool)); + + if (cb->old_value_p) + { + const svn_string_t *wanted_value = *cb->old_value_p; + const svn_string_t *present_value = svn_hash_gets(table, cb->name); + if ((!wanted_value != !present_value) + || (wanted_value && present_value + && !svn_string_compare(wanted_value, present_value))) + { + /* What we expected isn't what we found. */ + return svn_error_createf(SVN_ERR_FS_PROP_BASEVALUE_MISMATCH, NULL, + _("revprop '%s' has unexpected value in " + "filesystem"), + cb->name); + } + /* Fall through. */ + } + svn_hash_sets(table, cb->name, cb->value); + + return svn_fs_x__set_revision_proplist(cb->fs, cb->rev, table, pool); +} + +svn_error_t * +svn_fs_x__change_rev_prop(svn_fs_t *fs, + svn_revnum_t rev, + const char *name, + const svn_string_t *const *old_value_p, + const svn_string_t *value, + apr_pool_t *pool) +{ + struct change_rev_prop_baton cb; + + SVN_ERR(svn_fs__check_fs(fs, TRUE)); + + cb.fs = fs; + cb.rev = rev; + cb.name = name; + cb.old_value_p = old_value_p; + cb.value = value; + + return svn_fs_x__with_write_lock(fs, change_rev_prop_body, &cb, pool); +} + + +svn_error_t * +svn_fs_x__info_format(int *fs_format, + svn_version_t **supports_version, + svn_fs_t *fs, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + *fs_format = ffd->format; + *supports_version = apr_palloc(result_pool, sizeof(svn_version_t)); + + (*supports_version)->major = SVN_VER_MAJOR; + (*supports_version)->minor = 9; + (*supports_version)->patch = 0; + (*supports_version)->tag = ""; + + switch (ffd->format) + { + case 1: + break; +#ifdef SVN_DEBUG +# if SVN_FS_X__FORMAT_NUMBER != 1 +# error "Need to add a 'case' statement here" +# endif +#endif + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__info_config_files(apr_array_header_t **files, + svn_fs_t *fs, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + *files = apr_array_make(result_pool, 1, sizeof(const char *)); + APR_ARRAY_PUSH(*files, const char *) = svn_dirent_join(fs->path, PATH_CONFIG, + result_pool); + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_fs_x/fs_x.h b/subversion/libsvn_fs_x/fs_x.h new file mode 100644 index 0000000..b9a52c6 --- /dev/null +++ b/subversion/libsvn_fs_x/fs_x.h
@@ -0,0 +1,233 @@ +/* fs_x.h : interface to the native filesystem layer + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS__FS_X_H +#define SVN_LIBSVN_FS__FS_X_H + +#include "fs.h" + +/* Open the fsx filesystem pointed to by PATH and associate it with + filesystem object FS. Use POOL for temporary allocations. + + ### Some parts of *FS must have been initialized beforehand; some parts + (including FS->path) are initialized by this function. */ +svn_error_t *svn_fs_x__open(svn_fs_t *fs, + const char *path, + apr_pool_t *pool); + +/* Upgrade the fsx filesystem FS. Indicate progress via the optional + * NOTIFY_FUNC callback using NOTIFY_BATON. The optional CANCEL_FUNC + * will periodically be called with CANCEL_BATON to allow for preemption. + * Use POOL for temporary allocations. */ +svn_error_t *svn_fs_x__upgrade(svn_fs_t *fs, + svn_fs_upgrade_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool); + +/* Set *YOUNGEST to the youngest revision in filesystem FS. Do any + temporary allocation in POOL. */ +svn_error_t *svn_fs_x__youngest_rev(svn_revnum_t *youngest, + svn_fs_t *fs, + apr_pool_t *pool); + +/* For revision REV in fileysystem FS, open the revision (or packed rev) + file and seek to the start of the revision. Return it in *FILE, and + use POOL for allocations. */ +svn_error_t * +svn_fs_x__open_pack_or_rev_file(apr_file_t **file, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +/* Return SVN_ERR_FS_NO_SUCH_REVISION if the given revision REV is newer + than the current youngest revision in FS or is simply not a valid + revision number, else return success. */ +svn_error_t * +svn_fs_x__ensure_revision_exists(svn_revnum_t rev, + svn_fs_t *fs, + apr_pool_t *pool); + +/* Return an error iff REV does not exist in FS. */ +svn_error_t * +svn_fs_x__revision_exists(svn_revnum_t rev, + svn_fs_t *fs, + apr_pool_t *pool); + +/* Set *PROPLIST to be an apr_hash_t containing the property list of + revision REV as seen in filesystem FS. Use POOL for temporary + allocations. */ +svn_error_t *svn_fs_x__revision_proplist(apr_hash_t **proplist, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +/* Set *LENGTH to the be fulltext length of the node revision + specified by NODEREV. Use POOL for temporary allocations. */ +svn_error_t *svn_fs_x__file_length(svn_filesize_t *length, + node_revision_t *noderev, + apr_pool_t *pool); + +/* Return TRUE if the representation keys in A and B both point to the + same representation, else return FALSE. */ +svn_boolean_t svn_fs_x__noderev_same_rep_key(representation_t *a, + representation_t *b); + + +/* Return a copy of the representation REP allocated from POOL. */ +representation_t *svn_fs_x__rep_copy(representation_t *rep, + apr_pool_t *pool); + + +/* Return the recorded checksum of type KIND for the text representation + of NODREV into CHECKSUM, allocating from POOL. If no stored checksum is + available, put all NULL into CHECKSUM. */ +svn_error_t *svn_fs_x__file_checksum(svn_checksum_t **checksum, + node_revision_t *noderev, + svn_checksum_kind_t kind, + apr_pool_t *pool); + +/* Create a fs_x fileysystem referenced by FS at path PATH. Get any + temporary allocations from POOL. + + ### Some parts of *FS must have been initialized beforehand; some parts + (including FS->path) are initialized by this function. */ +svn_error_t *svn_fs_x__create(svn_fs_t *fs, + const char *path, + apr_pool_t *pool); + +/* Set the uuid of repository FS to UUID, if UUID is not NULL; + otherwise, set the uuid of FS to a newly generated UUID. Perform + temporary allocations in POOL. */ +svn_error_t *svn_fs_x__set_uuid(svn_fs_t *fs, + const char *uuid, + apr_pool_t *pool); + +/* Set *PATH to the path of REV in FS, whether in a pack file or not. + Allocate *PATH in POOL. + + Note: If the caller does not have the write lock on FS, then the path is + not guaranteed to be correct or to remain correct after the function + returns, because the revision might become packed before or after this + call. If a file exists at that path, then it is correct; if not, then + the caller should call update_min_unpacked_rev() and re-try once. */ +const char * +svn_fs_x__path_rev_absolute(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +/* Return the path to the 'current' file in FS. + Perform allocation in POOL. */ +const char * +svn_fs_x__path_current(svn_fs_t *fs, apr_pool_t *pool); + +/* Read the format number and maximum number of files per directory + from PATH and return them in *PFORMAT and *MAX_FILES_PER_DIR + respectively. + + *MAX_FILES_PER_DIR is obtained from the 'layout' format option, and + will be set to zero if a linear scheme should be used. + + Use POOL for temporary allocation. */ +svn_error_t * +svn_fs_x__write_format(svn_fs_t *fs, + svn_boolean_t overwrite, + apr_pool_t *pool); + +/* Find the value of the property named PROPNAME in transaction TXN. + Return the contents in *VALUE_P. The contents will be allocated + from POOL. */ +svn_error_t *svn_fs_x__revision_prop(svn_string_t **value_p, svn_fs_t *fs, + svn_revnum_t rev, + const char *propname, + apr_pool_t *pool); + +/* Change, add, or delete a property on a revision REV in filesystem + FS. NAME gives the name of the property, and value, if non-NULL, + gives the new contents of the property. If value is NULL, then the + property will be deleted. If OLD_VALUE_P is not NULL, do nothing unless the + preexisting value is *OLD_VALUE_P. Do any temporary allocation in POOL. */ +svn_error_t *svn_fs_x__change_rev_prop(svn_fs_t *fs, svn_revnum_t rev, + const char *name, + const svn_string_t *const *old_value_p, + const svn_string_t *value, + apr_pool_t *pool); + +/* If directory PATH does not exist, create it and give it the same + permissions as FS_PATH.*/ +svn_error_t *svn_fs_x__ensure_dir_exists(const char *path, + const char *fs_path, + apr_pool_t *pool); + +/* Update the node origin index for FS, recording the mapping from + NODE_ID to NODE_REV_ID. Use POOL for any temporary allocations. + + Because this is just an "optional" cache, this function does not + return an error if the underlying storage is readonly; it still + returns an error for other error conditions. + */ +svn_error_t * +svn_fs_x__set_node_origin(svn_fs_t *fs, + const svn_fs_x__id_part_t *node_id, + const svn_fs_id_t *node_rev_id, + apr_pool_t *pool); + +/* Set *ORIGIN_ID to the node revision ID from which the history of + all nodes in FS whose "Node ID" is NODE_ID springs, as determined + by a look in the index. ORIGIN_ID needs to be parsed in an + FS-backend-specific way. Use POOL for allocations. + + If there is no entry for NODE_ID in the cache, return NULL + in *ORIGIN_ID. */ +svn_error_t * +svn_fs_x__get_node_origin(const svn_fs_id_t **origin_id, + svn_fs_t *fs, + const svn_fs_x__id_part_t *node_id, + apr_pool_t *pool); + + +/* Initialize all session-local caches in FS according to the global + cache settings. Use POOL for allocations. + + Please note that it is permissible for this function to set some + or all of these caches to NULL, regardless of any setting. */ +svn_error_t * +svn_fs_x__initialize_caches(svn_fs_t *fs, apr_pool_t *pool); + +/* Initialize all transaction-local caches in FS according to the global + cache settings and make TXN_ID part of their key space. Use POOL for + allocations. + + Please note that it is permissible for this function to set some or all + of these caches to NULL, regardless of any setting. */ +svn_error_t * +svn_fs_x__initialize_txn_caches(svn_fs_t *fs, + const char *txn_id, + apr_pool_t *pool); + +/* Resets the svn_cache__t structures local to the current transaction in FS. + Calling it more than once per txn or from outside any txn is allowed. */ +void +svn_fs_x__reset_txn_caches(svn_fs_t *fs); + +#endif
diff --git a/subversion/libsvn_fs_x/hotcopy.c b/subversion/libsvn_fs_x/hotcopy.c new file mode 100644 index 0000000..7e022c0 --- /dev/null +++ b/subversion/libsvn_fs_x/hotcopy.c
@@ -0,0 +1,979 @@ +/* hotcopys.c --- FS hotcopy functionality for FSX + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +#include "svn_pools.h" +#include "svn_path.h" +#include "svn_dirent_uri.h" + +#include "fs_x.h" +#include "hotcopy.h" +#include "util.h" +#include "revprops.h" +#include "rep-cache.h" +#include "transaction.h" +#include "recovery.h" + +#include "../libsvn_fs/fs-loader.h" + +#include "svn_private_config.h" + +/* Like svn_io_dir_file_copy(), but doesn't copy files that exist at + * the destination and do not differ in terms of kind, size, and mtime. */ +static svn_error_t * +hotcopy_io_dir_file_copy(const char *src_path, + const char *dst_path, + const char *file, + apr_pool_t *scratch_pool) +{ + const svn_io_dirent2_t *src_dirent; + const svn_io_dirent2_t *dst_dirent; + const char *src_target; + const char *dst_target; + + /* Does the destination already exist? If not, we must copy it. */ + dst_target = svn_dirent_join(dst_path, file, scratch_pool); + SVN_ERR(svn_io_stat_dirent2(&dst_dirent, dst_target, FALSE, TRUE, + scratch_pool, scratch_pool)); + if (dst_dirent->kind != svn_node_none) + { + /* If the destination's stat information indicates that the file + * is equal to the source, don't bother copying the file again. */ + src_target = svn_dirent_join(src_path, file, scratch_pool); + SVN_ERR(svn_io_stat_dirent2(&src_dirent, src_target, FALSE, FALSE, + scratch_pool, scratch_pool)); + if (src_dirent->kind == dst_dirent->kind && + src_dirent->special == dst_dirent->special && + src_dirent->filesize == dst_dirent->filesize && + src_dirent->mtime <= dst_dirent->mtime) + return SVN_NO_ERROR; + } + + return svn_error_trace(svn_io_dir_file_copy(src_path, dst_path, file, + scratch_pool)); +} + +/* Set *NAME_P to the UTF-8 representation of directory entry NAME. + * NAME is in the internal encoding used by APR; PARENT is in + * UTF-8 and in internal (not local) style. + * + * Use PARENT only for generating an error string if the conversion + * fails because NAME could not be represented in UTF-8. In that + * case, return a two-level error in which the outer error's message + * mentions PARENT, but the inner error's message does not mention + * NAME (except possibly in hex) since NAME may not be printable. + * Such a compound error at least allows the user to go looking in the + * right directory for the problem. + * + * If there is any other error, just return that error directly. + * + * If there is any error, the effect on *NAME_P is undefined. + * + * *NAME_P and NAME may refer to the same storage. + */ +static svn_error_t * +entry_name_to_utf8(const char **name_p, + const char *name, + const char *parent, + apr_pool_t *pool) +{ + svn_error_t *err = svn_path_cstring_to_utf8(name_p, name, pool); + if (err && err->apr_err == APR_EINVAL) + { + return svn_error_createf(err->apr_err, err, + _("Error converting entry " + "in directory '%s' to UTF-8"), + svn_dirent_local_style(parent, pool)); + } + return err; +} + +/* Like svn_io_copy_dir_recursively() but doesn't copy regular files that + * exist in the destination and do not differ from the source in terms of + * kind, size, and mtime. */ +static svn_error_t * +hotcopy_io_copy_dir_recursively(const char *src, + const char *dst_parent, + const char *dst_basename, + svn_boolean_t copy_perms, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + svn_node_kind_t kind; + apr_status_t status; + const char *dst_path; + apr_dir_t *this_dir; + apr_finfo_t this_entry; + apr_int32_t flags = APR_FINFO_TYPE | APR_FINFO_NAME; + + /* Make a subpool for recursion */ + apr_pool_t *subpool = svn_pool_create(pool); + + /* The 'dst_path' is simply dst_parent/dst_basename */ + dst_path = svn_dirent_join(dst_parent, dst_basename, pool); + + /* Sanity checks: SRC and DST_PARENT are directories, and + DST_BASENAME doesn't already exist in DST_PARENT. */ + SVN_ERR(svn_io_check_path(src, &kind, subpool)); + if (kind != svn_node_dir) + return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL, + _("Source '%s' is not a directory"), + svn_dirent_local_style(src, pool)); + + SVN_ERR(svn_io_check_path(dst_parent, &kind, subpool)); + if (kind != svn_node_dir) + return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL, + _("Destination '%s' is not a directory"), + svn_dirent_local_style(dst_parent, pool)); + + SVN_ERR(svn_io_check_path(dst_path, &kind, subpool)); + + /* Create the new directory. */ + /* ### TODO: copy permissions (needs apr_file_attrs_get()) */ + SVN_ERR(svn_io_make_dir_recursively(dst_path, pool)); + + /* Loop over the dirents in SRC. ('.' and '..' are auto-excluded) */ + SVN_ERR(svn_io_dir_open(&this_dir, src, subpool)); + + for (status = apr_dir_read(&this_entry, flags, this_dir); + status == APR_SUCCESS; + status = apr_dir_read(&this_entry, flags, this_dir)) + { + if ((this_entry.name[0] == '.') + && ((this_entry.name[1] == '\0') + || ((this_entry.name[1] == '.') + && (this_entry.name[2] == '\0')))) + { + continue; + } + else + { + const char *entryname_utf8; + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + SVN_ERR(entry_name_to_utf8(&entryname_utf8, this_entry.name, + src, subpool)); + if (this_entry.filetype == APR_REG) /* regular file */ + { + SVN_ERR(hotcopy_io_dir_file_copy(src, dst_path, entryname_utf8, + subpool)); + } + else if (this_entry.filetype == APR_LNK) /* symlink */ + { + const char *src_target = svn_dirent_join(src, entryname_utf8, + subpool); + const char *dst_target = svn_dirent_join(dst_path, + entryname_utf8, + subpool); + SVN_ERR(svn_io_copy_link(src_target, dst_target, + subpool)); + } + else if (this_entry.filetype == APR_DIR) /* recurse */ + { + const char *src_target; + + /* Prevent infinite recursion by filtering off our + newly created destination path. */ + if (strcmp(src, dst_parent) == 0 + && strcmp(entryname_utf8, dst_basename) == 0) + continue; + + src_target = svn_dirent_join(src, entryname_utf8, subpool); + SVN_ERR(hotcopy_io_copy_dir_recursively(src_target, + dst_path, + entryname_utf8, + copy_perms, + cancel_func, + cancel_baton, + subpool)); + } + /* ### support other APR node types someday?? */ + + } + } + + if (! (APR_STATUS_IS_ENOENT(status))) + return svn_error_wrap_apr(status, _("Can't read directory '%s'"), + svn_dirent_local_style(src, pool)); + + status = apr_dir_close(this_dir); + if (status) + return svn_error_wrap_apr(status, _("Error closing directory '%s'"), + svn_dirent_local_style(src, pool)); + + /* Free any memory used by recursion */ + svn_pool_destroy(subpool); + + return SVN_NO_ERROR; +} + +/* Copy an un-packed revision or revprop file for revision REV from SRC_SUBDIR + * to DST_SUBDIR. Assume a sharding layout based on MAX_FILES_PER_DIR. + * If INCLUDE_INDEXES is set, copy rev index files as well. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_copy_shard_file(const char *src_subdir, + const char *dst_subdir, + svn_revnum_t rev, + int max_files_per_dir, + svn_boolean_t include_indexes, + apr_pool_t *scratch_pool) +{ + const char *src_subdir_shard = src_subdir, + *dst_subdir_shard = dst_subdir; + + const char *shard = apr_psprintf(scratch_pool, "%ld", + rev / max_files_per_dir); + src_subdir_shard = svn_dirent_join(src_subdir, shard, scratch_pool); + dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); + + if (rev % max_files_per_dir == 0) + { + SVN_ERR(svn_io_make_dir_recursively(dst_subdir_shard, scratch_pool)); + SVN_ERR(svn_io_copy_perms(dst_subdir, dst_subdir_shard, + scratch_pool)); + } + + SVN_ERR(hotcopy_io_dir_file_copy(src_subdir_shard, dst_subdir_shard, + apr_psprintf(scratch_pool, "%ld", rev), + scratch_pool)); + if (include_indexes) + { + SVN_ERR(hotcopy_io_dir_file_copy(src_subdir_shard, dst_subdir_shard, + apr_psprintf(scratch_pool, "%ld.l2p", + rev), + scratch_pool)); + SVN_ERR(hotcopy_io_dir_file_copy(src_subdir_shard, dst_subdir_shard, + apr_psprintf(scratch_pool, "%ld.p2l", + rev), + scratch_pool)); + } + + return SVN_NO_ERROR; +} + + +/* Copy a packed shard containing revision REV, and which contains + * MAX_FILES_PER_DIR revisions, from SRC_FS to DST_FS. + * Update *DST_MIN_UNPACKED_REV in case the shard is new in DST_FS. + * Do not re-copy data which already exists in DST_FS. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_copy_packed_shard(svn_revnum_t *dst_min_unpacked_rev, + svn_fs_t *src_fs, + svn_fs_t *dst_fs, + svn_revnum_t rev, + int max_files_per_dir, + apr_pool_t *scratch_pool) +{ + const char *src_subdir; + const char *dst_subdir; + const char *packed_shard; + const char *src_subdir_packed_shard; + svn_revnum_t revprop_rev; + apr_pool_t *iterpool; + fs_x_data_t *src_ffd = src_fs->fsap_data; + + /* Copy the packed shard. */ + src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, scratch_pool); + dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, scratch_pool); + packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD, + rev / max_files_per_dir); + src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, + scratch_pool); + SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard, + dst_subdir, packed_shard, + TRUE /* copy_perms */, + NULL /* cancel_func */, NULL, + scratch_pool)); + + /* Copy revprops belonging to revisions in this pack. */ + src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, scratch_pool); + dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, scratch_pool); + + if (src_ffd->min_unpacked_rev < rev + max_files_per_dir) + { + /* copy unpacked revprops rev by rev */ + iterpool = svn_pool_create(scratch_pool); + for (revprop_rev = rev; + revprop_rev < rev + max_files_per_dir; + revprop_rev++) + { + svn_pool_clear(iterpool); + + SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir, + revprop_rev, max_files_per_dir, + FALSE, iterpool)); + } + svn_pool_destroy(iterpool); + } + else + { + /* revprop for revision 0 will never be packed */ + if (rev == 0) + SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir, + 0, max_files_per_dir, FALSE, + scratch_pool)); + + /* packed revprops folder */ + packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD, + rev / max_files_per_dir); + src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, + scratch_pool); + SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard, + dst_subdir, packed_shard, + TRUE /* copy_perms */, + NULL /* cancel_func */, NULL, + scratch_pool)); + } + + /* If necessary, update the min-unpacked rev file in the hotcopy. */ + if (*dst_min_unpacked_rev < rev + max_files_per_dir) + { + *dst_min_unpacked_rev = rev + max_files_per_dir; + SVN_ERR(svn_fs_x__write_revnum_file(dst_fs, + *dst_min_unpacked_rev, + scratch_pool)); + } + + return SVN_NO_ERROR; +} + +/* If NEW_YOUNGEST is younger than *DST_YOUNGEST, update the 'current' + * file in DST_FS and set *DST_YOUNGEST to NEW_YOUNGEST. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_update_current(svn_revnum_t *dst_youngest, + svn_fs_t *dst_fs, + svn_revnum_t new_youngest, + apr_pool_t *scratch_pool) +{ + if (*dst_youngest >= new_youngest) + return SVN_NO_ERROR; + + /* Update 'current'. */ + SVN_ERR(svn_fs_x__write_current(dst_fs, new_youngest, scratch_pool)); + + *dst_youngest = new_youngest; + + return SVN_NO_ERROR; +} + +/* Remove FILE in SHARD folder. Use POOL for temporary allocations. */ +static svn_error_t * +hotcopy_remove_file(const char *shard, + const char *file, + apr_pool_t *pool) +{ + const char *rev_path = svn_dirent_join(shard, file, pool); + + /* Make the rev file writable and remove it. */ + SVN_ERR(svn_io_set_file_read_write(rev_path, TRUE, pool)); + SVN_ERR(svn_io_remove_file2(rev_path, TRUE, pool)); + + return SVN_NO_ERROR; +} + + +/* Remove revisions between START_REV (inclusive) and END_REV (non-inclusive) + * from DST_FS. Assume sharding as per MAX_FILES_PER_DIR. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_remove_rev_files(svn_fs_t *dst_fs, + svn_revnum_t start_rev, + svn_revnum_t end_rev, + int max_files_per_dir, + apr_pool_t *scratch_pool) +{ + const char *dst_subdir; + const char *shard; + const char *dst_subdir_shard; + svn_revnum_t rev; + apr_pool_t *iterpool; + + SVN_ERR_ASSERT(start_rev <= end_rev); + + dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, scratch_pool); + + /* Pre-compute paths for initial shard. */ + shard = apr_psprintf(scratch_pool, "%ld", start_rev / max_files_per_dir); + dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); + + iterpool = svn_pool_create(scratch_pool); + for (rev = start_rev; rev < end_rev; rev++) + { + svn_pool_clear(iterpool); + + /* If necessary, update paths for shard. */ + if (rev != start_rev && rev % max_files_per_dir == 0) + { + shard = apr_psprintf(iterpool, "%ld", rev / max_files_per_dir); + dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); + } + + /* remove files for REV */ + SVN_ERR(hotcopy_remove_file(dst_subdir_shard, + apr_psprintf(iterpool, "%ld", rev), + iterpool)); + SVN_ERR(hotcopy_remove_file(dst_subdir_shard, + apr_psprintf(iterpool, "%ld.p2l", rev), + iterpool)); + SVN_ERR(hotcopy_remove_file(dst_subdir_shard, + apr_psprintf(iterpool, "%ld.l2p", rev), + iterpool)); + } + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Verify that DST_FS is a suitable destination for an incremental + * hotcopy from SRC_FS. */ +static svn_error_t * +hotcopy_incremental_check_preconditions(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + apr_pool_t *pool) +{ + fs_x_data_t *src_ffd = src_fs->fsap_data; + fs_x_data_t *dst_ffd = dst_fs->fsap_data; + + /* We only support incremental hotcopy between the same format. */ + if (src_ffd->format != dst_ffd->format) + return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("The FSX format (%d) of the hotcopy source does not match the " + "FSX format (%d) of the hotcopy destination; please upgrade " + "both repositories to the same format"), + src_ffd->format, dst_ffd->format); + + /* Make sure the UUID of source and destination match up. + * We don't want to copy over a different repository. */ + if (strcmp(src_fs->uuid, dst_fs->uuid) != 0) + return svn_error_create(SVN_ERR_RA_UUID_MISMATCH, NULL, + _("The UUID of the hotcopy source does " + "not match the UUID of the hotcopy " + "destination")); + + /* Also require same shard size. */ + if (src_ffd->max_files_per_dir != dst_ffd->max_files_per_dir) + return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("The sharding layout configuration " + "of the hotcopy source does not match " + "the sharding layout configuration of " + "the hotcopy destination")); + return SVN_NO_ERROR; +} + + +/* Baton for hotcopy_body(). */ +struct hotcopy_body_baton { + svn_fs_t *src_fs; + svn_fs_t *dst_fs; + svn_boolean_t incremental; + svn_cancel_func_t cancel_func; + void *cancel_baton; +} hotcopy_body_baton; + +/* Perform a hotcopy, either normal or incremental. + * + * Normal hotcopy assumes that the destination exists as an empty + * directory. It behaves like an incremental hotcopy except that + * none of the copied files already exist in the destination. + * + * An incremental hotcopy copies only changed or new files to the destination, + * and removes files from the destination no longer present in the source. + * While the incremental hotcopy is running, readers should still be able + * to access the destintation repository without error and should not see + * revisions currently in progress of being copied. Readers are able to see + * new fully copied revisions even if the entire incremental hotcopy procedure + * has not yet completed. + * + * Writers are blocked out completely during the entire incremental hotcopy + * process to ensure consistency. This function assumes that the repository + * write-lock is held. + */ +static svn_error_t * +hotcopy_body(void *baton, apr_pool_t *pool) +{ + struct hotcopy_body_baton *hbb = baton; + svn_fs_t *src_fs = hbb->src_fs; + fs_x_data_t *src_ffd = src_fs->fsap_data; + svn_fs_t *dst_fs = hbb->dst_fs; + fs_x_data_t *dst_ffd = dst_fs->fsap_data; + int max_files_per_dir = src_ffd->max_files_per_dir; + svn_boolean_t incremental = hbb->incremental; + svn_cancel_func_t cancel_func = hbb->cancel_func; + void* cancel_baton = hbb->cancel_baton; + svn_revnum_t src_youngest; + svn_revnum_t dst_youngest; + svn_revnum_t rev; + svn_revnum_t src_min_unpacked_rev; + svn_revnum_t dst_min_unpacked_rev; + const char *src_subdir; + const char *dst_subdir; + const char *revprop_src_subdir; + const char *revprop_dst_subdir; + apr_pool_t *iterpool; + svn_node_kind_t kind; + + /* Try to copy the config. + * + * ### We try copying the config file before doing anything else, + * ### because higher layers will abort the hotcopy if we throw + * ### an error from this function, and that renders the hotcopy + * ### unusable anyway. */ + svn_error_t *err; + + err = svn_io_dir_file_copy(src_fs->path, dst_fs->path, PATH_CONFIG, + pool); + if (err) + { + if (APR_STATUS_IS_ENOENT(err->apr_err)) + { + /* 1.6.0 to 1.6.11 did not copy the configuration file during + * hotcopy. So if we're hotcopying a repository which has been + * created as a hotcopy itself, it's possible that fsx.conf + * does not exist. Ask the user to re-create it. + * + * ### It would be nice to make this a non-fatal error, + * ### but this function does not get an svn_fs_t object + * ### so we have no way of just printing a warning via + * ### the fs->warning() callback. */ + + const char *msg; + const char *src_abspath; + const char *dst_abspath; + const char *config_relpath; + svn_error_t *err2; + + config_relpath = svn_dirent_join(src_fs->path, PATH_CONFIG, pool); + err2 = svn_dirent_get_absolute(&src_abspath, src_fs->path, pool); + if (err2) + return svn_error_trace(svn_error_compose_create(err, err2)); + err2 = svn_dirent_get_absolute(&dst_abspath, dst_fs->path, pool); + if (err2) + return svn_error_trace(svn_error_compose_create(err, err2)); + + /* ### hack: strip off the 'db/' directory from paths so + * ### they make sense to the user */ + src_abspath = svn_dirent_dirname(src_abspath, pool); + dst_abspath = svn_dirent_dirname(dst_abspath, pool); + + msg = apr_psprintf(pool, + _("Failed to create hotcopy at '%s'. " + "The file '%s' is missing from the source " + "repository. Please create this file, for " + "instance by running 'svnadmin upgrade %s'"), + dst_abspath, config_relpath, src_abspath); + return svn_error_quick_wrap(err, msg); + } + else + return svn_error_trace(err); + } + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Find the youngest revision in the source and destination. + * We only support hotcopies from sources with an equal or greater amount + * of revisions than the destination. + * This also catches the case where users accidentally swap the + * source and destination arguments. */ + SVN_ERR(svn_fs_x__youngest_rev(&src_youngest, src_fs, pool)); + if (incremental) + { + SVN_ERR(svn_fs_x__youngest_rev(&dst_youngest, dst_fs, pool)); + if (src_youngest < dst_youngest) + return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("The hotcopy destination already contains more revisions " + "(%lu) than the hotcopy source contains (%lu); are source " + "and destination swapped?"), + dst_youngest, src_youngest); + } + else + dst_youngest = 0; + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Copy the min unpacked rev, and read its value. */ + SVN_ERR(svn_fs_x__read_min_unpacked_rev(&src_min_unpacked_rev, src_fs, + pool)); + SVN_ERR(svn_fs_x__read_min_unpacked_rev(&dst_min_unpacked_rev, dst_fs, + pool)); + + /* We only support packs coming from the hotcopy source. + * The destination should not be packed independently from + * the source. This also catches the case where users accidentally + * swap the source and destination arguments. */ + if (src_min_unpacked_rev < dst_min_unpacked_rev) + return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("The hotcopy destination already contains " + "more packed revisions (%lu) than the " + "hotcopy source contains (%lu)"), + dst_min_unpacked_rev - 1, + src_min_unpacked_rev - 1); + + SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path, + PATH_MIN_UNPACKED_REV, pool)); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* + * Copy the necessary rev files. + */ + + src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, pool); + dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, pool); + SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool)); + + iterpool = svn_pool_create(pool); + /* First, copy packed shards. */ + for (rev = 0; rev < src_min_unpacked_rev; rev += max_files_per_dir) + { + svn_pool_clear(iterpool); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Copy the packed shard. */ + SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev, + src_fs, dst_fs, + rev, max_files_per_dir, + iterpool)); + + /* If necessary, update 'current' to the most recent packed rev, + * so readers can see new revisions which arrived in this pack. */ + SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, + rev + max_files_per_dir - 1, + iterpool)); + + /* Remove revision files which are now packed. */ + if (incremental) + SVN_ERR(hotcopy_remove_rev_files(dst_fs, rev, rev + max_files_per_dir, + max_files_per_dir, iterpool)); + + /* Now that all revisions have moved into the pack, the original + * rev dir can be removed. */ + err = svn_io_remove_dir2(svn_fs_x__path_rev_shard(dst_fs, rev, iterpool), + TRUE, cancel_func, cancel_baton, iterpool); + if (err) + { + if (APR_STATUS_IS_ENOTEMPTY(err->apr_err)) + svn_error_clear(err); + else + return svn_error_trace(err); + } + } + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Now, copy pairs of non-packed revisions and revprop files. + * If necessary, update 'current' after copying all files from a shard. */ + SVN_ERR_ASSERT(rev == src_min_unpacked_rev); + SVN_ERR_ASSERT(src_min_unpacked_rev == dst_min_unpacked_rev); + revprop_src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, pool); + revprop_dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, pool); + SVN_ERR(svn_io_make_dir_recursively(revprop_dst_subdir, pool)); + for (; rev <= src_youngest; rev++) + { + svn_pool_clear(iterpool); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Copy the rev file. */ + err = hotcopy_copy_shard_file(src_subdir, dst_subdir, + rev, max_files_per_dir, TRUE, + iterpool); + if (err) + { + if (APR_STATUS_IS_ENOENT(err->apr_err)) + { + svn_error_clear(err); + + /* The source rev file does not exist. This can happen if the + * source repository is being packed concurrently with this + * hotcopy operation. + * + * If the new revision is now packed, and the youngest revision + * we're interested in is not inside this pack, try to copy the + * pack instead. + * + * If the youngest revision ended up being packed, don't try + * to be smart and work around this. Just abort the hotcopy. */ + SVN_ERR(svn_fs_x__update_min_unpacked_rev(src_fs, pool)); + if (svn_fs_x__is_packed_rev(src_fs, rev)) + { + if (svn_fs_x__is_packed_rev(src_fs, src_youngest)) + return svn_error_createf( + SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("The assumed HEAD revision (%lu) of the " + "hotcopy source has been packed while the " + "hotcopy was in progress; please restart " + "the hotcopy operation"), + src_youngest); + + SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev, + src_fs, dst_fs, + rev, max_files_per_dir, + iterpool)); + rev = dst_min_unpacked_rev; + continue; + } + else + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("Revision %lu disappeared from the " + "hotcopy source while hotcopy was " + "in progress"), rev); + } + else + return svn_error_trace(err); + } + + /* Copy the revprop file. */ + SVN_ERR(hotcopy_copy_shard_file(revprop_src_subdir, + revprop_dst_subdir, + rev, max_files_per_dir, FALSE, + iterpool)); + + /* After completing a full shard, update 'current'. */ + if (max_files_per_dir && rev % max_files_per_dir == 0) + SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, rev, iterpool)); + } + svn_pool_destroy(iterpool); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* We assume that all revisions were copied now, i.e. we didn't exit the + * above loop early. 'rev' was last incremented during exit of the loop. */ + SVN_ERR_ASSERT(rev == src_youngest + 1); + + /* All revisions were copied. Update 'current'. */ + SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, src_youngest, pool)); + + /* Replace the locks tree. + * This is racy in case readers are currently trying to list locks in + * the destination. However, we need to get rid of stale locks. + * This is the simplest way of doing this, so we accept this small race. */ + dst_subdir = svn_dirent_join(dst_fs->path, PATH_LOCKS_DIR, pool); + SVN_ERR(svn_io_remove_dir2(dst_subdir, TRUE, cancel_func, cancel_baton, + pool)); + src_subdir = svn_dirent_join(src_fs->path, PATH_LOCKS_DIR, pool); + SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); + if (kind == svn_node_dir) + SVN_ERR(svn_io_copy_dir_recursively(src_subdir, dst_fs->path, + PATH_LOCKS_DIR, TRUE, + cancel_func, cancel_baton, pool)); + + /* Now copy the node-origins cache tree. */ + src_subdir = svn_dirent_join(src_fs->path, PATH_NODE_ORIGINS_DIR, pool); + SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); + if (kind == svn_node_dir) + SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir, dst_fs->path, + PATH_NODE_ORIGINS_DIR, TRUE, + cancel_func, cancel_baton, pool)); + + /* + * NB: Data copied below is only read by writers, not readers. + * Writers are still locked out at this point. + */ + + /* Copy the rep cache and then remove entries for revisions + * younger than the destination's youngest revision. */ + src_subdir = svn_dirent_join(src_fs->path, REP_CACHE_DB_NAME, pool); + dst_subdir = svn_dirent_join(dst_fs->path, REP_CACHE_DB_NAME, pool); + SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); + if (kind == svn_node_file) + { + SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, pool)); + SVN_ERR(svn_fs_x__del_rep_reference(dst_fs, dst_youngest, pool)); + } + + /* Copy the txn-current file. */ + SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path, + PATH_TXN_CURRENT, pool)); + + /* If a revprop generation file exists in the source filesystem, + * reset it to zero (since this is on a different path, it will not + * overlap with data already in cache). Also, clean up stale files + * used for the named atomics implementation. */ + SVN_ERR(svn_io_check_path(svn_fs_x__path_revprop_generation(src_fs, pool), + &kind, pool)); + if (kind == svn_node_file) + SVN_ERR(svn_fs_x__write_revprop_generation_file(dst_fs, 0, pool)); + + SVN_ERR(svn_fs_x__cleanup_revprop_namespace(dst_fs)); + + /* Hotcopied FS is complete. Stamp it with a format file. */ + dst_ffd->max_files_per_dir = max_files_per_dir; + SVN_ERR(svn_fs_x__write_format(dst_fs, TRUE, pool)); + + return SVN_NO_ERROR; +} + + +/* Set up shared data between SRC_FS and DST_FS. */ +static void +hotcopy_setup_shared_fs_data(svn_fs_t *src_fs, svn_fs_t *dst_fs) +{ + fs_x_data_t *src_ffd = src_fs->fsap_data; + fs_x_data_t *dst_ffd = dst_fs->fsap_data; + + /* The common pool and mutexes are shared between src and dst filesystems. + * During hotcopy we only grab the mutexes for the destination, so there + * is no risk of dead-lock. We don't write to the src filesystem. Shared + * data for the src_fs has already been initialised in fs_hotcopy(). */ + dst_ffd->shared = src_ffd->shared; +} + +/* Create an empty filesystem at DST_FS at DST_PATH with the same + * configuration as SRC_FS (uuid, format, and other parameters). + * After creation DST_FS has no revisions, not even revision zero. */ +static svn_error_t * +hotcopy_create_empty_dest(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + const char *dst_path, + apr_pool_t *pool) +{ + fs_x_data_t *src_ffd = src_fs->fsap_data; + fs_x_data_t *dst_ffd = dst_fs->fsap_data; + + dst_fs->path = apr_pstrdup(pool, dst_path); + + dst_ffd->max_files_per_dir = src_ffd->max_files_per_dir; + dst_ffd->config = src_ffd->config; + dst_ffd->format = src_ffd->format; + + /* Create the revision data directories. */ + SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_rev_shard(dst_fs, 0, + pool), + pool)); + + /* Create the revprops directory. */ + SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_revprops_shard(dst_fs, + 0, pool), + pool)); + + /* Create the transaction directory. */ + SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, PATH_TXNS_DIR, + pool), + pool)); + + /* Create the protorevs directory. */ + SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, + PATH_TXN_PROTOS_DIR, + pool), + pool)); + + /* Create the 'current' file. */ + SVN_ERR(svn_io_file_create(svn_fs_x__path_current(dst_fs, pool), "0\n", + pool)); + + /* Create lock file and UUID. */ + SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_lock(dst_fs, pool), pool)); + SVN_ERR(svn_fs_x__set_uuid(dst_fs, src_fs->uuid, pool)); + + /* Create the min unpacked rev file. */ + SVN_ERR(svn_io_file_create(svn_fs_x__path_min_unpacked_rev(dst_fs, pool), + "0\n", pool)); + + /* Create the txn-current file if the repository supports + the transaction sequence file. */ + SVN_ERR(svn_io_file_create(svn_fs_x__path_txn_current(dst_fs, pool), + "0\n", pool)); + SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_txn_current_lock(dst_fs, + pool), + pool)); + + dst_ffd->youngest_rev_cache = 0; + + hotcopy_setup_shared_fs_data(src_fs, dst_fs); + SVN_ERR(svn_fs_x__initialize_caches(dst_fs, pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__hotcopy(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + const char *src_path, + const char *dst_path, + svn_boolean_t incremental, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + struct hotcopy_body_baton hbb; + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + SVN_ERR(svn_fs_x__open(src_fs, src_path, pool)); + + if (incremental) + { + const char *dst_format_abspath; + svn_node_kind_t dst_format_kind; + + /* Check destination format to be sure we know how to incrementally + * hotcopy to the destination FS. */ + dst_format_abspath = svn_dirent_join(dst_path, PATH_FORMAT, pool); + SVN_ERR(svn_io_check_path(dst_format_abspath, &dst_format_kind, pool)); + if (dst_format_kind == svn_node_none) + { + /* Destination doesn't exist yet. Perform a normal hotcopy to a + * empty destination using the same configuration as the source. */ + SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path, pool)); + } + else + { + /* Check the existing repository. */ + SVN_ERR(svn_fs_x__open(dst_fs, dst_path, pool)); + SVN_ERR(hotcopy_incremental_check_preconditions(src_fs, dst_fs, + pool)); + hotcopy_setup_shared_fs_data(src_fs, dst_fs); + SVN_ERR(svn_fs_x__initialize_caches(dst_fs, pool)); + } + } + else + { + /* Start out with an empty destination using the same configuration + * as the source. */ + SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path, pool)); + } + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + hbb.src_fs = src_fs; + hbb.dst_fs = dst_fs; + hbb.incremental = incremental; + hbb.cancel_func = cancel_func; + hbb.cancel_baton = cancel_baton; + SVN_ERR(svn_fs_x__with_write_lock(dst_fs, hotcopy_body, &hbb, pool)); + + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_fs_x/hotcopy.h b/subversion/libsvn_fs_x/hotcopy.h new file mode 100644 index 0000000..d39121e --- /dev/null +++ b/subversion/libsvn_fs_x/hotcopy.h
@@ -0,0 +1,40 @@ +/* hotcopy.h : interface to the native filesystem layer + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS__HOTCOPY_H +#define SVN_LIBSVN_FS__HOTCOPY_H + +#include "fs.h" + +/* Copy the fsx filesystem SRC_FS at SRC_PATH into a new copy DST_FS at + * DST_PATH. If INCREMENTAL is TRUE, do not re-copy data which already + * exists in DST_FS. Use POOL for temporary allocations. */ +svn_error_t * svn_fs_x__hotcopy(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + const char *src_path, + const char *dst_path, + svn_boolean_t incremental, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool); + +#endif
diff --git a/subversion/libsvn_fs_x/id.c b/subversion/libsvn_fs_x/id.c new file mode 100644 index 0000000..692f383 --- /dev/null +++ b/subversion/libsvn_fs_x/id.c
@@ -0,0 +1,553 @@ +/* id.c : operations on node-revision IDs + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include <string.h> +#include <stdlib.h> + +#include "id.h" +#include "index.h" + +#include "../libsvn_fs/fs-loader.h" +#include "private/svn_temp_serializer.h" +#include "private/svn_string_private.h" + + +typedef struct fs_x__id_t +{ + /* API visible part */ + svn_fs_id_t generic_id; + + /* private members */ + svn_fs_x__id_part_t node_id; + svn_fs_x__id_part_t copy_id; + svn_fs_x__id_part_t txn_id; + svn_fs_x__id_part_t rev_item; +} fs_x__id_t; + + + +/* Parse the NUL-terminated ID part at DATA and write the result into *PART. + * Return TRUE if no errors were detected. */ +static svn_boolean_t +part_parse(svn_fs_x__id_part_t *part, + const char *data) +{ + /* special case: ID inside some transaction */ + if (data[0] == '_') + { + part->revision = SVN_INVALID_REVNUM; + part->number = svn__base36toui64(&data, data + 1); + return *data == '\0'; + } + + /* special case: 0 / default ID */ + if (data[0] == '0' && data[1] == '\0') + { + part->revision = 0; + part->number = 0; + return TRUE; + } + + /* read old style / new style ID */ + part->number = svn__base36toui64(&data, data); + if (data[0] != '-') + { + part->revision = 0; + return *data == '\0'; + } + + part->revision = SVN_STR_TO_REV(++data); + + return TRUE; +} + +/* Parse the transaction id in DATA and store the result in *TXN_ID. + * Return FALSE if there was some problem. + */ +static svn_boolean_t +txn_id_parse(svn_fs_x__id_part_t *txn_id, + const char *data) +{ + txn_id->revision = SVN_STR_TO_REV(data); + data = strchr(data, '-'); + if (data == NULL) + return FALSE; + + txn_id->number = svn__base36toui64(&data, ++data); + return *data == '\0'; +} + +/* Write the textual representation of *PART into P and return a pointer + * to the first position behind that string. + */ +static char * +unparse_id_part(char *p, + const svn_fs_x__id_part_t *part) +{ + if (SVN_IS_VALID_REVNUM(part->revision)) + { + /* ordinary old style / new style ID */ + p += svn__ui64tobase36(p, part->number); + if (part->revision > 0) + { + *(p++) = '-'; + p += svn__i64toa(p, part->revision); + } + } + else + { + /* in txn: mark with "_" prefix */ + *(p++) = '_'; + p += svn__ui64tobase36(p, part->number); + } + + *(p++) = '.'; + + return p; +} + + + +/* Operations on ID parts */ + +svn_boolean_t +svn_fs_x__id_part_is_root(const svn_fs_x__id_part_t* part) +{ + return part->revision == 0 && part->number == 0; +} + +svn_boolean_t +svn_fs_x__id_part_eq(const svn_fs_x__id_part_t *lhs, + const svn_fs_x__id_part_t *rhs) +{ + return lhs->revision == rhs->revision && lhs->number == rhs->number; +} + +svn_boolean_t +svn_fs_x__id_txn_used(const svn_fs_x__id_part_t *txn_id) +{ + return SVN_IS_VALID_REVNUM(txn_id->revision) || (txn_id->number != 0); +} + +void +svn_fs_x__id_txn_reset(svn_fs_x__id_part_t *txn_id) +{ + txn_id->revision = SVN_INVALID_REVNUM; + txn_id->number = 0; +} + +svn_error_t * +svn_fs_x__id_txn_parse(svn_fs_x__id_part_t *txn_id, + const char *data) +{ + if (! txn_id_parse(txn_id, data)) + return svn_error_createf(SVN_ERR_FS_MALFORMED_TXN_ID, NULL, + "malformed txn id '%s'", data); + + return SVN_NO_ERROR; +} + +const char * +svn_fs_x__id_txn_unparse(const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + char string[2 * SVN_INT64_BUFFER_SIZE + 1]; + char *p = string; + + p += svn__i64toa(p, txn_id->revision); + *(p++) = '-'; + p += svn__ui64tobase36(p, txn_id->number); + + return apr_pstrmemdup(pool, string, p - string); +} + + + +/* Accessing ID Pieces. */ + +const svn_fs_x__id_part_t * +svn_fs_x__id_node_id(const svn_fs_id_t *fs_id) +{ + fs_x__id_t *id = (fs_x__id_t *)fs_id; + + return &id->node_id; +} + + +const svn_fs_x__id_part_t * +svn_fs_x__id_copy_id(const svn_fs_id_t *fs_id) +{ + fs_x__id_t *id = (fs_x__id_t *)fs_id; + + return &id->copy_id; +} + + +const svn_fs_x__id_part_t * +svn_fs_x__id_txn_id(const svn_fs_id_t *fs_id) +{ + fs_x__id_t *id = (fs_x__id_t *)fs_id; + + return &id->txn_id; +} + + +const svn_fs_x__id_part_t * +svn_fs_x__id_rev_item(const svn_fs_id_t *fs_id) +{ + fs_x__id_t *id = (fs_x__id_t *)fs_id; + + return &id->rev_item; +} + +svn_revnum_t +svn_fs_x__id_rev(const svn_fs_id_t *fs_id) +{ + fs_x__id_t *id = (fs_x__id_t *)fs_id; + + return id->rev_item.revision; +} + + +apr_uint64_t +svn_fs_x__id_item(const svn_fs_id_t *fs_id) +{ + fs_x__id_t *id = (fs_x__id_t *)fs_id; + + return id->rev_item.number; +} + +svn_boolean_t +svn_fs_x__id_is_txn(const svn_fs_id_t *fs_id) +{ + fs_x__id_t *id = (fs_x__id_t *)fs_id; + + return svn_fs_x__id_txn_used(&id->txn_id); +} + +svn_string_t * +svn_fs_x__id_unparse(const svn_fs_id_t *fs_id, + apr_pool_t *pool) +{ + char string[6 * SVN_INT64_BUFFER_SIZE + 10]; + fs_x__id_t *id = (fs_x__id_t *)fs_id; + + char *p = unparse_id_part(string, &id->node_id); + p = unparse_id_part(p, &id->copy_id); + + if (svn_fs_x__id_txn_used(&id->txn_id)) + { + *(p++) = 't'; + p += svn__i64toa(p, id->txn_id.revision); + *(p++) = '-'; + p += svn__ui64tobase36(p, id->txn_id.number); + } + else + { + *(p++) = 'r'; + p += svn__i64toa(p, id->rev_item.revision); + *(p++) = '/'; + p += svn__i64toa(p, id->rev_item.number); + } + + return svn_string_ncreate(string, p - string, pool); +} + + +/*** Comparing node IDs ***/ + +svn_boolean_t +svn_fs_x__id_eq(const svn_fs_id_t *a, + const svn_fs_id_t *b) +{ + fs_x__id_t *id_a = (fs_x__id_t *)a; + fs_x__id_t *id_b = (fs_x__id_t *)b; + + if (a == b) + return TRUE; + + return memcmp(&id_a->node_id, &id_b->node_id, + sizeof(*id_a) - sizeof(id_a->generic_id)) == 0; +} + + +svn_boolean_t +svn_fs_x__id_check_related(const svn_fs_id_t *a, + const svn_fs_id_t *b) +{ + fs_x__id_t *id_a = (fs_x__id_t *)a; + fs_x__id_t *id_b = (fs_x__id_t *)b; + + if (a == b) + return TRUE; + + /* If both node_ids start with _ and they have differing transaction + IDs, then it is impossible for them to be related. */ + if (id_a->node_id.revision == SVN_INVALID_REVNUM) + { + if ( !svn_fs_x__id_part_eq(&id_a->txn_id, &id_b->txn_id) + || !svn_fs_x__id_txn_used(&id_a->txn_id)) + return FALSE; + } + + return svn_fs_x__id_part_eq(&id_a->node_id, &id_b->node_id); +} + + +int +svn_fs_x__id_compare(const svn_fs_id_t *a, + const svn_fs_id_t *b) +{ + if (svn_fs_x__id_eq(a, b)) + return 0; + return (svn_fs_x__id_check_related(a, b) ? 1 : -1); +} + +int +svn_fs_x__id_part_compare(const svn_fs_x__id_part_t *a, + const svn_fs_x__id_part_t *b) +{ + if (a->revision < b->revision) + return -1; + if (a->revision > b->revision) + return 1; + + return a->number < b->number ? -1 : a->number == b->number ? 0 : 1; +} + + + +/* Creating ID's. */ + +static id_vtable_t id_vtable = { + svn_fs_x__id_unparse, + svn_fs_x__id_compare +}; + +svn_fs_id_t * +svn_fs_x__id_txn_create_root(const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + fs_x__id_t *id = apr_pcalloc(pool, sizeof(*id)); + + /* node ID and copy ID are "0" */ + + id->txn_id = *txn_id; + id->rev_item.revision = SVN_INVALID_REVNUM; + + id->generic_id.vtable = &id_vtable; + id->generic_id.fsap_data = &id; + + return (svn_fs_id_t *)id; +} + +svn_fs_id_t *svn_fs_x__id_create_root(const svn_revnum_t revision, + apr_pool_t *pool) +{ + fs_x__id_t *id = apr_pcalloc(pool, sizeof(*id)); + + id->txn_id.revision = SVN_INVALID_REVNUM; + id->rev_item.revision = revision; + id->rev_item.number = SVN_FS_X__ITEM_INDEX_ROOT_NODE; + + id->generic_id.vtable = &id_vtable; + id->generic_id.fsap_data = &id; + + return (svn_fs_id_t *)id; +} + +svn_fs_id_t * +svn_fs_x__id_txn_create(const svn_fs_x__id_part_t *node_id, + const svn_fs_x__id_part_t *copy_id, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + fs_x__id_t *id = apr_pcalloc(pool, sizeof(*id)); + + id->node_id = *node_id; + id->copy_id = *copy_id; + id->txn_id = *txn_id; + id->rev_item.revision = SVN_INVALID_REVNUM; + + id->generic_id.vtable = &id_vtable; + id->generic_id.fsap_data = &id; + + return (svn_fs_id_t *)id; +} + + +svn_fs_id_t * +svn_fs_x__id_rev_create(const svn_fs_x__id_part_t *node_id, + const svn_fs_x__id_part_t *copy_id, + const svn_fs_x__id_part_t *rev_item, + apr_pool_t *pool) +{ + fs_x__id_t *id = apr_pcalloc(pool, sizeof(*id)); + + id->node_id = *node_id; + id->copy_id = *copy_id; + id->txn_id.revision = SVN_INVALID_REVNUM; + id->rev_item = *rev_item; + + id->generic_id.vtable = &id_vtable; + id->generic_id.fsap_data = &id; + + return (svn_fs_id_t *)id; +} + + +svn_fs_id_t * +svn_fs_x__id_copy(const svn_fs_id_t *source, apr_pool_t *pool) +{ + fs_x__id_t *id = (fs_x__id_t *)source; + fs_x__id_t *new_id = apr_palloc(pool, sizeof(*new_id)); + + *new_id = *id; + new_id->generic_id.fsap_data = new_id; + + return (svn_fs_id_t *)new_id; +} + + +svn_fs_id_t * +svn_fs_x__id_parse(const char *data, + apr_size_t len, + apr_pool_t *pool) +{ + fs_x__id_t *id; + char *data_copy, *str; + + /* Dup the ID data into POOL. Our returned ID will have references + into this memory. */ + data_copy = apr_pstrmemdup(pool, data, len); + + /* Alloc a new svn_fs_id_t structure. */ + id = apr_pcalloc(pool, sizeof(*id)); + id->generic_id.vtable = &id_vtable; + id->generic_id.fsap_data = &id; + + /* Now, we basically just need to "split" this data on `.' + characters. We will use svn_cstring_tokenize, which will put + terminators where each of the '.'s used to be. Then our new + id field will reference string locations inside our duplicate + string.*/ + + /* Node Id */ + str = svn_cstring_tokenize(".", &data_copy); + if (str == NULL) + return NULL; + if (! part_parse(&id->node_id, str)) + return NULL; + + /* Copy Id */ + str = svn_cstring_tokenize(".", &data_copy); + if (str == NULL) + return NULL; + if (! part_parse(&id->copy_id, str)) + return NULL; + + /* Txn/Rev Id */ + str = svn_cstring_tokenize(".", &data_copy); + if (str == NULL) + return NULL; + + if (str[0] == 'r') + { + apr_int64_t val; + svn_error_t *err; + + /* This is a revision type ID */ + id->txn_id.revision = SVN_INVALID_REVNUM; + id->txn_id.number = 0; + + data_copy = str + 1; + str = svn_cstring_tokenize("/", &data_copy); + if (str == NULL) + return NULL; + id->rev_item.revision = SVN_STR_TO_REV(str); + + err = svn_cstring_atoi64(&val, data_copy); + if (err) + { + svn_error_clear(err); + return NULL; + } + id->rev_item.number = (apr_uint64_t)val; + } + else if (str[0] == 't') + { + /* This is a transaction type ID */ + id->rev_item.revision = SVN_INVALID_REVNUM; + id->rev_item.number = 0; + + if (! txn_id_parse(&id->txn_id, str + 1)) + return NULL; + } + else + return NULL; + + return (svn_fs_id_t *)id; +} + +/* (de-)serialization support */ + +/* Serialize an ID within the serialization CONTEXT. + */ +void +svn_fs_x__id_serialize(svn_temp_serializer__context_t *context, + const svn_fs_id_t * const *in) +{ + const fs_x__id_t *id = (const fs_x__id_t *)*in; + + /* nothing to do for NULL ids */ + if (id == NULL) + return; + + /* serialize the id data struct itself */ + svn_temp_serializer__add_leaf(context, + (const void * const *)in, + sizeof(fs_x__id_t)); +} + +/* Deserialize an ID inside the BUFFER. + */ +void +svn_fs_x__id_deserialize(void *buffer, svn_fs_id_t **in_out) +{ + fs_x__id_t *id; + + /* The id maybe all what is in the whole buffer. + * Don't try to fixup the pointer in that case*/ + if (*in_out != buffer) + svn_temp_deserializer__resolve(buffer, (void**)in_out); + + id = (fs_x__id_t *)*in_out; + + /* no id, no sub-structure fixup necessary */ + if (id == NULL) + return; + + /* the stored vtable is bogus at best -> set the right one */ + id->generic_id.vtable = &id_vtable; + id->generic_id.fsap_data = id; +} +
diff --git a/subversion/libsvn_fs_x/id.h b/subversion/libsvn_fs_x/id.h new file mode 100644 index 0000000..11d7879 --- /dev/null +++ b/subversion/libsvn_fs_x/id.h
@@ -0,0 +1,174 @@ +/* id.h : interface to node ID functions, private to libsvn_fs_x + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS_X_ID_H +#define SVN_LIBSVN_FS_X_ID_H + +#include "svn_fs.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/* A rev node ID in FSX consists of a 3 of sub-IDs ("parts") that consist + * of a creation REVISION number and some revision-local counter value + * (NUMBER). Old-style ID parts use global counter values. + */ +typedef struct svn_fs_x__id_part_t +{ + /* SVN_INVALID_REVNUM for txns -> not a txn, COUNTER must be 0. + SVN_INVALID_REVNUM for others -> not assigned to a revision, yet. + 0 for others -> old-style ID or the root in rev 0. */ + svn_revnum_t revision; + + /* some numerical value. */ + apr_uint64_t number; +} svn_fs_x__id_part_t; + + +/*** Operations on ID parts. ***/ + +/* Return TRUE, if both elements of the PART is 0, i.e. this is the default + * value if e.g. no copies were made of this node. */ +svn_boolean_t svn_fs_x__id_part_is_root(const svn_fs_x__id_part_t *part); + +/* Return TRUE, if all element values of *LHS and *RHS match. */ +svn_boolean_t svn_fs_x__id_part_eq(const svn_fs_x__id_part_t *lhs, + const svn_fs_x__id_part_t *rhs); + +/* Return TRUE, if TXN_ID is used, i.e. doesn't contain just the defaults. */ +svn_boolean_t svn_fs_x__id_txn_used(const svn_fs_x__id_part_t *txn_id); + +/* Reset TXN_ID to the defaults. */ +void svn_fs_x__id_txn_reset(svn_fs_x__id_part_t *txn_id); + +/* Parse the transaction id in DATA and store the result in *TXN_ID */ +svn_error_t *svn_fs_x__id_txn_parse(svn_fs_x__id_part_t *txn_id, + const char *data); + +/* Convert the transaction id in *TXN_ID into a textual representation + * allocated in POOL. */ +const char *svn_fs_x__id_txn_unparse(const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + + +/*** ID accessor functions. ***/ + +/* Get the "node id" portion of ID. */ +const svn_fs_x__id_part_t *svn_fs_x__id_node_id(const svn_fs_id_t *id); + +/* Get the "copy id" portion of ID. */ +const svn_fs_x__id_part_t *svn_fs_x__id_copy_id(const svn_fs_id_t *id); + +/* Get the "txn id" portion of ID, or NULL if it is a permanent ID. */ +const svn_fs_x__id_part_t *svn_fs_x__id_txn_id(const svn_fs_id_t *id); + +/* Get the "rev,item" portion of ID. */ +const svn_fs_x__id_part_t *svn_fs_x__id_rev_item(const svn_fs_id_t *id); + +/* Get the "rev" portion of ID, or SVN_INVALID_REVNUM if it is a + transaction ID. */ +svn_revnum_t svn_fs_x__id_rev(const svn_fs_id_t *id); + +/* Access the "item" portion of the ID, or 0 if it is a transaction + ID. */ +apr_uint64_t svn_fs_x__id_item(const svn_fs_id_t *id); + +/* Return TRUE, if this is a transaction ID. */ +svn_boolean_t svn_fs_x__id_is_txn(const svn_fs_id_t *id); + +/* Convert ID into string form, allocated in POOL. */ +svn_string_t *svn_fs_x__id_unparse(const svn_fs_id_t *id, + apr_pool_t *pool); + +/* Return true if A and B are equal. */ +svn_boolean_t svn_fs_x__id_eq(const svn_fs_id_t *a, + const svn_fs_id_t *b); + +/* Return true if A and B are related. */ +svn_boolean_t svn_fs_x__id_check_related(const svn_fs_id_t *a, + const svn_fs_id_t *b); + +/* Return 0 if A and B are equal, 1 if they are related, -1 otherwise. */ +int svn_fs_x__id_compare(const svn_fs_id_t *a, + const svn_fs_id_t *b); + +/* Return 0 if A and B are equal, 1 if A is "greater than" B, -1 otherwise. */ +int svn_fs_x__id_part_compare(const svn_fs_x__id_part_t *a, + const svn_fs_x__id_part_t *b); + +/* Create the txn root ID for transaction TXN_ID. Allocate it in POOL. */ +svn_fs_id_t *svn_fs_x__id_txn_create_root(const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +/* Create the root ID for REVISION. Allocate it in POOL. */ +svn_fs_id_t *svn_fs_x__id_create_root(const svn_revnum_t revision, + apr_pool_t *pool); + +/* Create an ID within a transaction based on NODE_ID, COPY_ID, and + TXN_ID, allocated in POOL. */ +svn_fs_id_t *svn_fs_x__id_txn_create(const svn_fs_x__id_part_t *node_id, + const svn_fs_x__id_part_t *copy_id, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +/* Create a permanent ID based on NODE_ID, COPY_ID and REV_ITEM, + allocated in POOL. */ +svn_fs_id_t *svn_fs_x__id_rev_create(const svn_fs_x__id_part_t *node_id, + const svn_fs_x__id_part_t *copy_id, + const svn_fs_x__id_part_t *rev_item, + apr_pool_t *pool); + +/* Return a copy of ID, allocated from POOL. */ +svn_fs_id_t *svn_fs_x__id_copy(const svn_fs_id_t *id, + apr_pool_t *pool); + +/* Return an ID resulting from parsing the string DATA (with length + LEN), or NULL if DATA is an invalid ID string. */ +svn_fs_id_t *svn_fs_x__id_parse(const char *data, + apr_size_t len, + apr_pool_t *pool); + + +/* (de-)serialization support*/ + +struct svn_temp_serializer__context_t; + +/** + * Serialize an @a id within the serialization @a context. + */ +void +svn_fs_x__id_serialize(struct svn_temp_serializer__context_t *context, + const svn_fs_id_t * const *id); + +/** + * Deserialize an @a id within the @a buffer. + */ +void +svn_fs_x__id_deserialize(void *buffer, + svn_fs_id_t **id); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* SVN_LIBSVN_FS_X_ID_H */
diff --git a/subversion/libsvn_fs_x/index.c b/subversion/libsvn_fs_x/index.c new file mode 100644 index 0000000..f84584c --- /dev/null +++ b/subversion/libsvn_fs_x/index.c
@@ -0,0 +1,2927 @@ +/* index.c indexing support for FSX support + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include <assert.h> + +#include "svn_io.h" +#include "svn_pools.h" +#include "svn_sorts.h" + +#include "index.h" +#include "util.h" +#include "pack.h" + +#include "private/svn_subr_private.h" +#include "private/svn_temp_serializer.h" + +#include "svn_private_config.h" +#include "temp_serializer.h" + +#include "../libsvn_fs/fs-loader.h" + +/* maximum length of a uint64 in an 7/8b encoding */ +#define ENCODED_INT_LENGTH 10 + +/* Page tables in the log-to-phys index file exclusively contain entries + * of this type to describe position and size of a given page. + */ +typedef struct l2p_page_table_entry_t +{ + /* global offset on the page within the index file */ + apr_uint64_t offset; + + /* number of mapping entries in that page */ + apr_uint32_t entry_count; + + /* size of the page on disk (in the index file) */ + apr_uint32_t size; +} l2p_page_table_entry_t; + +/* Master run-time data structure of an log-to-phys index. It contains + * the page tables of every revision covered by that index - but not the + * pages themselves. + */ +typedef struct l2p_header_t +{ + /* first revision covered by this index */ + svn_revnum_t first_revision; + + /* number of revisions covered */ + apr_size_t revision_count; + + /* (max) number of entries per page */ + apr_size_t page_size; + + /* indexes into PAGE_TABLE that mark the first page of the respective + * revision. PAGE_TABLE_INDEX[REVISION_COUNT] points to the end of + * PAGE_TABLE. + */ + apr_size_t * page_table_index; + + /* Page table covering all pages in the index */ + l2p_page_table_entry_t * page_table; +} l2p_header_t; + +/* Run-time data structure containing a single log-to-phys index page. + */ +typedef struct l2p_page_t +{ + /* number of entries in the OFFSETS array */ + apr_uint32_t entry_count; + + /* global file offsets (item index is the array index) within the + * packed or non-packed rev file. Offset will be -1 for unused / + * invalid item index values. */ + apr_off_t *offsets; + + /* In case that the item is stored inside a container, this is the + * identifying index of the item within that container. 0 for the + * container itself or for items that aren't containers. */ + apr_uint32_t *sub_items; +} l2p_page_t; + +/* All of the log-to-phys proto index file consist of entires of this type. + */ +typedef struct l2p_proto_entry_t +{ + /* phys offset + 1 of the data container. 0 for "new revision" entries. */ + apr_uint64_t offset; + + /* corresponding item index. 0 for "new revision" entries. */ + apr_uint64_t item_index; + + /* index within the container starting @ offset. 0 for "new revision" + * entries and for items with no outer container. */ + apr_uint32_t sub_item; +} l2p_proto_entry_t; + +/* Master run-time data structure of an phys-to-log index. It contains + * an array with one offset value for each rev file cluster. + */ +typedef struct p2l_header_t +{ + /* first revision covered by the index (and rev file) */ + svn_revnum_t first_revision; + + /* number of bytes in the rev files covered by each p2l page */ + apr_uint64_t page_size; + + /* number of pages / clusters in that rev file */ + apr_size_t page_count; + + /* offsets of the pages / cluster descriptions within the index file */ + apr_off_t *offsets; +} p2l_header_t; + +/* + * packed stream array + */ + +/* How many numbers we will pre-fetch and buffer in a packed number stream. + */ +enum { MAX_NUMBER_PREFETCH = 64 }; + +/* Prefetched number entry in a packed number stream. + */ +typedef struct value_position_pair_t +{ + /* prefetched number */ + apr_uint64_t value; + + /* number of bytes read, *including* this number, since the buffer start */ + apr_size_t total_len; +} value_position_pair_t; + +/* State of a prefetching packed number stream. It will read compressed + * index data efficiently and present it as a series of non-packed uint64. + */ +typedef struct packed_number_stream_t +{ + /* underlying data file containing the packed values */ + apr_file_t *file; + + /* number of used entries in BUFFER (starting at index 0) */ + apr_size_t used; + + /* index of the next number to read from the BUFFER (0 .. USED). + * If CURRENT == USED, we need to read more data upon get() */ + apr_size_t current; + + /* offset in FILE from which the first entry in BUFFER has been read */ + apr_off_t start_offset; + + /* offset in FILE from which the next number has to be read */ + apr_off_t next_offset; + + /* read the file in chunks of this size */ + apr_size_t block_size; + + /* pool to be used for file ops etc. */ + apr_pool_t *pool; + + /* buffer for prefetched values */ + value_position_pair_t buffer[MAX_NUMBER_PREFETCH]; +} packed_number_stream_t; + +/* Return an svn_error_t * object for error ERR on STREAM with the given + * MESSAGE string. The latter must have a placeholder for the index file + * name ("%s") and the current read offset (e.g. "0x%lx"). + */ +static svn_error_t * +stream_error_create(packed_number_stream_t *stream, + apr_status_t err, + const char *message) +{ + const char *file_name; + apr_off_t offset = 0; + SVN_ERR(svn_io_file_name_get(&file_name, stream->file, + stream->pool)); + SVN_ERR(svn_io_file_seek(stream->file, SEEK_CUR, &offset, stream->pool)); + + return svn_error_createf(err, NULL, message, file_name, + (apr_uint64_t)offset); +} + +/* Read up to MAX_NUMBER_PREFETCH numbers from the STREAM->NEXT_OFFSET in + * STREAM->FILE and buffer them. + * + * We don't want GCC and others to inline this function into get() because + * it prevents get() from being inlined itself. + */ +SVN__PREVENT_INLINE +static svn_error_t * +packed_stream_read(packed_number_stream_t *stream) +{ + unsigned char buffer[MAX_NUMBER_PREFETCH]; + apr_size_t read = 0; + apr_size_t i; + value_position_pair_t *target; + apr_off_t block_start = 0; + apr_off_t block_left = 0; + apr_status_t err; + + /* all buffered data will have been read starting here */ + stream->start_offset = stream->next_offset; + + /* packed numbers are usually not aligned to MAX_NUMBER_PREFETCH blocks, + * i.e. the last number has been incomplete (and not buffered in stream) + * and need to be re-read. Therefore, always correct the file pointer. + */ + SVN_ERR(svn_io_file_aligned_seek(stream->file, stream->block_size, + &block_start, stream->next_offset, + stream->pool)); + + /* prefetch at least one number but, if feasible, don't cross block + * boundaries. This shall prevent jumping back and forth between two + * blocks because the extra data was not actually request _now_. + */ + read = sizeof(buffer); + block_left = stream->block_size - (stream->next_offset - block_start); + if (block_left >= 10 && block_left < read) + read = block_left; + + err = apr_file_read(stream->file, buffer, &read); + if (err && !APR_STATUS_IS_EOF(err)) + return stream_error_create(stream, err, + _("Can't read index file '%s' at offset 0x%" APR_UINT64_T_HEX_FMT)); + + /* if the last number is incomplete, trim it from the buffer */ + while (read > 0 && buffer[read-1] >= 0x80) + --read; + + /* we call read() only if get() requires more data. So, there must be + * at least *one* further number. */ + if SVN__PREDICT_FALSE(read == 0) + return stream_error_create(stream, err, + _("Unexpected end of index file %s at offset 0x%"APR_UINT64_T_HEX_FMT)); + + /* parse file buffer and expand into stream buffer */ + target = stream->buffer; + for (i = 0; i < read;) + { + if (buffer[i] < 0x80) + { + /* numbers < 128 are relatively frequent and particularly easy + * to decode. Give them special treatment. */ + target->value = buffer[i]; + ++i; + target->total_len = i; + ++target; + } + else + { + apr_uint64_t value = 0; + apr_uint64_t shift = 0; + while (buffer[i] >= 0x80) + { + value += ((apr_uint64_t)buffer[i] & 0x7f) << shift; + shift += 7; + ++i; + } + + target->value = value + ((apr_uint64_t)buffer[i] << shift); + ++i; + target->total_len = i; + ++target; + + /* let's catch corrupted data early. It would surely cause + * havoc further down the line. */ + if SVN__PREDICT_FALSE(shift > 8 * sizeof(value)) + return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_CORRUPTION, NULL, + _("Corrupt index: number too large")); + } + } + + /* update stream state */ + stream->used = target - stream->buffer; + stream->next_offset = stream->start_offset + i; + stream->current = 0; + + return SVN_NO_ERROR; +}; + +/* Create and open a packed number stream reading from FILE_NAME and + * return it in *STREAM. Access the file in chunks of BLOCK_SIZE bytes. + * Use POOL for allocations. + */ +static svn_error_t * +packed_stream_open(packed_number_stream_t **stream, + const char *file_name, + apr_size_t block_size, + apr_pool_t *pool) +{ + packed_number_stream_t *result = apr_palloc(pool, sizeof(*result)); + result->pool = svn_pool_create(pool); + + SVN_ERR(svn_io_file_open(&result->file, file_name, + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, + result->pool)); + + result->used = 0; + result->current = 0; + result->start_offset = 0; + result->next_offset = 0; + result->block_size = block_size; + + *stream = result; + + return SVN_NO_ERROR; +} + +/* Close STREAM which may be NULL. + */ +SVN__FORCE_INLINE +static svn_error_t * +packed_stream_close(packed_number_stream_t *stream) +{ + if (stream) + { + SVN_ERR(svn_io_file_close(stream->file, stream->pool)); + svn_pool_destroy(stream->pool); + } + + return SVN_NO_ERROR; +} + +/* + * The forced inline is required as e.g. GCC would inline read() into here + * instead of lining the simple buffer access into callers of get(). + */ +SVN__FORCE_INLINE +static svn_error_t* +packed_stream_get(apr_uint64_t *value, + packed_number_stream_t *stream) +{ + if (stream->current == stream->used) + SVN_ERR(packed_stream_read(stream)); + + *value = stream->buffer[stream->current].value; + ++stream->current; + + return SVN_NO_ERROR; +} + +/* Navigate STREAM to packed file offset OFFSET. There will be no checks + * whether the given OFFSET is valid. + */ +static void +packed_stream_seek(packed_number_stream_t *stream, + apr_off_t offset) +{ + if ( stream->used == 0 + || offset < stream->start_offset + || offset >= stream->next_offset) + { + /* outside buffered data. Next get() will read() from OFFSET. */ + stream->start_offset = offset; + stream->next_offset = offset; + stream->current = 0; + stream->used = 0; + } + else + { + /* Find the suitable location in the stream buffer. + * Since our buffer is small, it is efficient enough to simply scan + * it for the desired position. */ + apr_size_t i; + for (i = 0; i < stream->used; ++i) + if (stream->buffer[i].total_len > offset - stream->start_offset) + break; + + stream->current = i; + } +} + +/* Return the packed file offset of at which the next number in the stream + * can be found. + */ +static apr_off_t +packed_stream_offset(packed_number_stream_t *stream) +{ + return stream->current == 0 + ? stream->start_offset + : stream->buffer[stream->current-1].total_len + stream->start_offset; +} + +/* + * log-to-phys index + */ +svn_error_t * +svn_fs_x__l2p_proto_index_open(apr_file_t **proto_index, + const char *file_name, + apr_pool_t *pool) +{ + SVN_ERR(svn_io_file_open(proto_index, file_name, APR_READ | APR_WRITE + | APR_CREATE | APR_APPEND | APR_BUFFERED, + APR_OS_DEFAULT, pool)); + + return SVN_NO_ERROR; +} + +/* Write ENTRY to log-to-phys PROTO_INDEX file and verify the results. + * Use POOL for allocations. + */ +static svn_error_t * +write_entry_to_proto_index(apr_file_t *proto_index, + l2p_proto_entry_t entry, + apr_pool_t *pool) +{ + apr_size_t written = sizeof(entry); + + SVN_ERR(svn_io_file_write(proto_index, &entry, &written, pool)); + SVN_ERR_ASSERT(written == sizeof(entry)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__l2p_proto_index_add_revision(apr_file_t *proto_index, + apr_pool_t *pool) +{ + l2p_proto_entry_t entry; + entry.offset = 0; + entry.item_index = 0; + + return svn_error_trace(write_entry_to_proto_index(proto_index, entry, + pool)); +} + +svn_error_t * +svn_fs_x__l2p_proto_index_add_entry(apr_file_t *proto_index, + apr_off_t offset, + apr_uint32_t sub_item, + apr_uint64_t item_index, + apr_pool_t *pool) +{ + l2p_proto_entry_t entry; + + /* make sure the conversion to uint64 works */ + SVN_ERR_ASSERT(offset >= -1); + + /* we support offset '-1' as a "not used" indication */ + entry.offset = (apr_uint64_t)offset + 1; + + /* make sure we can use item_index as an array index when building the + * final index file */ + SVN_ERR_ASSERT(item_index < UINT_MAX / 2); + entry.item_index = item_index; + + /* no limits on the container sub-item index */ + entry.sub_item = sub_item; + + return svn_error_trace(write_entry_to_proto_index(proto_index, entry, + pool)); +} + +/* Encode VALUE as 7/8b into P and return the number of bytes written. + */ +static apr_size_t +encode_uint(unsigned char *p, apr_uint64_t value) +{ + unsigned char *start = p; + while (value >= 0x80) + { + *p = (unsigned char)((value % 0x80) + 0x80); + value /= 0x80; + ++p; + } + + *p = (unsigned char)(value % 0x80); + return (p - start) + 1; +} + +/* Encode VALUE as 7/8b into P and return the number of bytes written. + */ +static apr_size_t +encode_int(unsigned char *p, apr_int64_t value) +{ + return encode_uint(p, (apr_uint64_t)(value < 0 ? -1 - 2*value : 2*value)); +} + +/* Run-length-encode the uint64 numbers in ARRAY starting at index START + * up to but not including END. All numbers must be > 0. + * Return the number of remaining entries in ARRAY after START. + */ +static int +rle_array(apr_array_header_t *array, int start, int end) +{ + int i; + int target = start; + for (i = start; i < end; ++i) + { + apr_uint64_t value = APR_ARRAY_IDX(array, i, apr_uint64_t); + assert(value > 0); + + if (value == 1) + { + int counter; + for (counter = 1; i + counter < end; ++counter) + if (APR_ARRAY_IDX(array, i + counter, apr_uint64_t) != 1) + break; + + if (--counter) + { + APR_ARRAY_IDX(array, target, apr_uint64_t) = 0; + APR_ARRAY_IDX(array, target + 1, apr_uint64_t) = counter; + target += 2; + i += counter; + continue; + } + } + + APR_ARRAY_IDX(array, target, apr_uint64_t) = value; + ++target; + } + + return target; +} + +/* Utility data structure describing an log-2-phys page entry. + * This is only used as a transient representation during index creation. + */ +typedef struct l2p_page_entry_t +{ + apr_uint64_t offset; + apr_uint32_t sub_item; +} l2p_page_entry_t; + +/* qsort-compatible compare function taking two l2p_page_entry_t and + * ordering them by offset. + */ +static int +compare_l2p_entries_by_offset(const l2p_page_entry_t *lhs, + const l2p_page_entry_t *rhs) +{ + return lhs->offset > rhs->offset ? 1 + : lhs->offset == rhs->offset ? 0 : -1; +} + +/* Write the log-2-phys index page description for the l2p_page_entry_t + * array ENTRIES, starting with element START up to but not including END. + * Write the resulting represenation into BUFFER. Use POOL for tempoary + * allocations. + */ +static svn_error_t * +encode_l2p_page(apr_array_header_t *entries, + int start, + int end, + svn_spillbuf_t *buffer, + apr_pool_t *pool) +{ + unsigned char encoded[ENCODED_INT_LENGTH]; + apr_hash_t *containers = apr_hash_make(pool); + int count = end - start; + int container_count = 0; + apr_uint64_t last_offset = 0; + int i; + + apr_size_t data_size = count * sizeof(l2p_page_entry_t); + svn_stringbuf_t *container_offsets + = svn_stringbuf_create_ensure(count * 2, pool); + + /* SORTED: relevant items from ENTRIES, sorted by offset */ + l2p_page_entry_t *sorted + = apr_pmemdup(pool, + entries->elts + start * sizeof(l2p_page_entry_t), + data_size); + qsort(sorted, end - start, sizeof(l2p_page_entry_t), + (int (*)(const void *, const void *))compare_l2p_entries_by_offset); + + /* identify container offsets and create container list */ + for (i = 0; i < count; ++i) + { + /* skip "unused" entries */ + if (sorted[i].offset == 0) + continue; + + /* offset already covered? */ + if (i > 0 && sorted[i].offset == sorted[i-1].offset) + continue; + + /* is this a container item + * (appears more than once or accesses to sub-items other than 0)? */ + if ( (i != count-1 && sorted[i].offset == sorted[i+1].offset) + || (sorted[i].sub_item != 0)) + { + svn_stringbuf_appendbytes(container_offsets, (const char *)encoded, + encode_uint(encoded, sorted[i].offset + - last_offset)); + last_offset = sorted[i].offset; + apr_hash_set(containers, + &sorted[i].offset, + sizeof(sorted[i].offset), + (void *)(apr_uintptr_t)++container_count); + } + } + + /* write container list to BUFFER */ + SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, + encode_uint(encoded, container_count), + pool)); + SVN_ERR(svn_spillbuf__write(buffer, (const char *)container_offsets->data, + container_offsets->len, pool)); + + /* encode items */ + for (i = start; i < end; ++i) + { + l2p_page_entry_t *entry = &APR_ARRAY_IDX(entries, i, l2p_page_entry_t); + if (entry->offset == 0) + { + SVN_ERR(svn_spillbuf__write(buffer, "\0", 1, pool)); + } + else + { + apr_uintptr_t idx + = (apr_uintptr_t)apr_hash_get(containers, &entry->offset, + sizeof(entry->offset)); + if (idx == 0) + { + apr_uint64_t value = entry->offset + container_count; + SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, + encode_uint(encoded, value), pool)); + } + else + { + apr_uint64_t value = entry->sub_item; + SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, + encode_uint(encoded, idx), pool)); + SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, + encode_uint(encoded, value), pool)); + } + } + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__l2p_index_create(svn_fs_t *fs, + const char *file_name, + const char *proto_file_name, + svn_revnum_t revision, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + apr_file_t *proto_index = NULL; + int i; + int end; + apr_uint64_t entry; + svn_boolean_t eof = FALSE; + apr_file_t *index_file; + unsigned char encoded[ENCODED_INT_LENGTH]; + + int last_page_count = 0; /* total page count at the start of + the current revision */ + + /* temporary data structures that collect the data which will be moved + to the target file in a second step */ + apr_pool_t *local_pool = svn_pool_create(pool); + apr_pool_t *iterpool = svn_pool_create(local_pool); + apr_array_header_t *page_counts + = apr_array_make(local_pool, 16, sizeof(apr_uint64_t)); + apr_array_header_t *page_sizes + = apr_array_make(local_pool, 16, sizeof(apr_uint64_t)); + apr_array_header_t *entry_counts + = apr_array_make(local_pool, 16, sizeof(apr_uint64_t)); + + /* collect the item offsets and sub-item value for the current revision */ + apr_array_header_t *entries + = apr_array_make(local_pool, 256, sizeof(l2p_page_entry_t)); + + /* 64k blocks, spill after 16MB */ + svn_spillbuf_t *buffer + = svn_spillbuf__create(0x10000, 0x1000000, local_pool); + + /* start at the beginning of the source file */ + SVN_ERR(svn_io_file_open(&proto_index, proto_file_name, + APR_READ | APR_CREATE | APR_BUFFERED, + APR_OS_DEFAULT, pool)); + + /* process all entries until we fail due to EOF */ + for (entry = 0; !eof; ++entry) + { + l2p_proto_entry_t proto_entry; + apr_size_t read = 0; + + /* (attempt to) read the next entry from the source */ + SVN_ERR(svn_io_file_read_full2(proto_index, + &proto_entry, sizeof(proto_entry), + &read, &eof, local_pool)); + SVN_ERR_ASSERT(eof || read == sizeof(proto_entry)); + + /* handle new revision */ + if ((entry > 0 && proto_entry.offset == 0) || eof) + { + /* dump entries, grouped into pages */ + + int entry_count = 0; + for (i = 0; i < entries->nelts; i += entry_count) + { + /* 1 page with up to 8k entries */ + apr_size_t last_buffer_size = svn_spillbuf__get_size(buffer); + entry_count = MIN(entries->nelts - i, ffd->l2p_page_size); + + SVN_ERR(encode_l2p_page(entries, i, i + entry_count, + buffer, iterpool)); + + APR_ARRAY_PUSH(entry_counts, apr_uint64_t) = entry_count; + APR_ARRAY_PUSH(page_sizes, apr_uint64_t) + = svn_spillbuf__get_size(buffer) - last_buffer_size; + + svn_pool_clear(iterpool); + } + + apr_array_clear(entries); + + /* store the number of pages in this revision */ + APR_ARRAY_PUSH(page_counts, apr_uint64_t) + = page_sizes->nelts - last_page_count; + + last_page_count = page_sizes->nelts; + } + else + { + /* store the mapping in our array */ + l2p_page_entry_t page_entry = { 0 }; + int idx = (apr_size_t)proto_entry.item_index; + + while (idx >= entries->nelts) + APR_ARRAY_PUSH(entries, l2p_page_entry_t) = page_entry; + + page_entry.offset = proto_entry.offset; + page_entry.sub_item = proto_entry.sub_item; + APR_ARRAY_IDX(entries, idx, l2p_page_entry_t) = page_entry; + } + } + + /* create the target file */ + SVN_ERR(svn_io_file_open(&index_file, file_name, APR_WRITE + | APR_CREATE | APR_TRUNCATE | APR_BUFFERED, + APR_OS_DEFAULT, local_pool)); + + /* write header info */ + SVN_ERR(svn_io_file_write_full(index_file, encoded, + encode_uint(encoded, revision), + NULL, local_pool)); + SVN_ERR(svn_io_file_write_full(index_file, encoded, + encode_uint(encoded, page_counts->nelts), + NULL, local_pool)); + SVN_ERR(svn_io_file_write_full(index_file, encoded, + encode_uint(encoded, ffd->l2p_page_size), + NULL, local_pool)); + SVN_ERR(svn_io_file_write_full(index_file, encoded, + encode_uint(encoded, page_sizes->nelts), + NULL, local_pool)); + + /* write the revision table */ + end = rle_array(page_counts, 0, page_counts->nelts); + for (i = 0; i < end; ++i) + { + apr_uint64_t value = APR_ARRAY_IDX(page_counts, i, apr_uint64_t); + SVN_ERR(svn_io_file_write_full(index_file, encoded, + encode_uint(encoded, value), + NULL, local_pool)); + } + + /* write the page table */ + for (i = 0; i < page_sizes->nelts; ++i) + { + apr_uint64_t value = APR_ARRAY_IDX(page_sizes, i, apr_uint64_t); + SVN_ERR(svn_io_file_write_full(index_file, encoded, + encode_uint(encoded, value), + NULL, local_pool)); + value = APR_ARRAY_IDX(entry_counts, i, apr_uint64_t); + SVN_ERR(svn_io_file_write_full(index_file, encoded, + encode_uint(encoded, value), + NULL, local_pool)); + } + + /* append page contents */ + SVN_ERR(svn_stream_copy3(svn_stream__from_spillbuf(buffer, local_pool), + svn_stream_from_aprfile2(index_file, TRUE, + local_pool), + NULL, NULL, local_pool)); + + /* finalize the index file */ + SVN_ERR(svn_io_file_close(index_file, local_pool)); + SVN_ERR(svn_io_set_file_read_only(file_name, FALSE, local_pool)); + + svn_pool_destroy(local_pool); + + return SVN_NO_ERROR; +} + +/* Return the base revision used to identify the p2l or lp2 index covering + * REVISION in FS. + */ +static svn_revnum_t +base_revision(svn_fs_t *fs, svn_revnum_t revision) +{ + fs_x_data_t *ffd = fs->fsap_data; + return svn_fs_x__is_packed_rev(fs, revision) + ? revision - (revision % ffd->max_files_per_dir) + : revision; +} + +/* Data structure that describes which l2p page info shall be extracted + * from the cache and contains the fields that receive the result. + */ +typedef struct l2p_page_info_baton_t +{ + /* input data: we want the page covering (REVISION,ITEM_INDEX) */ + svn_revnum_t revision; + apr_uint64_t item_index; + + /* out data */ + /* page location and size of the page within the l2p index file */ + l2p_page_table_entry_t entry; + + /* page number within the pages for REVISION (not l2p index global!) */ + apr_size_t page_no; + + /* offset of ITEM_INDEX within that page */ + apr_uint32_t page_offset; + + /* revision identifying the l2p index file, also the first rev in that */ + svn_revnum_t first_revision; +} l2p_page_info_baton_t; + + +/* Utility function that copies the info requested by BATON->REVISION and + * BATON->ITEM_INDEX and from HEADER and PAGE_TABLE into the output fields + * of *BATON. + */ +static svn_error_t * +l2p_header_copy(l2p_page_info_baton_t *baton, + const l2p_header_t *header, + const l2p_page_table_entry_t *page_table, + const apr_size_t *page_table_index) +{ + /* revision offset within the index file */ + apr_size_t rel_revision = baton->revision - header->first_revision; + if (rel_revision >= header->revision_count) + return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_REVISION , NULL, + _("Revision %ld not covered by item index"), + baton->revision); + + /* select the relevant page */ + if (baton->item_index < header->page_size) + { + /* most revs fit well into a single page */ + baton->page_offset = (apr_size_t)baton->item_index; + baton->page_no = 0; + baton->entry = page_table[page_table_index[rel_revision]]; + } + else + { + const l2p_page_table_entry_t *first_entry; + const l2p_page_table_entry_t *last_entry; + + /* all pages are of the same size and full, except for the last one */ + baton->page_offset = (apr_size_t)(baton->item_index % header->page_size); + baton->page_no = (apr_uint32_t)(baton->item_index / header->page_size); + + /* range of pages for this rev */ + first_entry = page_table + page_table_index[rel_revision]; + last_entry = page_table + page_table_index[rel_revision + 1]; + + if (last_entry - first_entry > baton->page_no) + { + baton->entry = first_entry[baton->page_no]; + } + else + { + /* limit page index to the valid range */ + baton->entry = last_entry[-1]; + + /* cause index overflow further down the road */ + baton->page_offset = header->page_size + 1; + } + } + + baton->first_revision = header->first_revision; + + return SVN_NO_ERROR; +} + +/* Implement svn_cache__partial_getter_func_t: copy the data requested in + * l2p_page_info_baton_t *BATON from l2p_header_t *DATA into the output + * fields in *BATON. + */ +static svn_error_t * +l2p_header_access_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *result_pool) +{ + /* resolve all pointer values of in-cache data */ + const l2p_header_t *header = data; + const l2p_page_table_entry_t *page_table + = svn_temp_deserializer__ptr(header, + (const void *const *)&header->page_table); + const apr_size_t *page_table_index + = svn_temp_deserializer__ptr(header, + (const void *const *)&header->page_table_index); + + /* copy the info */ + return l2p_header_copy(baton, header, page_table, page_table_index); +} + +/* Read COUNT run-length-encoded (see rle_array) uint64 from STREAM and + * return them in VALUES. + */ +static svn_error_t * +expand_rle(apr_array_header_t *values, + packed_number_stream_t *stream, + apr_size_t count) +{ + apr_array_clear(values); + + while (count) + { + apr_uint64_t value; + SVN_ERR(packed_stream_get(&value, stream)); + + if (value) + { + APR_ARRAY_PUSH(values, apr_uint64_t) = value; + --count; + } + else + { + apr_uint64_t i; + apr_uint64_t repetitions; + SVN_ERR(packed_stream_get(&repetitions, stream)); + if (++repetitions > count) + repetitions = count; + + for (i = 0; i < repetitions; ++i) + APR_ARRAY_PUSH(values, apr_uint64_t) = 1; + + count -= repetitions; + } + } + + return SVN_NO_ERROR; +} + +/* Read the header data structure of the log-to-phys index for REVISION + * in FS and return it in *HEADER. To maximize efficiency, use or return + * the data stream in *STREAM. Use POOL for allocations. + */ +static svn_error_t * +get_l2p_header_body(l2p_header_t **header, + packed_number_stream_t **stream, + svn_fs_t *fs, + svn_revnum_t revision, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + apr_uint64_t value; + int i; + apr_size_t page, page_count; + apr_off_t offset; + l2p_header_t *result = apr_pcalloc(pool, sizeof(*result)); + apr_size_t page_table_index; + apr_array_header_t *expanded_values + = apr_array_make(pool, 16, sizeof(apr_uint64_t)); + + pair_cache_key_t key; + key.revision = base_revision(fs, revision); + key.second = svn_fs_x__is_packed_rev(fs, revision); + + if (*stream == NULL) + SVN_ERR(packed_stream_open(stream, + svn_fs_x__path_l2p_index(fs, revision, pool), + ffd->block_size, pool)); + else + packed_stream_seek(*stream, 0); + + /* read the table sizes */ + SVN_ERR(packed_stream_get(&value, *stream)); + result->first_revision = (svn_revnum_t)value; + SVN_ERR(packed_stream_get(&value, *stream)); + result->revision_count = (int)value; + SVN_ERR(packed_stream_get(&value, *stream)); + result->page_size = (apr_size_t)value; + SVN_ERR(packed_stream_get(&value, *stream)); + page_count = (apr_size_t)value; + + /* allocate the page tables */ + result->page_table + = apr_pcalloc(pool, page_count * sizeof(*result->page_table)); + result->page_table_index + = apr_pcalloc(pool, (result->revision_count + 1) + * sizeof(*result->page_table_index)); + + /* read per-revision page table sizes (i.e. number of pages per rev) */ + page_table_index = 0; + result->page_table_index[0] = page_table_index; + SVN_ERR(expand_rle(expanded_values, *stream, result->revision_count)); + for (i = 0; i < result->revision_count; ++i) + { + page_table_index + += (apr_size_t)APR_ARRAY_IDX(expanded_values, i, apr_uint64_t); + result->page_table_index[i+1] = page_table_index; + } + + /* read actual page tables */ + for (page = 0; page < page_count; ++page) + { + SVN_ERR(packed_stream_get(&value, *stream)); + result->page_table[page].size = (apr_uint32_t)value; + SVN_ERR(packed_stream_get(&value, *stream)); + result->page_table[page].entry_count = (apr_uint32_t)value; + } + + /* correct the page description offsets */ + offset = packed_stream_offset(*stream); + for (page = 0; page < page_count; ++page) + { + result->page_table[page].offset = offset; + offset += result->page_table[page].size; + } + + /* return and cache the header */ + *header = result; + SVN_ERR(svn_cache__set(ffd->l2p_header_cache, &key, result, pool)); + + return SVN_NO_ERROR; +} + +/* Get the page info requested in *BATON from FS and set the output fields + * in *BATON. + * To maximize efficiency, use or return the data stream in *STREAM. + * Use POOL for allocations. + */ +static svn_error_t * +get_l2p_page_info(l2p_page_info_baton_t *baton, + packed_number_stream_t **stream, + svn_fs_t *fs, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + l2p_header_t *result; + svn_boolean_t is_cached = FALSE; + void *dummy = NULL; + + /* try to find the info in the cache */ + pair_cache_key_t key; + key.revision = base_revision(fs, baton->revision); + key.second = svn_fs_x__is_packed_rev(fs, baton->revision); + SVN_ERR(svn_cache__get_partial((void**)&dummy, &is_cached, + ffd->l2p_header_cache, &key, + l2p_header_access_func, baton, + pool)); + if (is_cached) + return SVN_NO_ERROR; + + /* read from disk, cache and copy the result */ + SVN_ERR(get_l2p_header_body(&result, stream, fs, baton->revision, pool)); + SVN_ERR(l2p_header_copy(baton, result, result->page_table, + result->page_table_index)); + + return SVN_NO_ERROR; +} + +/* Read the log-to-phys header info of the index covering REVISION from FS + * and return it in *HEADER. To maximize efficiency, use or return the + * data stream in *STREAM. Use POOL for allocations. + */ +static svn_error_t * +get_l2p_header(l2p_header_t **header, + packed_number_stream_t **stream, + svn_fs_t *fs, + svn_revnum_t revision, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_boolean_t is_cached = FALSE; + + /* first, try cache lookop */ + pair_cache_key_t key; + key.revision = base_revision(fs, revision); + key.second = svn_fs_x__is_packed_rev(fs, revision); + SVN_ERR(svn_cache__get((void**)header, &is_cached, ffd->l2p_header_cache, + &key, pool)); + if (is_cached) + return SVN_NO_ERROR; + + /* read from disk and cache the result */ + SVN_ERR(get_l2p_header_body(header, stream, fs, revision, pool)); + + return SVN_NO_ERROR; +} + +/* From the log-to-phys index file starting at START_REVISION in FS, read + * the mapping page identified by TABLE_ENTRY and return it in *PAGE. + * To maximize efficiency, use or return the data stream in *STREAM. + * Use POOL for allocations. + */ +static svn_error_t * +get_l2p_page(l2p_page_t **page, + packed_number_stream_t **stream, + svn_fs_t *fs, + svn_revnum_t start_revision, + l2p_page_table_entry_t *table_entry, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + apr_uint64_t value, last_value = 0; + apr_uint32_t i; + l2p_page_t *result = apr_pcalloc(pool, sizeof(*result)); + apr_uint64_t container_count; + apr_off_t *container_offsets; + + /* open index file and select page */ + if (*stream == NULL) + SVN_ERR(packed_stream_open(stream, + svn_fs_x__path_l2p_index(fs, start_revision, + pool), + ffd->block_size, + pool)); + + packed_stream_seek(*stream, table_entry->offset); + + /* initialize the page content */ + result->entry_count = table_entry->entry_count; + result->offsets = apr_pcalloc(pool, result->entry_count + * sizeof(*result->offsets)); + result->sub_items = apr_pcalloc(pool, result->entry_count + * sizeof(*result->sub_items)); + + /* container offsets array */ + + SVN_ERR(packed_stream_get(&container_count, *stream)); + container_offsets = apr_pcalloc(pool, container_count * sizeof(*result)); + for (i = 0; i < container_count; ++i) + { + SVN_ERR(packed_stream_get(&value, *stream)); + last_value += value; + container_offsets[i] = (apr_off_t)last_value - 1; + /* '-1' is represented as '0' in the index file */ + } + + /* read all page entries (offsets in rev file and container sub-items) */ + for (i = 0; i < result->entry_count; ++i) + { + SVN_ERR(packed_stream_get(&value, *stream)); + if (value == 0) + { + result->offsets[i] = -1; + result->sub_items[i] = 0; + } + else if (value <= container_count) + { + result->offsets[i] = container_offsets[value - 1]; + SVN_ERR(packed_stream_get(&value, *stream)); + result->sub_items[i] = (apr_uint32_t)value; + } + else + { + result->offsets[i] = (apr_off_t)(value - 1 - container_count); + result->sub_items[i] = 0; + } + } + + *page = result; + + return SVN_NO_ERROR; +} + +/* Request data structure for l2p_page_access_func. + */ +typedef struct l2p_page_baton_t +{ + /* in data */ + /* revision. Used for error messages only */ + svn_revnum_t revision; + + /* item index to look up. Used for error messages only */ + apr_uint64_t item_index; + + /* offset within the cached page */ + apr_uint32_t page_offset; + + /* out data */ + /* absolute item or container offset in rev / pack file */ + apr_off_t offset; + + /* 0 -> container / item itself; sub-item in container otherwise */ + apr_uint32_t sub_item; + +} l2p_page_baton_t; + +/* Return the rev / pack file offset of the item at BATON->PAGE_OFFSET in + * OFFSETS of PAGE and write it to *OFFSET. + */ +static svn_error_t * +l2p_page_get_offset(l2p_page_baton_t *baton, + const l2p_page_t *page, + const apr_off_t *offsets, + const apr_uint32_t *sub_items) +{ + /* overflow check */ + if (page->entry_count <= baton->page_offset) + return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_OVERFLOW , NULL, + _("Item index %" APR_UINT64_T_FMT + " too large in revision %ld"), + baton->item_index, baton->revision); + + /* return the result */ + baton->offset = offsets[baton->page_offset]; + baton->sub_item = sub_items[baton->page_offset]; + + return SVN_NO_ERROR; +} + +/* Implement svn_cache__partial_getter_func_t: copy the data requested in + * l2p_page_baton_t *BATON from l2p_page_t *DATA into apr_off_t *OUT. + */ +static svn_error_t * +l2p_page_access_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *result_pool) +{ + /* resolve all in-cache pointers */ + const l2p_page_t *page = data; + const apr_off_t *offsets + = svn_temp_deserializer__ptr(page, (const void *const *)&page->offsets); + const apr_uint32_t *sub_items + = svn_temp_deserializer__ptr(page, (const void *const *)&page->sub_items); + + /* return the requested data */ + return l2p_page_get_offset(baton, page, offsets, sub_items); +} + +/* Data request structure used by l2p_page_table_access_func. + */ +typedef struct l2p_page_table_baton_t +{ + /* revision for which to read the page table */ + svn_revnum_t revision; + + /* page table entries (of type l2p_page_table_entry_t). + * Must be created by caller and will be filled by callee. */ + apr_array_header_t *pages; +} l2p_page_table_baton_t; + +/* Implement svn_cache__partial_getter_func_t: copy the data requested in + * l2p_page_baton_t *BATON from l2p_page_t *DATA into apr_off_t *OUT. + */ +static svn_error_t * +l2p_page_table_access_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *result_pool) +{ + /* resolve in-cache pointers */ + l2p_page_table_baton_t *table_baton = baton; + const l2p_header_t *header = (const l2p_header_t *)data; + const l2p_page_table_entry_t *page_table + = svn_temp_deserializer__ptr(header, + (const void *const *)&header->page_table); + const apr_size_t *page_table_index + = svn_temp_deserializer__ptr(header, + (const void *const *)&header->page_table_index); + + /* copy the revision's page table into BATON */ + apr_size_t rel_revision = table_baton->revision - header->first_revision; + if (rel_revision < header->revision_count) + { + const l2p_page_table_entry_t *entry + = page_table + page_table_index[rel_revision]; + const l2p_page_table_entry_t *last_entry + = page_table + page_table_index[rel_revision + 1]; + + for (; entry < last_entry; ++entry) + APR_ARRAY_PUSH(table_baton->pages, l2p_page_table_entry_t) + = *entry; + } + + /* set output as a courtesy to the caller */ + *out = table_baton->pages; + + return SVN_NO_ERROR; +} + +/* Read the l2p index page table for REVISION in FS from cache and return + * it in PAGES. The later must be provided by the caller (and can be + * re-used); existing entries will be removed before writing the result. + * If the data cannot be found in the cache, the result will be empty + * (it never can be empty for a valid REVISION if the data is cached). + * Use POOL for temporary allocations. + */ +static svn_error_t * +get_l2p_page_table(apr_array_header_t *pages, + svn_fs_t *fs, + svn_revnum_t revision, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_boolean_t is_cached = FALSE; + l2p_page_table_baton_t baton; + + pair_cache_key_t key; + key.revision = base_revision(fs, revision); + key.second = svn_fs_x__is_packed_rev(fs, revision); + + apr_array_clear(pages); + baton.revision = revision; + baton.pages = pages; + SVN_ERR(svn_cache__get_partial((void**)&pages, &is_cached, + ffd->l2p_header_cache, &key, + l2p_page_table_access_func, &baton, pool)); + + return SVN_NO_ERROR; +} + +/* Utility function. Read the l2p index pages for REVISION in FS from + * STREAM and put them into the cache. Skip page number EXLCUDED_PAGE_NO + * (use -1 for 'skip none') and pages outside the MIN_OFFSET, MAX_OFFSET + * range in the l2p index file. The index is being identified by + * FIRST_REVISION. PAGES is a scratch container provided by the caller. + * SCRATCH_POOL is used for temporary allocations. + */ +static svn_error_t * +prefetch_l2p_pages(svn_boolean_t *end, + svn_fs_t *fs, + packed_number_stream_t *stream, + svn_revnum_t first_revision, + svn_revnum_t revision, + apr_array_header_t *pages, + int exlcuded_page_no, + apr_off_t min_offset, + apr_off_t max_offset, + apr_pool_t *scratch_pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + int i; + apr_pool_t *iterpool; + svn_fs_x__page_cache_key_t key = { 0 }; + + /* get the page table for REVISION from cache */ + *end = FALSE; + SVN_ERR(get_l2p_page_table(pages, fs, revision, scratch_pool)); + if (pages->nelts == 0) + { + /* not found -> we can't continue without hitting the disk again */ + *end = TRUE; + return SVN_NO_ERROR; + } + + /* prefetch pages individually until all are done or we found one in + * the cache */ + iterpool = svn_pool_create(scratch_pool); + assert(revision <= APR_UINT32_MAX); + key.revision = (apr_uint32_t)revision; + key.is_packed = svn_fs_x__is_packed_rev(fs, revision); + + for (i = 0; i < pages->nelts && !*end; ++i) + { + l2p_page_table_entry_t *entry + = &APR_ARRAY_IDX(pages, i, l2p_page_table_entry_t); + if (i == exlcuded_page_no) + continue; + + /* skip pages outside the specified index file range */ + if ( entry->offset < min_offset + || entry->offset + entry->size > max_offset) + { + *end = TRUE; + continue; + } + + /* page already in cache? */ + key.page = i; + SVN_ERR(svn_cache__has_key(end, ffd->l2p_page_cache, + &key, iterpool)); + if (!*end) + { + /* no in cache -> read from stream (data already buffered in APR) + * and cache the result */ + l2p_page_t *page = NULL; + SVN_ERR(get_l2p_page(&page, &stream, fs, first_revision, + entry, iterpool)); + + SVN_ERR(svn_cache__set(ffd->l2p_page_cache, &key, page, + iterpool)); + } + + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Using the log-to-phys indexes in FS, find the absolute offset in the + * rev file for (REVISION, ITEM_INDEX) and return it in *OFFSET. + * Use POOL for allocations. + */ +static svn_error_t * +l2p_index_lookup(apr_off_t *offset, + apr_uint32_t *sub_item, + svn_fs_t *fs, + svn_revnum_t revision, + apr_uint64_t item_index, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + l2p_page_info_baton_t info_baton; + l2p_page_baton_t page_baton; + l2p_page_t *page = NULL; + packed_number_stream_t *stream = NULL; + svn_fs_x__page_cache_key_t key = { 0 }; + svn_boolean_t is_cached = FALSE; + void *dummy = NULL; + + /* read index master data structure and extract the info required to + * access the l2p index page for (REVISION,ITEM_INDEX)*/ + info_baton.revision = revision; + info_baton.item_index = item_index; + SVN_ERR(get_l2p_page_info(&info_baton, &stream, fs, pool)); + + /* try to find the page in the cache and get the OFFSET from it */ + page_baton.revision = revision; + page_baton.item_index = item_index; + page_baton.page_offset = info_baton.page_offset; + + assert(revision <= APR_UINT32_MAX); + key.revision = (apr_uint32_t)revision; + key.is_packed = svn_fs_x__is_packed_rev(fs, revision); + key.page = info_baton.page_no; + + SVN_ERR(svn_cache__get_partial(&dummy, &is_cached, + ffd->l2p_page_cache, &key, + l2p_page_access_func, &page_baton, pool)); + + if (!is_cached) + { + /* we need to read the info from disk (might already be in the + * APR file buffer, though) */ + apr_array_header_t *pages; + svn_revnum_t prefetch_revision; + svn_revnum_t last_revision + = info_baton.first_revision + + (key.is_packed ? ffd->max_files_per_dir : 1); + apr_pool_t *iterpool = svn_pool_create(pool); + svn_boolean_t end; + apr_off_t max_offset + = APR_ALIGN(info_baton.entry.offset + info_baton.entry.size, + 0x10000); + apr_off_t min_offset = max_offset - 0x10000; + + /* read the relevant page */ + SVN_ERR(get_l2p_page(&page, &stream, fs, info_baton.first_revision, + &info_baton.entry, pool)); + + /* cache the page and extract the result we need */ + SVN_ERR(svn_cache__set(ffd->l2p_page_cache, &key, page, pool)); + SVN_ERR(l2p_page_get_offset(&page_baton, page, page->offsets, + page->sub_items)); + + /* prefetch pages from following and preceding revisions */ + pages = apr_array_make(pool, 16, sizeof(l2p_page_table_entry_t)); + end = FALSE; + for (prefetch_revision = revision; + prefetch_revision < last_revision && !end; + ++prefetch_revision) + { + int excluded_page_no = prefetch_revision == revision + ? info_baton.page_no + : -1; + SVN_ERR(prefetch_l2p_pages(&end, fs, stream, + info_baton.first_revision, + prefetch_revision, pages, + excluded_page_no, min_offset, + max_offset, iterpool)); + svn_pool_clear(iterpool); + } + + end = FALSE; + for (prefetch_revision = revision-1; + prefetch_revision >= info_baton.first_revision && !end; + --prefetch_revision) + { + SVN_ERR(prefetch_l2p_pages(&end, fs, stream, + info_baton.first_revision, + prefetch_revision, pages, -1, + min_offset, max_offset, iterpool)); + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + } + + SVN_ERR(packed_stream_close(stream)); + + *offset = page_baton.offset; + *sub_item = page_baton.sub_item; + + return SVN_NO_ERROR; +} + +/* Using the log-to-phys proto index in transaction TXN_ID in FS, find the + * absolute offset in the proto rev file for the given ITEM_INDEX and return + * it in *OFFSET. Use POOL for allocations. + */ +static svn_error_t * +l2p_proto_index_lookup(apr_off_t *offset, + apr_uint32_t *sub_item, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_uint64_t item_index, + apr_pool_t *pool) +{ + svn_boolean_t eof = FALSE; + apr_file_t *file = NULL; + SVN_ERR(svn_io_file_open(&file, + svn_fs_x__path_l2p_proto_index(fs, txn_id, pool), + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool)); + + /* process all entries until we fail due to EOF */ + *offset = -1; + while (!eof) + { + l2p_proto_entry_t entry; + apr_size_t read = 0; + + /* (attempt to) read the next entry from the source */ + SVN_ERR(svn_io_file_read_full2(file, &entry, sizeof(entry), + &read, &eof, pool)); + SVN_ERR_ASSERT(eof || read == sizeof(entry)); + + /* handle new revision */ + if (!eof && entry.item_index == item_index) + { + *offset = (apr_off_t)entry.offset - 1; + *sub_item = (apr_off_t)entry.sub_item; + break; + } + } + + SVN_ERR(svn_io_file_close(file, pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__l2p_get_max_ids(apr_array_header_t **max_ids, + svn_fs_t *fs, + svn_revnum_t start_rev, + apr_size_t count, + apr_pool_t *pool) +{ + l2p_header_t *header = NULL; + svn_revnum_t revision; + svn_revnum_t last_rev = (svn_revnum_t)(start_rev + count); + packed_number_stream_t *stream = NULL; + apr_pool_t *header_pool = svn_pool_create(pool); + + /* read index master data structure for the index covering START_REV */ + SVN_ERR(get_l2p_header(&header, &stream, fs, start_rev, header_pool)); + SVN_ERR(packed_stream_close(stream)); + stream = NULL; + + /* Determine the length of the item index list for each rev. + * Read new index headers as required. */ + *max_ids = apr_array_make(pool, (int)count, sizeof(apr_uint64_t)); + for (revision = start_rev; revision < last_rev; ++revision) + { + apr_uint64_t full_page_count; + apr_uint64_t item_count; + apr_size_t first_page_index, last_page_index; + + if (revision >= header->first_revision + header->revision_count) + { + /* need to read the next index. Clear up memory used for the + * previous one. */ + svn_pool_clear(header_pool); + SVN_ERR(get_l2p_header(&header, &stream, fs, revision, + header_pool)); + SVN_ERR(packed_stream_close(stream)); + stream = NULL; + } + + /* in a revision with N index pages, the first N-1 index pages are + * "full", i.e. contain HEADER->PAGE_SIZE entries */ + first_page_index + = header->page_table_index[revision - header->first_revision]; + last_page_index + = header->page_table_index[revision - header->first_revision + 1]; + full_page_count = last_page_index - first_page_index - 1; + item_count = full_page_count * header->page_size + + header->page_table[last_page_index - 1].entry_count; + + APR_ARRAY_PUSH(*max_ids, apr_uint64_t) = item_count; + } + + svn_pool_destroy(header_pool); + return SVN_NO_ERROR; +} + +/* + * phys-to-log index + */ +svn_fs_x__p2l_entry_t * +svn_fs_x__p2l_entry_dup(svn_fs_x__p2l_entry_t *entry, + apr_pool_t *pool) +{ + svn_fs_x__p2l_entry_t *new_entry = apr_palloc(pool, sizeof(*new_entry)); + *new_entry = *entry; + + if (new_entry->item_count) + new_entry->items = apr_pmemdup(pool, + entry->items, + entry->item_count * sizeof(*entry->items)); + + return new_entry; +} + +svn_error_t * +svn_fs_x__p2l_proto_index_open(apr_file_t **proto_index, + const char *file_name, + apr_pool_t *pool) +{ + SVN_ERR(svn_io_file_open(proto_index, file_name, APR_READ | APR_WRITE + | APR_CREATE | APR_APPEND | APR_BUFFERED, + APR_OS_DEFAULT, pool)); + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__p2l_proto_index_add_entry(apr_file_t *proto_index, + svn_fs_x__p2l_entry_t *entry, + apr_pool_t *pool) +{ + apr_size_t written = sizeof(*entry); + + SVN_ERR(svn_io_file_write_full(proto_index, entry, sizeof(*entry), + &written, pool)); + SVN_ERR_ASSERT(written == sizeof(*entry)); + + if (entry->item_count) + { + written = entry->item_count * sizeof(*entry->items); + SVN_ERR(svn_io_file_write_full(proto_index, entry->items, written, + &written, pool)); + SVN_ERR_ASSERT(written == entry->item_count * sizeof(*entry->items)); + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__p2l_index_create(svn_fs_t *fs, + const char *file_name, + const char *proto_file_name, + svn_revnum_t revision, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + apr_uint64_t page_size = ffd->p2l_page_size; + apr_file_t *proto_index = NULL; + int i; + apr_uint32_t sub_item; + svn_boolean_t eof = FALSE; + apr_file_t *index_file; + unsigned char encoded[ENCODED_INT_LENGTH]; + + apr_uint64_t last_entry_end = 0; + apr_uint64_t last_page_end = 0; + apr_size_t last_buffer_size = 0; /* byte offset in the spill buffer at + the begin of the current revision */ + + /* temporary data structures that collect the data which will be moved + to the target file in a second step */ + apr_pool_t *local_pool = svn_pool_create(pool); + apr_array_header_t *table_sizes + = apr_array_make(local_pool, 16, sizeof(apr_uint64_t)); + + /* 64k blocks, spill after 16MB */ + svn_spillbuf_t *buffer + = svn_spillbuf__create(0x10000, 0x1000000, local_pool); + + /* for loop temps ... */ + apr_pool_t *iter_pool = svn_pool_create(pool); + + /* start at the beginning of the source file */ + SVN_ERR(svn_io_file_open(&proto_index, proto_file_name, + APR_READ | APR_CREATE | APR_BUFFERED, + APR_OS_DEFAULT, pool)); + + /* process all entries until we fail due to EOF */ + while (!eof) + { + svn_fs_x__p2l_entry_t entry; + apr_size_t read = 0; + apr_size_t to_read; + apr_uint64_t entry_end; + svn_boolean_t new_page = svn_spillbuf__get_size(buffer) == 0; + svn_revnum_t last_revision = revision; + apr_uint64_t last_number = 0; + + /* (attempt to) read the next entry from the source */ + SVN_ERR(svn_io_file_read_full2(proto_index, &entry, sizeof(entry), + &read, &eof, iter_pool)); + SVN_ERR_ASSERT(eof || read == sizeof(entry)); + + if (entry.item_count && !eof) + { + to_read = entry.item_count * sizeof(*entry.items); + entry.items = apr_palloc(iter_pool, to_read); + + SVN_ERR(svn_io_file_read_full2(proto_index, entry.items, to_read, + &read, &eof, iter_pool)); + SVN_ERR_ASSERT(eof || read == to_read); + } + + /* "unused" (and usually non-existent) section to cover the offsets + at the end the of the last page. */ + if (eof) + { + entry.offset = last_entry_end; + entry.size = APR_ALIGN(entry.offset, page_size) - entry.offset; + entry.type = 0; + entry.item_count = 0; + entry.items = NULL; + } + + for (sub_item = 0; sub_item < entry.item_count; ++sub_item) + if (entry.items[sub_item].revision == SVN_INVALID_REVNUM) + entry.items[sub_item].revision = revision; + + /* end pages if entry is extending beyond their boundaries */ + entry_end = entry.offset + entry.size; + while (entry_end - last_page_end > page_size) + { + apr_uint64_t buffer_size = svn_spillbuf__get_size(buffer); + APR_ARRAY_PUSH(table_sizes, apr_uint64_t) + = buffer_size - last_buffer_size; + + last_buffer_size = buffer_size; + last_page_end += page_size; + new_page = TRUE; + } + + /* this entry starts a new table -> store its offset + (all following entries in the same table will store sizes only) */ + if (new_page) + { + SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, + encode_uint(encoded, entry.offset), + iter_pool)); + last_revision = revision; + } + + /* write simple item / container entry */ + SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, + encode_uint(encoded, entry.size), + iter_pool)); + SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, + encode_uint(encoded, entry.type + entry.item_count * 16), + iter_pool)); + + /* container contents (only one for non-container items) */ + for (sub_item = 0; sub_item < entry.item_count; ++sub_item) + { + apr_int64_t diff = entry.items[sub_item].revision - last_revision; + SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, + encode_int(encoded, diff), + iter_pool)); + last_revision = entry.items[sub_item].revision; + } + + for (sub_item = 0; sub_item < entry.item_count; ++sub_item) + { + apr_int64_t diff = entry.items[sub_item].number - last_number; + SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, + encode_int(encoded, diff), + iter_pool)); + last_number = entry.items[sub_item].number; + } + + last_entry_end = entry_end; + + svn_pool_clear(iter_pool); + } + + /* store length of last table */ + APR_ARRAY_PUSH(table_sizes, apr_uint64_t) + = svn_spillbuf__get_size(buffer) - last_buffer_size; + + /* create the target file */ + SVN_ERR(svn_io_file_open(&index_file, file_name, APR_WRITE + | APR_CREATE | APR_TRUNCATE | APR_BUFFERED, + APR_OS_DEFAULT, local_pool)); + + /* write the start revision and page size */ + SVN_ERR(svn_io_file_write_full(index_file, encoded, + encode_uint(encoded, revision), + NULL, local_pool)); + SVN_ERR(svn_io_file_write_full(index_file, encoded, + encode_uint(encoded, page_size), + NULL, local_pool)); + + /* write the page table (actually, the sizes of each page description) */ + SVN_ERR(svn_io_file_write_full(index_file, encoded, + encode_uint(encoded, table_sizes->nelts), + NULL, local_pool)); + for (i = 0; i < table_sizes->nelts; ++i) + { + apr_uint64_t value = APR_ARRAY_IDX(table_sizes, i, apr_uint64_t); + SVN_ERR(svn_io_file_write_full(index_file, encoded, + encode_uint(encoded, value), + NULL, local_pool)); + } + + /* append page contents */ + SVN_ERR(svn_stream_copy3(svn_stream__from_spillbuf(buffer, local_pool), + svn_stream_from_aprfile2(index_file, TRUE, + local_pool), + NULL, NULL, local_pool)); + + /* finalize the index file */ + SVN_ERR(svn_io_file_close(index_file, local_pool)); + SVN_ERR(svn_io_set_file_read_only(file_name, FALSE, local_pool)); + + svn_pool_destroy(iter_pool); + svn_pool_destroy(local_pool); + + return SVN_NO_ERROR; +} + +/* Data structure that describes which p2l page info shall be extracted + * from the cache and contains the fields that receive the result. + */ +typedef struct p2l_page_info_baton_t +{ + /* input variables */ + /* revision identifying the index file */ + svn_revnum_t revision; + + /* offset within the page in rev / pack file */ + apr_off_t offset; + + /* output variables */ + /* page containing OFFSET */ + apr_size_t page_no; + + /* first revision in this p2l index */ + svn_revnum_t first_revision; + + /* offset within the p2l index file describing this page */ + apr_off_t start_offset; + + /* offset within the p2l index file describing the following page */ + apr_off_t next_offset; + + /* PAGE_NO * PAGE_SIZE (is <= OFFSET) */ + apr_off_t page_start; + + /* total number of pages indexed */ + apr_size_t page_count; + + /* size of each page in pack / rev file */ + apr_uint64_t page_size; +} p2l_page_info_baton_t; + +/* From HEADER and the list of all OFFSETS, fill BATON with the page info + * requested by BATON->OFFSET. + */ +static void +p2l_page_info_copy(p2l_page_info_baton_t *baton, + const p2l_header_t *header, + const apr_off_t *offsets) +{ + /* if the requested offset is out of bounds, return info for + * a zero-sized empty page right behind the last page. + */ + if (baton->offset / header->page_size < header->page_count) + { + baton->page_no = baton->offset / header->page_size; + baton->start_offset = offsets[baton->page_no]; + baton->next_offset = offsets[baton->page_no + 1]; + baton->page_size = header->page_size; + } + else + { + baton->page_no = header->page_count; + baton->start_offset = offsets[baton->page_no]; + baton->next_offset = offsets[baton->page_no]; + baton->page_size = 0; + } + + baton->first_revision = header->first_revision; + baton->page_start = (apr_off_t)(header->page_size * baton->page_no); + baton->page_count = header->page_count; +} + +/* Implement svn_cache__partial_getter_func_t: extract the p2l page info + * requested by BATON and return it in BATON. + */ +static svn_error_t * +p2l_page_info_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *result_pool) +{ + /* all the pointers to cached data we need */ + const p2l_header_t *header = data; + const apr_off_t *offsets + = svn_temp_deserializer__ptr(header, + (const void *const *)&header->offsets); + + /* copy data from cache to BATON */ + p2l_page_info_copy(baton, header, offsets); + return SVN_NO_ERROR; +} + +/* Read the header data structure of the phys-to-log index for REVISION in + * FS and return it in *HEADER. + * + * To maximize efficiency, use or return the data stream in *STREAM. + * If *STREAM is yet to be constructed, do so in STREAM_POOL. + * Use POOL for allocations. + */ +static svn_error_t * +get_p2l_header(p2l_header_t **header, + packed_number_stream_t **stream, + svn_fs_t *fs, + svn_revnum_t revision, + apr_pool_t *stream_pool, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + apr_uint64_t value; + apr_size_t i; + apr_off_t offset; + p2l_header_t *result; + svn_boolean_t is_cached = FALSE; + + /* look for the header data in our cache */ + pair_cache_key_t key; + key.revision = base_revision(fs, revision); + key.second = svn_fs_x__is_packed_rev(fs, revision); + + SVN_ERR(svn_cache__get((void**)header, &is_cached, ffd->p2l_header_cache, + &key, pool)); + if (is_cached) + return SVN_NO_ERROR; + + /* not found -> must read it from disk. + * Open index file or position read pointer to the begin of the file */ + if (*stream == NULL) + SVN_ERR(packed_stream_open(stream, + svn_fs_x__path_p2l_index(fs, key.revision, + pool), + ffd->block_size, stream_pool)); + else + packed_stream_seek(*stream, 0); + + /* allocate result data structure */ + result = apr_pcalloc(pool, sizeof(*result)); + + /* read table sizes and allocate page array */ + SVN_ERR(packed_stream_get(&value, *stream)); + result->first_revision = (svn_revnum_t)value; + SVN_ERR(packed_stream_get(&value, *stream)); + result->page_size = value; + SVN_ERR(packed_stream_get(&value, *stream)); + result->page_count = (apr_size_t)value; + result->offsets + = apr_pcalloc(pool, (result->page_count + 1) * sizeof(*result->offsets)); + + /* read page sizes and derive page description offsets from them */ + result->offsets[0] = 0; + for (i = 0; i < result->page_count; ++i) + { + SVN_ERR(packed_stream_get(&value, *stream)); + result->offsets[i+1] = result->offsets[i] + (apr_off_t)value; + } + + /* correct the offset values */ + offset = packed_stream_offset(*stream); + for (i = 0; i <= result->page_count; ++i) + result->offsets[i] += offset; + + /* cache the header data */ + SVN_ERR(svn_cache__set(ffd->p2l_header_cache, &key, result, pool)); + + /* return the result */ + *header = result; + + return SVN_NO_ERROR; +} + +/* Read the header data structure of the phys-to-log index for revision + * BATON->REVISION in FS. Return in *BATON all info relevant to read the + * index page for the rev / pack file offset BATON->OFFSET. + * + * To maximize efficiency, use or return the data stream in *STREAM. + * If *STREAM is yet to be constructed, do so in STREAM_POOL. + * Use POOL for allocations. + */ +static svn_error_t * +get_p2l_page_info(p2l_page_info_baton_t *baton, + packed_number_stream_t **stream, + svn_fs_t *fs, + apr_pool_t *stream_pool, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + p2l_header_t *header; + svn_boolean_t is_cached = FALSE; + void *dummy = NULL; + + /* look for the header data in our cache */ + pair_cache_key_t key; + key.revision = base_revision(fs, baton->revision); + key.second = svn_fs_x__is_packed_rev(fs, baton->revision); + + SVN_ERR(svn_cache__get_partial(&dummy, &is_cached, ffd->p2l_header_cache, + &key, p2l_page_info_func, baton, pool)); + if (is_cached) + return SVN_NO_ERROR; + + SVN_ERR(get_p2l_header(&header, stream, fs, baton->revision, + stream_pool, pool)); + + /* copy the requested info into *BATON */ + p2l_page_info_copy(baton, header, header->offsets); + + return SVN_NO_ERROR; +} + +/* Read a mapping entry from the phys-to-log index STREAM and append it to + * RESULT. *ITEM_INDEX contains the phys offset for the entry and will + * be moved forward by the size of entry. Use POOL for allocations. + */ +static svn_error_t * +read_entry(packed_number_stream_t *stream, + apr_off_t *item_offset, + svn_revnum_t revision, + apr_array_header_t *result, + apr_pool_t *pool) +{ + apr_uint64_t value; + apr_uint64_t number = 0; + apr_uint32_t sub_item; + + svn_fs_x__p2l_entry_t entry; + + entry.offset = *item_offset; + SVN_ERR(packed_stream_get(&value, stream)); + entry.size = (apr_off_t)value; + SVN_ERR(packed_stream_get(&value, stream)); + entry.type = (int)value % 16; + entry.item_count = (apr_uint32_t)(value / 16); + + if (entry.item_count == 0) + { + entry.items = NULL; + } + else + { + entry.items + = apr_pcalloc(pool, entry.item_count * sizeof(*entry.items)); + + for (sub_item = 0; sub_item < entry.item_count; ++sub_item) + { + SVN_ERR(packed_stream_get(&value, stream)); + revision += (svn_revnum_t)(value % 2 ? -1 - value / 2 : value / 2); + entry.items[sub_item].revision = revision; + } + + for (sub_item = 0; sub_item < entry.item_count; ++sub_item) + { + SVN_ERR(packed_stream_get(&value, stream)); + number += value % 2 ? -1 - value / 2 : value / 2; + entry.items[sub_item].number = number; + } + } + + APR_ARRAY_PUSH(result, svn_fs_x__p2l_entry_t) = entry; + *item_offset += entry.size; + + return SVN_NO_ERROR; +} + +/* Read the phys-to-log mappings for the cluster beginning at rev file + * offset PAGE_START from the index for START_REVISION in FS. The data + * can be found in the index page beginning at START_OFFSET with the next + * page beginning at NEXT_OFFSET. Return the relevant index entries in + * *ENTRIES. To maximize efficiency, use or return the data stream in + * STREAM. If the latter is yet to be constructed, do so in STREAM_POOL. + * Use POOL for other allocations. + */ +static svn_error_t * +get_p2l_page(apr_array_header_t **entries, + packed_number_stream_t **stream, + svn_fs_t *fs, + svn_revnum_t start_revision, + apr_off_t start_offset, + apr_off_t next_offset, + apr_off_t page_start, + apr_uint64_t page_size, + apr_pool_t *stream_pool, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + apr_uint64_t value; + apr_array_header_t *result + = apr_array_make(pool, 16, sizeof(svn_fs_x__p2l_entry_t)); + apr_off_t item_offset; + apr_off_t offset; + + /* open index and navigate to page start */ + if (*stream == NULL) + SVN_ERR(packed_stream_open(stream, + svn_fs_x__path_p2l_index(fs, start_revision, pool), + ffd->block_size, stream_pool)); + packed_stream_seek(*stream, start_offset); + + /* read rev file offset of the first page entry (all page entries will + * only store their sizes). */ + SVN_ERR(packed_stream_get(&value, *stream)); + item_offset = (apr_off_t)value; + + /* read all entries of this page */ + do + { + SVN_ERR(read_entry(*stream, &item_offset, start_revision, result, + pool)); + offset = packed_stream_offset(*stream); + } + while (offset < next_offset); + + /* if we haven't covered the cluster end yet, we must read the first + * entry of the next page */ + if (item_offset < page_start + page_size) + { + SVN_ERR(packed_stream_get(&value, *stream)); + item_offset = (apr_off_t)value; + SVN_ERR(read_entry(*stream, &item_offset, start_revision, result, + pool)); + } + + *entries = result; + + return SVN_NO_ERROR; +} + +/* If it cannot be found in FS's caches, read the p2l index page selected + * by BATON->OFFSET from *STREAM. If the latter is yet to be constructed, + * do so in STREAM_POOL. Don't read the page if it precedes MIN_OFFSET. + * Set *END to TRUE if the caller should stop refeching. + * + * *BATON will be updated with the selected page's info and SCRATCH_POOL + * will be used for temporary allocations. If the data is alread in the + * cache, descrease *LEAKING_BUCKET and increase it otherwise. With that + * pattern we will still read all pages from the block even if some of + * them survived in the cached. + */ +static svn_error_t * +prefetch_p2l_page(svn_boolean_t *end, + int *leaking_bucket, + svn_fs_t *fs, + packed_number_stream_t **stream, + p2l_page_info_baton_t *baton, + apr_off_t min_offset, + apr_pool_t *stream_pool, + apr_pool_t *scratch_pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_boolean_t already_cached; + apr_array_header_t *page; + svn_fs_x__page_cache_key_t key = { 0 }; + + /* fetch the page info */ + *end = FALSE; + baton->revision = baton->first_revision; + SVN_ERR(get_p2l_page_info(baton, stream, fs, stream_pool, scratch_pool)); + if (baton->start_offset < min_offset) + { + /* page outside limits -> stop prefetching */ + *end = TRUE; + return SVN_NO_ERROR; + } + + /* do we have that page in our caches already? */ + assert(baton->first_revision <= APR_UINT32_MAX); + key.revision = (apr_uint32_t)baton->first_revision; + key.is_packed = svn_fs_x__is_packed_rev(fs, baton->first_revision); + key.page = baton->page_no; + SVN_ERR(svn_cache__has_key(&already_cached, ffd->p2l_page_cache, + &key, scratch_pool)); + + /* yes, already cached */ + if (already_cached) + { + /* stop prefetching if most pages are already cached. */ + if (!--*leaking_bucket) + *end = TRUE; + + return SVN_NO_ERROR; + } + + ++*leaking_bucket; + + /* read from disk */ + SVN_ERR(get_p2l_page(&page, stream, fs, + baton->first_revision, + baton->start_offset, + baton->next_offset, + baton->page_start, + baton->page_size, + stream_pool, + scratch_pool)); + + /* and put it into our cache */ + SVN_ERR(svn_cache__set(ffd->p2l_page_cache, &key, page, scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Lookup & construct the baton and key information that we will need for + * a P2L page cache lookup. We want the page covering OFFSET in the rev / + * pack file containing REVSION in FS. Return the results in *PAGE_INFO_P + * and *KEY_P. Read data through the auto-allocated *STREAM. + * Use POOL for allocations. + */ +static svn_error_t * +get_p2l_keys(p2l_page_info_baton_t *page_info_p, + svn_fs_x__page_cache_key_t *key_p, + packed_number_stream_t **stream, + svn_fs_t *fs, + svn_revnum_t revision, + apr_off_t offset, + apr_pool_t *pool) +{ + p2l_page_info_baton_t page_info; + + /* request info for the index pages that describes the pack / rev file + * contents at pack / rev file position OFFSET. */ + page_info.offset = offset; + page_info.revision = revision; + SVN_ERR(get_p2l_page_info(&page_info, stream, fs, pool, pool)); + + /* if the offset refers to a non-existent page, bail out */ + if (page_info.page_count <= page_info.page_no) + { + SVN_ERR(packed_stream_close(*stream)); + return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_OVERFLOW , NULL, + _("Offset %s too large in revision %ld"), + apr_off_t_toa(pool, offset), revision); + } + + /* return results */ + if (page_info_p) + *page_info_p = page_info; + + /* construct cache key */ + if (key_p) + { + svn_fs_x__page_cache_key_t key = { 0 }; + assert(page_info.first_revision <= APR_UINT32_MAX); + key.revision = (apr_uint32_t)page_info.first_revision; + key.is_packed = svn_fs_x__is_packed_rev(fs, revision); + key.page = page_info.page_no; + + *key_p = key; + } + + return SVN_NO_ERROR; +} + +/* Body of svn_fs_x__p2l_index_lookup. Use / autoconstruct *STREAM as + * your input based on REVISION. + */ +static svn_error_t * +p2l_index_lookup(apr_array_header_t **entries, + packed_number_stream_t **stream, + svn_fs_t *fs, + svn_revnum_t revision, + apr_off_t offset, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__page_cache_key_t key; + svn_boolean_t is_cached = FALSE; + p2l_page_info_baton_t page_info; + + /* look for this page in our cache */ + SVN_ERR(get_p2l_keys(&page_info, &key, stream, fs, revision, offset, + pool)); + SVN_ERR(svn_cache__get((void**)entries, &is_cached, ffd->p2l_page_cache, + &key, pool)); + if (!is_cached) + { + svn_boolean_t end; + apr_pool_t *iterpool = svn_pool_create(pool); + apr_off_t original_page_start = page_info.page_start; + int leaking_bucket = 4; + p2l_page_info_baton_t prefetch_info = page_info; + + apr_off_t max_offset + = APR_ALIGN(page_info.next_offset, ffd->block_size); + apr_off_t min_offset + = APR_ALIGN(page_info.start_offset, ffd->block_size) - ffd->block_size; + + /* Since we read index data in larger chunks, we probably got more + * page data than we requested. Parse & cache that until either we + * encounter pages already cached or reach the end of the buffer. + */ + + /* pre-fetch preceding pages */ + end = FALSE; + prefetch_info.offset = original_page_start; + while (prefetch_info.offset >= prefetch_info.page_size && !end) + { + prefetch_info.offset -= prefetch_info.page_size; + SVN_ERR(prefetch_p2l_page(&end, &leaking_bucket, fs, stream, + &prefetch_info, min_offset, + pool, iterpool)); + svn_pool_clear(iterpool); + } + + /* fetch page from disk and put it into the cache */ + SVN_ERR(get_p2l_page(entries, stream, fs, + page_info.first_revision, + page_info.start_offset, + page_info.next_offset, + page_info.page_start, + page_info.page_size, pool, pool)); + + SVN_ERR(svn_cache__set(ffd->p2l_page_cache, &key, *entries, pool)); + + /* pre-fetch following pages */ + end = FALSE; + leaking_bucket = 4; + prefetch_info = page_info; + prefetch_info.offset = original_page_start; + while ( prefetch_info.next_offset < max_offset + && prefetch_info.page_no + 1 < prefetch_info.page_count + && !end) + { + prefetch_info.offset += prefetch_info.page_size; + SVN_ERR(prefetch_p2l_page(&end, &leaking_bucket, fs, stream, + &prefetch_info, min_offset, + pool, iterpool)); + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__p2l_index_lookup(apr_array_header_t **entries, + svn_fs_t *fs, + svn_revnum_t revision, + apr_off_t offset, + apr_pool_t *pool) +{ + packed_number_stream_t *stream = NULL; + + /* look for this page in our cache */ + SVN_ERR(p2l_index_lookup(entries, &stream, fs, revision, offset, pool)); + + /* make sure we close files after usage */ + SVN_ERR(packed_stream_close(stream)); + + return SVN_NO_ERROR; +} + +/* compare_fn_t comparing a svn_fs_x__p2l_entry_t at LHS with an offset + * RHS. + */ +static int +compare_p2l_entry_offsets(const void *lhs, const void *rhs) +{ + const svn_fs_x__p2l_entry_t *entry = (const svn_fs_x__p2l_entry_t *)lhs; + apr_off_t offset = *(const apr_off_t *)rhs; + + return entry->offset < offset ? -1 : (entry->offset == offset ? 0 : 1); +} + +/* Cached data extraction utility. DATA is a P2L index page, e.g. an APR + * array of svn_fs_x__p2l_entry_t elements. Return the entry for the item + * starting at OFFSET or NULL if that's not an the start offset of any item. + */ +static svn_fs_x__p2l_entry_t * +get_p2l_entry_from_cached_page(const void *data, + apr_off_t offset, + apr_pool_t *pool) +{ + /* resolve all pointer values of in-cache data */ + const apr_array_header_t *page = data; + apr_array_header_t *entries = apr_pmemdup(pool, page, sizeof(*page)); + int idx; + + entries->elts = (char *)svn_temp_deserializer__ptr(page, + (const void *const *)&page->elts); + + /* search of the offset we want */ + idx = svn_sort__bsearch_lower_bound(&offset, entries, + (int (*)(const void *, const void *))compare_p2l_entry_offsets); + + /* return it, if it is a perfect match */ + if (idx < entries->nelts) + { + svn_fs_x__p2l_entry_t *entry + = &APR_ARRAY_IDX(entries, idx, svn_fs_x__p2l_entry_t); + if (entry->offset == offset) + { + svn_fs_x__p2l_entry_t *result + = apr_pmemdup(pool, entry, sizeof(*result)); + result->items + = (svn_fs_x__id_part_t *)svn_temp_deserializer__ptr(entries->elts, + (const void *const *)&entry->items); + return result; + } + } + + return NULL; +} + +/* Implements svn_cache__partial_getter_func_t for P2L index pages, copying + * the entry for the apr_off_t at BATON into *OUT. *OUT will be NULL if + * there is no matching entry in the index page at DATA. + */ +static svn_error_t * +p2l_entry_lookup_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *result_pool) +{ + svn_fs_x__p2l_entry_t *entry + = get_p2l_entry_from_cached_page(data, *(apr_off_t *)baton, result_pool); + + *out = entry && entry->offset == *(apr_off_t *)baton + ? svn_fs_x__p2l_entry_dup(entry, result_pool) + : NULL; + + return SVN_NO_ERROR; +} + +static svn_error_t * +p2l_entry_lookup(svn_fs_x__p2l_entry_t **entry_p, + packed_number_stream_t **stream, + svn_fs_t *fs, + svn_revnum_t revision, + apr_off_t offset, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__page_cache_key_t key = { 0 }; + svn_boolean_t is_cached = FALSE; + p2l_page_info_baton_t page_info; + + *entry_p = NULL; + + /* look for this info in our cache */ + SVN_ERR(get_p2l_keys(&page_info, &key, stream, fs, revision, offset, pool)); + SVN_ERR(svn_cache__get_partial((void**)entry_p, &is_cached, + ffd->p2l_page_cache, &key, + p2l_entry_lookup_func, &offset, pool)); + if (!is_cached) + { + int idx; + + /* do a standard index lookup. This is will automatically prefetch + * data to speed up future lookups. */ + apr_array_header_t *entries; + SVN_ERR(p2l_index_lookup(&entries, stream, fs, revision, offset, pool)); + + /* Find the entry that we want. */ + idx = svn_sort__bsearch_lower_bound(&offset, entries, + (int (*)(const void *, const void *))compare_p2l_entry_offsets); + + /* return it, if it is a perfect match */ + if (idx < entries->nelts) + { + svn_fs_x__p2l_entry_t *entry + = &APR_ARRAY_IDX(entries, idx, svn_fs_x__p2l_entry_t); + if (entry->offset == offset) + *entry_p = entry; + } + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__p2l_entry_lookup(svn_fs_x__p2l_entry_t **entry_p, + svn_fs_t *fs, + svn_revnum_t revision, + apr_off_t offset, + apr_pool_t *pool) +{ + packed_number_stream_t *stream = NULL; + + /* look for this info in our cache */ + SVN_ERR(p2l_entry_lookup(entry_p, &stream, fs, revision, offset, pool)); + + /* make sure we close files after usage */ + SVN_ERR(packed_stream_close(stream)); + + return SVN_NO_ERROR; +} + +/* Baton structure for p2l_item_lookup_func. It describes which sub_item + * info shall be returned. + */ +typedef struct p2l_item_lookup_baton_t +{ + /* file offset to find the P2L index entry for */ + apr_off_t offset; + + /* return the sub-item at this position within that entry */ + apr_uint32_t sub_item; +} p2l_item_lookup_baton_t; + +/* Implements svn_cache__partial_getter_func_t for P2L index pages, copying + * the svn_fs_x__id_part_t for the item described 2l_item_lookup_baton_t + * *BATON. *OUT will be NULL if there is no matching index entry or the + * sub-item is out of range. + */ +static svn_error_t * +p2l_item_lookup_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *result_pool) +{ + p2l_item_lookup_baton_t *lookup_baton = baton; + svn_fs_x__p2l_entry_t *entry + = get_p2l_entry_from_cached_page(data, lookup_baton->offset, result_pool); + + *out = entry + && entry->offset == lookup_baton->offset + && entry->item_count > lookup_baton->sub_item + ? apr_pmemdup(result_pool, + entry->items + lookup_baton->sub_item, + sizeof(*entry->items)) + : NULL; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__p2l_item_lookup(svn_fs_x__id_part_t **item, + svn_fs_t *fs, + svn_revnum_t revision, + apr_off_t offset, + apr_uint32_t sub_item, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + packed_number_stream_t *stream = NULL; + svn_fs_x__page_cache_key_t key = { 0 }; + svn_boolean_t is_cached = FALSE; + p2l_page_info_baton_t page_info; + p2l_item_lookup_baton_t baton; + + *item = NULL; + + /* look for this info in our cache */ + SVN_ERR(get_p2l_keys(&page_info, &key, &stream, fs, revision, offset, + pool)); + baton.offset = offset; + baton.sub_item = sub_item; + SVN_ERR(svn_cache__get_partial((void**)item, &is_cached, + ffd->p2l_page_cache, &key, + p2l_item_lookup_func, &baton, pool)); + if (!is_cached) + { + /* do a standard index lookup. This is will automatically prefetch + * data to speed up future lookups. */ + svn_fs_x__p2l_entry_t *entry; + SVN_ERR(p2l_entry_lookup(&entry, &stream, fs, revision, offset, pool)); + + /* return result */ + if (entry && entry->item_count > sub_item) + *item = apr_pmemdup(pool, entry->items + sub_item, sizeof(**item)); + } + + /* make sure we close files after usage */ + SVN_ERR(packed_stream_close(stream)); + + return SVN_NO_ERROR; +} + +/* Implements svn_cache__partial_getter_func_t for P2L headers, setting *OUT + * to the largest the first offset not covered by this P2L index. + */ +static svn_error_t * +p2l_get_max_offset_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *result_pool) +{ + const p2l_header_t *header = data; + apr_off_t max_offset = header->page_size * header->page_count; + *out = apr_pmemdup(result_pool, &max_offset, sizeof(max_offset)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__p2l_get_max_offset(apr_off_t *offset, + svn_fs_t *fs, + svn_revnum_t revision, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + packed_number_stream_t *stream = NULL; + p2l_header_t *header; + svn_boolean_t is_cached = FALSE; + apr_off_t *offset_p; + + /* look for the header data in our cache */ + pair_cache_key_t key; + key.revision = base_revision(fs, revision); + key.second = svn_fs_x__is_packed_rev(fs, revision); + + SVN_ERR(svn_cache__get_partial((void **)&offset_p, &is_cached, + ffd->p2l_header_cache, &key, + p2l_get_max_offset_func, NULL, pool)); + if (is_cached) + { + *offset = *offset_p; + return SVN_NO_ERROR; + } + + SVN_ERR(get_p2l_header(&header, &stream, fs, revision, pool, pool)); + *offset = header->page_count * header->page_size; + + /* make sure we close files after usage */ + SVN_ERR(packed_stream_close(stream)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__item_offset(apr_off_t *offset, + apr_uint32_t *sub_item, + svn_fs_t *fs, + svn_revnum_t revision, + const svn_fs_x__id_part_t *txn_id, + apr_uint64_t item_index, + apr_pool_t *pool) +{ + if (txn_id) + SVN_ERR(l2p_proto_index_lookup(offset, sub_item, + fs, txn_id, item_index, pool)); + else + SVN_ERR(l2p_index_lookup(offset, sub_item, + fs, revision, item_index, pool)); + + return SVN_NO_ERROR; +} + +/* + * Standard (de-)serialization functions + */ + +svn_error_t * +svn_fs_x__serialize_l2p_header(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + l2p_header_t *header = in; + svn_temp_serializer__context_t *context; + svn_stringbuf_t *serialized; + apr_size_t page_count = header->page_table_index[header->revision_count]; + apr_size_t page_table_size = page_count * sizeof(*header->page_table); + apr_size_t index_size + = (header->revision_count + 1) * sizeof(*header->page_table_index); + apr_size_t data_size = sizeof(*header) + index_size + page_table_size; + + /* serialize header and all its elements */ + context = svn_temp_serializer__init(header, + sizeof(*header), + data_size + 32, + pool); + + /* page table index array */ + svn_temp_serializer__add_leaf(context, + (const void * const *)&header->page_table_index, + index_size); + + /* page table array */ + svn_temp_serializer__add_leaf(context, + (const void * const *)&header->page_table, + page_table_size); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_l2p_header(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + l2p_header_t *header = (l2p_header_t *)data; + + /* resolve the pointers in the struct */ + svn_temp_deserializer__resolve(header, (void**)&header->page_table_index); + svn_temp_deserializer__resolve(header, (void**)&header->page_table); + + /* done */ + *out = header; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__serialize_l2p_page(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + l2p_page_t *page = in; + svn_temp_serializer__context_t *context; + svn_stringbuf_t *serialized; + apr_size_t of_table_size = page->entry_count * sizeof(*page->offsets); + apr_size_t si_table_size = page->entry_count * sizeof(*page->sub_items); + + /* serialize struct and all its elements */ + context = svn_temp_serializer__init(page, + sizeof(*page), + of_table_size + si_table_size + + sizeof(*page) + 32, + pool); + + /* offsets and sub_items arrays */ + svn_temp_serializer__add_leaf(context, + (const void * const *)&page->offsets, + of_table_size); + svn_temp_serializer__add_leaf(context, + (const void * const *)&page->sub_items, + si_table_size); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_l2p_page(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + l2p_page_t *page = data; + + /* resolve the pointers in the struct */ + svn_temp_deserializer__resolve(page, (void**)&page->offsets); + svn_temp_deserializer__resolve(page, (void**)&page->sub_items); + + /* done */ + *out = page; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__serialize_p2l_header(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + p2l_header_t *header = in; + svn_temp_serializer__context_t *context; + svn_stringbuf_t *serialized; + apr_size_t table_size = (header->page_count + 1) * sizeof(*header->offsets); + + /* serialize header and all its elements */ + context = svn_temp_serializer__init(header, + sizeof(*header), + table_size + sizeof(*header) + 32, + pool); + + /* offsets array */ + svn_temp_serializer__add_leaf(context, + (const void * const *)&header->offsets, + table_size); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_p2l_header(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + p2l_header_t *header = data; + + /* resolve the only pointer in the struct */ + svn_temp_deserializer__resolve(header, (void**)&header->offsets); + + /* done */ + *out = header; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__serialize_p2l_page(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + apr_array_header_t *page = in; + svn_temp_serializer__context_t *context; + svn_stringbuf_t *serialized; + apr_size_t table_size = page->elt_size * page->nelts; + svn_fs_x__p2l_entry_t *entries = (svn_fs_x__p2l_entry_t *)page->elts; + int i; + + /* serialize array header and all its elements */ + context = svn_temp_serializer__init(page, + sizeof(*page), + table_size + sizeof(*page) + 32, + pool); + + /* items in the array */ + svn_temp_serializer__push(context, + (const void * const *)&page->elts, + table_size); + + for (i = 0; i < page->nelts; ++i) + svn_temp_serializer__add_leaf(context, + (const void * const *)&entries[i].items, + entries[i].item_count + * sizeof(*entries[i].items)); + + svn_temp_serializer__pop(context); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_p2l_page(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + apr_array_header_t *page = (apr_array_header_t *)data; + svn_fs_x__p2l_entry_t *entries; + int i; + + /* resolve the only pointer in the struct */ + svn_temp_deserializer__resolve(page, (void**)&page->elts); + + /* resolve sub-struct pointers*/ + entries = (svn_fs_x__p2l_entry_t *)page->elts; + for (i = 0; i < page->nelts; ++i) + svn_temp_deserializer__resolve(entries, (void**)&entries[i].items); + + /* patch up members */ + page->pool = pool; + page->nalloc = page->nelts; + + /* done */ + *out = page; + + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_fs_x/index.h b/subversion/libsvn_fs_x/index.h new file mode 100644 index 0000000..75a39da --- /dev/null +++ b/subversion/libsvn_fs_x/index.h
@@ -0,0 +1,337 @@ +/* index.h : interface to FSX indexing functionality + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS__INDEX_H +#define SVN_LIBSVN_FS__INDEX_H + +#include "fs.h" + +/* Per-defined item index values. They are used to identify empty or + * mandatory items. + */ +#define SVN_FS_X__ITEM_INDEX_UNUSED 0 /* invalid / reserved value */ +#define SVN_FS_X__ITEM_INDEX_CHANGES 1 /* list of changed paths */ +#define SVN_FS_X__ITEM_INDEX_ROOT_NODE 2 /* the root noderev */ +#define SVN_FS_X__ITEM_INDEX_FIRST_USER 3 /* first noderev to be freely + assigned */ + +/* Data / item types as stored in the phys-to-log index. + */ +#define SVN_FS_X__ITEM_TYPE_UNUSED 0 /* file section not used */ +#define SVN_FS_X__ITEM_TYPE_FILE_REP 1 /* item is a file representation */ +#define SVN_FS_X__ITEM_TYPE_DIR_REP 2 /* item is a directory rep. */ +#define SVN_FS_X__ITEM_TYPE_FILE_PROPS 3 /* item is a file property rep. */ +#define SVN_FS_X__ITEM_TYPE_DIR_PROPS 4 /* item is a directory prop rep */ +#define SVN_FS_X__ITEM_TYPE_NODEREV 5 /* item is a noderev */ +#define SVN_FS_X__ITEM_TYPE_CHANGES 6 /* item is a changed paths list */ + +#define SVN_FS_X__ITEM_TYPE_ANY_REP 7 /* item is any representation. + Only used in pre-format7. */ + +#define SVN_FS_X__ITEM_TYPE_CHANGES_CONT 8 /* item is a changes container */ +#define SVN_FS_X__ITEM_TYPE_NODEREVS_CONT 9 /* item is a noderevs container */ +#define SVN_FS_X__ITEM_TYPE_REPS_CONT 10 /* item is a representations + container */ + +/* (user visible) entry in the phys-to-log index. It describes a section + * of some packed / non-packed rev file as containing a specific item. + * There must be no overlapping / conflicting entries. + */ +typedef struct svn_fs_x__p2l_entry_t +{ + /* offset of the first byte that belongs to the item */ + apr_off_t offset; + + /* length of the item in bytes */ + apr_off_t size; + + /* type of the item (see SVN_FS_X__ITEM_TYPE_*) defines */ + unsigned type; + + /* Number of items in this block / container. Their list can be found + * in *ITEMS. 0 for unused sections. 1 for non-container items, + * > 1 for containers. */ + apr_uint32_t item_count; + + /* List of items in that block / container */ + svn_fs_x__id_part_t *items; +} svn_fs_x__p2l_entry_t; + +/* Return a (deep) copy of ENTRY, allocated in POOL. + */ +svn_fs_x__p2l_entry_t * +svn_fs_x__p2l_entry_dup(svn_fs_x__p2l_entry_t *entry, + apr_pool_t *pool); + +/* Open / create a log-to-phys index file with the full file path name + * FILE_NAME. Return the open file in *PROTO_INDEX and use POOL for + * allocations. + */ +svn_error_t * +svn_fs_x__l2p_proto_index_open(apr_file_t **proto_index, + const char *file_name, + apr_pool_t *pool); + +/* Call this function before adding entries for the next revision to the + * log-to-phys index file in PROTO_INDEX. Use POOL for allocations. + */ +svn_error_t * +svn_fs_x__l2p_proto_index_add_revision(apr_file_t *proto_index, + apr_pool_t *pool); + +/* Add a new mapping, ITEM_INDEX to the (OFFSET, SUB_ITEM) pair, to log-to- + * phys index file in PROTO_INDEX. Please note that mappings may be added + * in any order but duplicate entries for the same ITEM_INDEX, SUB_ITEM + * are not supported. Not all possible index values need to be used. + * (OFFSET, SUB_ITEM) may be (-1, 0) to mark 'invalid' item indexes but + * that is already implied for all item indexes not explicitly given a + * mapping. + * + * Use POOL for allocations. + */ +svn_error_t * +svn_fs_x__l2p_proto_index_add_entry(apr_file_t *proto_index, + apr_off_t offset, + apr_uint32_t sub_item, + apr_uint64_t item_index, + apr_pool_t *pool); + +/* Use the proto index file stored at PROTO_FILE_NAME and construct the + * final log-to-phys index file at FILE_NAME. The first revision will + * be REVISION, entries to the next revision will be assigned to REVISION+1 + * and so forth. Use POOL for allocations. + */ +svn_error_t * +svn_fs_x__l2p_index_create(svn_fs_t *fs, + const char *file_name, + const char *proto_file_name, + svn_revnum_t revision, + apr_pool_t *pool); + +/* Open / create a phys-to-log index file with the full file path name + * FILE_NAME. Return the open file in *PROTO_INDEX and use POOL for + * allocations. + */ +svn_error_t * +svn_fs_x__p2l_proto_index_open(apr_file_t **proto_index, + const char *file_name, + apr_pool_t *pool); + +/* Add a new mapping ENTRY to the phys-to-log index file in PROTO_INDEX. + * The entries must be added in ascending offset order and must not leave + * intermittent ranges uncovered. The revision value in ENTRY may be + * SVN_INVALID_REVISION. Use POOL for allocations. + */ +svn_error_t * +svn_fs_x__p2l_proto_index_add_entry(apr_file_t *proto_index, + svn_fs_x__p2l_entry_t *entry, + apr_pool_t *pool); + +/* Use the proto index file stored at PROTO_FILE_NAME and construct the + * final phys-to-log index file at FILE_NAME. Entries without a valid + * revision will be assigned to the REVISION given here. + * Use POOL for allocations. + */ +svn_error_t * +svn_fs_x__p2l_index_create(svn_fs_t *fs, + const char *file_name, + const char *proto_file_name, + svn_revnum_t revision, + apr_pool_t *pool); + +/* Use the phys-to-log mapping files in FS to build a list of entries + * that (partly) share in the same cluster as the item at global OFFSET + * in the rep file containing REVISION. Return the array in *ENTRIES, + * elements being of type svn_fs_x__p2l_entry_t. + * Use POOL for allocations. + * + * Note that (only) the first and the last mapping may cross a cluster + * boundary. + */ +svn_error_t * +svn_fs_x__p2l_index_lookup(apr_array_header_t **entries, + svn_fs_t *fs, + svn_revnum_t revision, + apr_off_t offset, + apr_pool_t *pool); + +/* Use the phys-to-log mapping files in FS to return the entry for the + * container or single item starting at global OFFSET in the rep file + * containing REVISION in *ENTRY. Sets *ENTRY to NULL if no item starts + * at exactly that offset. Use POOL for allocations. + */ +svn_error_t * +svn_fs_x__p2l_entry_lookup(svn_fs_x__p2l_entry_t **entry, + svn_fs_t *fs, + svn_revnum_t revision, + apr_off_t offset, + apr_pool_t *pool); + +/* Use the phys-to-log mapping files in FS to return the svn_fs_x__id_part_t + * for the SUB_ITEM of the container starting at global OFFSET in the rep / + * pack file containing REVISION in *ITEM. Sets *ITEM to NULL if no element + * starts at exactly that offset or if it contains no more than SUB_ITEM + * sub-items. Use POOL for allocations. + */ +svn_error_t * +svn_fs_x__p2l_item_lookup(svn_fs_x__id_part_t **item, + svn_fs_t *fs, + svn_revnum_t revision, + apr_off_t offset, + apr_uint32_t sub_item, + apr_pool_t *pool); + +/* Use the log-to-phys mapping files in FS to find the packed / non-packed / + * proto-rev file offset and container sub-item of either (REVISION, + * ITEM_INDEX) or (TXN_ID, ITEM_INDEX). *SUB_ITEM will be 0 for non- + * container items. For committed revision, TXN_ID must be NULL. For + * format 6 and older repositories, we simply map the revision local offset + * given as ITEM_INDEX to the actual file offset (when packed). + * Use POOL for allocations. + */ +svn_error_t * +svn_fs_x__item_offset(apr_off_t *offset, + apr_uint32_t *sub_item, + svn_fs_t *fs, + svn_revnum_t revision, + const svn_fs_x__id_part_t *txn_id, + apr_uint64_t item_index, + apr_pool_t *pool); + +/* Use the log-to-phys indexes in FS to determine the maximum item indexes + * assigned to revision START_REV to START_REV + COUNT - 1. That is a + * close upper limit to the actual number of items in the respective revs. + * Return the results in *MAX_IDS, allocated in POOL. + */ +svn_error_t * +svn_fs_x__l2p_get_max_ids(apr_array_header_t **max_ids, + svn_fs_t *fs, + svn_revnum_t start_rev, + apr_size_t count, + apr_pool_t *pool); + +/* In *OFFSET, return the first OFFSET in the pack / rev file containing + * REVISION in FS not covered by the log-to-phys index. + * Use POOL for allocations. + */ +svn_error_t * +svn_fs_x__p2l_get_max_offset(apr_off_t *offset, + svn_fs_t *fs, + svn_revnum_t revision, + apr_pool_t *pool); + +/* Serialization and caching interface + */ + +/* We use this key type to address individual pages from both index types. + */ +typedef struct svn_fs_x__page_cache_key_t +{ + /* in l2p: this is the revision of the items being mapped + in p2l: this is the start revision identifying the pack / rev file */ + apr_uint32_t revision; + + /* if TRUE, this is the index to a pack file + */ + svn_boolean_t is_packed; + + /* in l2p: page number within the revision + * in p2l: page number with the rev / pack file + */ + apr_uint64_t page; +} svn_fs_x__page_cache_key_t; + +/* + * Implements svn_cache__serialize_func_t for l2p_header_t objects. + */ +svn_error_t * +svn_fs_x__serialize_l2p_header(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/* + * Implements svn_cache__deserialize_func_t for l2p_header_t objects. + */ +svn_error_t * +svn_fs_x__deserialize_l2p_header(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/* + * Implements svn_cache__serialize_func_t for l2p_page_t objects. + */ +svn_error_t * +svn_fs_x__serialize_l2p_page(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/* + * Implements svn_cache__deserialize_func_t for l2p_page_t objects. + */ +svn_error_t * +svn_fs_x__deserialize_l2p_page(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/* + * Implements svn_cache__serialize_func_t for p2l_header_t objects. + */ +svn_error_t * +svn_fs_x__serialize_p2l_header(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/* + * Implements svn_cache__deserialize_func_t for p2l_header_t objects. + */ +svn_error_t * +svn_fs_x__deserialize_p2l_header(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/* + * Implements svn_cache__serialize_func_t for apr_array_header_t objects + * with elements of type svn_fs_x__p2l_entry_t. + */ +svn_error_t * +svn_fs_x__serialize_p2l_page(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/* + * Implements svn_cache__deserialize_func_t for apr_array_header_t objects + * with elements of type svn_fs_x__p2l_entry_t. + */ +svn_error_t * +svn_fs_x__deserialize_p2l_page(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +#endif
diff --git a/subversion/libsvn_fs_x/lock.c b/subversion/libsvn_fs_x/lock.c new file mode 100644 index 0000000..5c701ba --- /dev/null +++ b/subversion/libsvn_fs_x/lock.c
@@ -0,0 +1,1079 @@ +/* lock.c : functions for manipulating filesystem locks. + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + + +#include "svn_pools.h" +#include "svn_error.h" +#include "svn_dirent_uri.h" +#include "svn_path.h" +#include "svn_fs.h" +#include "svn_hash.h" +#include "svn_time.h" +#include "svn_utf.h" + +#include <apr_uuid.h> +#include <apr_file_io.h> +#include <apr_file_info.h> + +#include "lock.h" +#include "tree.h" +#include "fs_x.h" +#include "transaction.h" +#include "../libsvn_fs/fs-loader.h" + +#include "private/svn_fs_util.h" +#include "private/svn_fspath.h" +#include "svn_private_config.h" + +/* Names of hash keys used to store a lock for writing to disk. */ +#define PATH_KEY "path" +#define TOKEN_KEY "token" +#define OWNER_KEY "owner" +#define CREATION_DATE_KEY "creation_date" +#define EXPIRATION_DATE_KEY "expiration_date" +#define COMMENT_KEY "comment" +#define IS_DAV_COMMENT_KEY "is_dav_comment" +#define CHILDREN_KEY "children" + +/* Number of characters from the head of a digest file name used to + calculate a subdirectory in which to drop that file. */ +#define DIGEST_SUBDIR_LEN 3 + + + +/*** Generic helper functions. ***/ + +/* Set *DIGEST to the MD5 hash of STR. */ +static svn_error_t * +make_digest(const char **digest, + const char *str, + apr_pool_t *pool) +{ + svn_checksum_t *checksum; + + SVN_ERR(svn_checksum(&checksum, svn_checksum_md5, str, strlen(str), pool)); + + *digest = svn_checksum_to_cstring_display(checksum, pool); + return SVN_NO_ERROR; +} + + +/* Set the value of KEY (whose size is KEY_LEN, or APR_HASH_KEY_STRING + if unknown) to an svn_string_t-ized version of VALUE (whose size is + VALUE_LEN, or APR_HASH_KEY_STRING if unknown) in HASH. The value + will be allocated in POOL; KEY will not be duped. If either KEY or VALUE + is NULL, this function will do nothing. */ +static void +hash_store(apr_hash_t *hash, + const char *key, + apr_ssize_t key_len, + const char *value, + apr_ssize_t value_len, + apr_pool_t *pool) +{ + if (! (key && value)) + return; + if (value_len == APR_HASH_KEY_STRING) + value_len = strlen(value); + apr_hash_set(hash, key, key_len, + svn_string_ncreate(value, value_len, pool)); +} + + +/* Fetch the value of KEY from HASH, returning only the cstring data + of that value (if it exists). */ +static const char * +hash_fetch(apr_hash_t *hash, + const char *key, + apr_pool_t *pool) +{ + svn_string_t *str = svn_hash_gets(hash, key); + return str ? str->data : NULL; +} + + +/* SVN_ERR_FS_CORRUPT: the lockfile for PATH in FS is corrupt. */ +static svn_error_t * +err_corrupt_lockfile(const char *fs_path, const char *path) +{ + return + svn_error_createf( + SVN_ERR_FS_CORRUPT, 0, + _("Corrupt lockfile for path '%s' in filesystem '%s'"), + path, fs_path); +} + + +/*** Digest file handling functions. ***/ + +/* Return the path of the lock/entries file for which DIGEST is the + hashed repository relative path. */ +static const char * +digest_path_from_digest(const char *fs_path, + const char *digest, + apr_pool_t *pool) +{ + return svn_dirent_join_many(pool, fs_path, PATH_LOCKS_DIR, + apr_pstrmemdup(pool, digest, DIGEST_SUBDIR_LEN), + digest, NULL); +} + + +/* Set *DIGEST_PATH to the path to the lock/entries digest file associate + with PATH, where PATH is the path to the lock file or lock entries file + in FS. */ +static svn_error_t * +digest_path_from_path(const char **digest_path, + const char *fs_path, + const char *path, + apr_pool_t *pool) +{ + const char *digest; + SVN_ERR(make_digest(&digest, path, pool)); + *digest_path = svn_dirent_join_many(pool, fs_path, PATH_LOCKS_DIR, + apr_pstrmemdup(pool, digest, + DIGEST_SUBDIR_LEN), + digest, NULL); + return SVN_NO_ERROR; +} + + +/* Write to DIGEST_PATH a representation of CHILDREN (which may be + empty, if the versioned path in FS represented by DIGEST_PATH has + no children) and LOCK (which may be NULL if that versioned path is + lock itself locked). Set the permissions of DIGEST_PATH to those of + PERMS_REFERENCE. Use POOL for all allocations. + */ +static svn_error_t * +write_digest_file(apr_hash_t *children, + svn_lock_t *lock, + const char *fs_path, + const char *digest_path, + const char *perms_reference, + apr_pool_t *pool) +{ + svn_error_t *err = SVN_NO_ERROR; + svn_stream_t *stream; + apr_hash_index_t *hi; + apr_hash_t *hash = apr_hash_make(pool); + const char *tmp_path; + + SVN_ERR(svn_fs_x__ensure_dir_exists(svn_dirent_join(fs_path, PATH_LOCKS_DIR, + pool), fs_path, pool)); + SVN_ERR(svn_fs_x__ensure_dir_exists(svn_dirent_dirname(digest_path, pool), + fs_path, pool)); + + if (lock) + { + const char *creation_date = NULL, *expiration_date = NULL; + if (lock->creation_date) + creation_date = svn_time_to_cstring(lock->creation_date, pool); + if (lock->expiration_date) + expiration_date = svn_time_to_cstring(lock->expiration_date, pool); + hash_store(hash, PATH_KEY, sizeof(PATH_KEY)-1, + lock->path, APR_HASH_KEY_STRING, pool); + hash_store(hash, TOKEN_KEY, sizeof(TOKEN_KEY)-1, + lock->token, APR_HASH_KEY_STRING, pool); + hash_store(hash, OWNER_KEY, sizeof(OWNER_KEY)-1, + lock->owner, APR_HASH_KEY_STRING, pool); + hash_store(hash, COMMENT_KEY, sizeof(COMMENT_KEY)-1, + lock->comment, APR_HASH_KEY_STRING, pool); + hash_store(hash, IS_DAV_COMMENT_KEY, sizeof(IS_DAV_COMMENT_KEY)-1, + lock->is_dav_comment ? "1" : "0", 1, pool); + hash_store(hash, CREATION_DATE_KEY, sizeof(CREATION_DATE_KEY)-1, + creation_date, APR_HASH_KEY_STRING, pool); + hash_store(hash, EXPIRATION_DATE_KEY, sizeof(EXPIRATION_DATE_KEY)-1, + expiration_date, APR_HASH_KEY_STRING, pool); + } + if (apr_hash_count(children)) + { + svn_stringbuf_t *children_list = svn_stringbuf_create_empty(pool); + for (hi = apr_hash_first(pool, children); hi; hi = apr_hash_next(hi)) + { + svn_stringbuf_appendbytes(children_list, + svn__apr_hash_index_key(hi), + svn__apr_hash_index_klen(hi)); + svn_stringbuf_appendbyte(children_list, '\n'); + } + hash_store(hash, CHILDREN_KEY, sizeof(CHILDREN_KEY)-1, + children_list->data, children_list->len, pool); + } + + SVN_ERR(svn_stream_open_unique(&stream, &tmp_path, + svn_dirent_dirname(digest_path, pool), + svn_io_file_del_none, pool, pool)); + if ((err = svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, pool))) + { + svn_error_clear(svn_stream_close(stream)); + return svn_error_createf(err->apr_err, + err, + _("Cannot write lock/entries hashfile '%s'"), + svn_dirent_local_style(tmp_path, pool)); + } + + SVN_ERR(svn_stream_close(stream)); + SVN_ERR(svn_io_file_rename(tmp_path, digest_path, pool)); + SVN_ERR(svn_io_copy_perms(perms_reference, digest_path, pool)); + return SVN_NO_ERROR; +} + + +/* Parse the file at DIGEST_PATH, populating the lock LOCK_P in that + file (if it exists, and if *LOCK_P is non-NULL) and the hash of + CHILDREN_P (if any exist, and if *CHILDREN_P is non-NULL). Use POOL + for all allocations. */ +static svn_error_t * +read_digest_file(apr_hash_t **children_p, + svn_lock_t **lock_p, + const char *fs_path, + const char *digest_path, + apr_pool_t *pool) +{ + svn_error_t *err = SVN_NO_ERROR; + svn_lock_t *lock; + apr_hash_t *hash; + svn_stream_t *stream; + const char *val; + + if (lock_p) + *lock_p = NULL; + if (children_p) + *children_p = apr_hash_make(pool); + + err = svn_stream_open_readonly(&stream, digest_path, pool, pool); + if (err && APR_STATUS_IS_ENOENT(err->apr_err)) + { + svn_error_clear(err); + return SVN_NO_ERROR; + } + SVN_ERR(err); + + /* If our caller doesn't care about anything but the presence of the + file... whatever. */ + if (! (lock_p || children_p)) + return svn_stream_close(stream); + + hash = apr_hash_make(pool); + if ((err = svn_hash_read2(hash, stream, SVN_HASH_TERMINATOR, pool))) + { + svn_error_clear(svn_stream_close(stream)); + return svn_error_createf(err->apr_err, + err, + _("Can't parse lock/entries hashfile '%s'"), + svn_dirent_local_style(digest_path, pool)); + } + SVN_ERR(svn_stream_close(stream)); + + /* If our caller cares, see if we have a lock path in our hash. If + so, we'll assume we have a lock here. */ + val = hash_fetch(hash, PATH_KEY, pool); + if (val && lock_p) + { + const char *path = val; + + /* Create our lock and load it up. */ + lock = svn_lock_create(pool); + lock->path = path; + + if (! ((lock->token = hash_fetch(hash, TOKEN_KEY, pool)))) + return svn_error_trace(err_corrupt_lockfile(fs_path, path)); + + if (! ((lock->owner = hash_fetch(hash, OWNER_KEY, pool)))) + return svn_error_trace(err_corrupt_lockfile(fs_path, path)); + + if (! ((val = hash_fetch(hash, IS_DAV_COMMENT_KEY, pool)))) + return svn_error_trace(err_corrupt_lockfile(fs_path, path)); + lock->is_dav_comment = (val[0] == '1'); + + if (! ((val = hash_fetch(hash, CREATION_DATE_KEY, pool)))) + return svn_error_trace(err_corrupt_lockfile(fs_path, path)); + SVN_ERR(svn_time_from_cstring(&(lock->creation_date), val, pool)); + + if ((val = hash_fetch(hash, EXPIRATION_DATE_KEY, pool))) + SVN_ERR(svn_time_from_cstring(&(lock->expiration_date), val, pool)); + + lock->comment = hash_fetch(hash, COMMENT_KEY, pool); + + *lock_p = lock; + } + + /* If our caller cares, see if we have any children for this path. */ + val = hash_fetch(hash, CHILDREN_KEY, pool); + if (val && children_p) + { + apr_array_header_t *kiddos = svn_cstring_split(val, "\n", FALSE, pool); + int i; + + for (i = 0; i < kiddos->nelts; i++) + { + svn_hash_sets(*children_p, APR_ARRAY_IDX(kiddos, i, const char *), + (void *)1); + } + } + return SVN_NO_ERROR; +} + + + +/*** Lock helper functions (path here are still FS paths, not on-disk + schema-supporting paths) ***/ + + +/* Write LOCK in FS to the actual OS filesystem. + + Use PERMS_REFERENCE for the permissions of any digest files. + + Note: this takes an FS_PATH because it's called from the hotcopy logic. + */ +static svn_error_t * +set_lock(const char *fs_path, + svn_lock_t *lock, + const char *perms_reference, + apr_pool_t *pool) +{ + svn_stringbuf_t *this_path = svn_stringbuf_create(lock->path, pool); + const char *lock_digest_path = NULL; + apr_pool_t *subpool; + + SVN_ERR_ASSERT(lock); + + /* Iterate in reverse, creating the lock for LOCK->path, and then + just adding entries for its parent, until we reach a parent + that's already listed in *its* parent. */ + subpool = svn_pool_create(pool); + while (1729) + { + const char *digest_path, *digest_file; + apr_hash_t *this_children; + svn_lock_t *this_lock; + + svn_pool_clear(subpool); + + /* Calculate the DIGEST_PATH for the currently FS path, and then + get its DIGEST_FILE basename. */ + SVN_ERR(digest_path_from_path(&digest_path, fs_path, this_path->data, + subpool)); + digest_file = svn_dirent_basename(digest_path, subpool); + + SVN_ERR(read_digest_file(&this_children, &this_lock, fs_path, + digest_path, subpool)); + + /* We're either writing a new lock (first time through only) or + a new entry (every time but the first). */ + if (lock) + { + this_lock = lock; + lock = NULL; + lock_digest_path = apr_pstrdup(pool, digest_file); + } + else + { + /* If we already have an entry for this path, we're done. */ + if (svn_hash_gets(this_children, lock_digest_path)) + break; + svn_hash_sets(this_children, lock_digest_path, (void *)1); + } + SVN_ERR(write_digest_file(this_children, this_lock, fs_path, + digest_path, perms_reference, subpool)); + + /* Prep for next iteration, or bail if we're done. */ + if (svn_fspath__is_root(this_path->data, this_path->len)) + break; + svn_stringbuf_set(this_path, + svn_fspath__dirname(this_path->data, subpool)); + } + + svn_pool_destroy(subpool); + return SVN_NO_ERROR; +} + +/* Delete LOCK from FS in the actual OS filesystem. */ +static svn_error_t * +delete_lock(svn_fs_t *fs, + svn_lock_t *lock, + apr_pool_t *pool) +{ + svn_stringbuf_t *this_path = svn_stringbuf_create(lock->path, pool); + const char *child_to_kill = NULL; + apr_pool_t *subpool; + + SVN_ERR_ASSERT(lock); + + /* Iterate in reverse, deleting the lock for LOCK->path, and then + deleting its entry as it appears in each of its parents. */ + subpool = svn_pool_create(pool); + while (1729) + { + const char *digest_path, *digest_file; + apr_hash_t *this_children; + svn_lock_t *this_lock; + + svn_pool_clear(subpool); + + /* Calculate the DIGEST_PATH for the currently FS path, and then + get its DIGEST_FILE basename. */ + SVN_ERR(digest_path_from_path(&digest_path, fs->path, this_path->data, + subpool)); + digest_file = svn_dirent_basename(digest_path, subpool); + + SVN_ERR(read_digest_file(&this_children, &this_lock, fs->path, + digest_path, subpool)); + + /* Delete the lock (first time through only). */ + if (lock) + { + this_lock = NULL; + lock = NULL; + child_to_kill = apr_pstrdup(pool, digest_file); + } + + if (child_to_kill) + svn_hash_sets(this_children, child_to_kill, NULL); + + if (! (this_lock || apr_hash_count(this_children) != 0)) + { + /* Special case: no goodz, no file. And remember to nix + the entry for it in its parent. */ + SVN_ERR(svn_io_remove_file2(digest_path, FALSE, subpool)); + } + else + { + const char *rev_0_path + = svn_fs_x__path_rev_absolute(fs, 0, pool); + SVN_ERR(write_digest_file(this_children, this_lock, fs->path, + digest_path, rev_0_path, subpool)); + } + + /* Prep for next iteration, or bail if we're done. */ + if (svn_fspath__is_root(this_path->data, this_path->len)) + break; + svn_stringbuf_set(this_path, + svn_fspath__dirname(this_path->data, subpool)); + } + + svn_pool_destroy(subpool); + return SVN_NO_ERROR; +} + +/* Set *LOCK_P to the lock for PATH in FS. HAVE_WRITE_LOCK should be + TRUE if the caller (or one of its callers) has taken out the + repository-wide write lock, FALSE otherwise. If MUST_EXIST is + not set, the function will simply return NULL in *LOCK_P instead + of creating an SVN_FS__ERR_NO_SUCH_LOCK error in case the lock + was not found (much faster). Use POOL for allocations. */ +static svn_error_t * +get_lock(svn_lock_t **lock_p, + svn_fs_t *fs, + const char *path, + svn_boolean_t have_write_lock, + svn_boolean_t must_exist, + apr_pool_t *pool) +{ + svn_lock_t *lock = NULL; + const char *digest_path; + svn_node_kind_t kind; + + SVN_ERR(digest_path_from_path(&digest_path, fs->path, path, pool)); + SVN_ERR(svn_io_check_path(digest_path, &kind, pool)); + + *lock_p = NULL; + if (kind != svn_node_none) + SVN_ERR(read_digest_file(NULL, &lock, fs->path, digest_path, pool)); + + if (! lock) + return must_exist ? SVN_FS__ERR_NO_SUCH_LOCK(fs, path) : SVN_NO_ERROR; + + /* Don't return an expired lock. */ + if (lock->expiration_date && (apr_time_now() > lock->expiration_date)) + { + /* Only remove the lock if we have the write lock. + Read operations shouldn't change the filesystem. */ + if (have_write_lock) + SVN_ERR(delete_lock(fs, lock, pool)); + return SVN_FS__ERR_LOCK_EXPIRED(fs, lock->token); + } + + *lock_p = lock; + return SVN_NO_ERROR; +} + + +/* Set *LOCK_P to the lock for PATH in FS. HAVE_WRITE_LOCK should be + TRUE if the caller (or one of its callers) has taken out the + repository-wide write lock, FALSE otherwise. Use POOL for + allocations. */ +static svn_error_t * +get_lock_helper(svn_fs_t *fs, + svn_lock_t **lock_p, + const char *path, + svn_boolean_t have_write_lock, + apr_pool_t *pool) +{ + svn_lock_t *lock; + svn_error_t *err; + + err = get_lock(&lock, fs, path, have_write_lock, FALSE, pool); + + /* We've deliberately decided that this function doesn't tell the + caller *why* the lock is unavailable. */ + if (err && ((err->apr_err == SVN_ERR_FS_NO_SUCH_LOCK) + || (err->apr_err == SVN_ERR_FS_LOCK_EXPIRED))) + { + svn_error_clear(err); + *lock_p = NULL; + return SVN_NO_ERROR; + } + else + SVN_ERR(err); + + *lock_p = lock; + return SVN_NO_ERROR; +} + + +/* Baton for locks_walker(). */ +struct walk_locks_baton { + svn_fs_get_locks_callback_t get_locks_func; + void *get_locks_baton; + svn_fs_t *fs; +}; + +/* Implements walk_digests_callback_t. */ +static svn_error_t * +locks_walker(void *baton, + const char *fs_path, + const char *digest_path, + apr_hash_t *children, + svn_lock_t *lock, + svn_boolean_t have_write_lock, + apr_pool_t *pool) +{ + struct walk_locks_baton *wlb = baton; + + if (lock) + { + /* Don't report an expired lock. */ + if (lock->expiration_date == 0 + || (apr_time_now() <= lock->expiration_date)) + { + if (wlb->get_locks_func) + SVN_ERR(wlb->get_locks_func(wlb->get_locks_baton, lock, pool)); + } + else + { + /* Only remove the lock if we have the write lock. + Read operations shouldn't change the filesystem. */ + if (have_write_lock) + SVN_ERR(delete_lock(wlb->fs, lock, pool)); + } + } + + return SVN_NO_ERROR; +} + +/* Callback type for walk_digest_files(). + * + * CHILDREN and LOCK come from a read_digest_file(digest_path) call. + */ +typedef svn_error_t *(*walk_digests_callback_t)(void *baton, + const char *fs_path, + const char *digest_path, + apr_hash_t *children, + svn_lock_t *lock, + svn_boolean_t have_write_lock, + apr_pool_t *pool); + +/* A recursive function that calls WALK_DIGESTS_FUNC/WALK_DIGESTS_BATON for + all lock digest files in and under PATH in FS. + HAVE_WRITE_LOCK should be true if the caller (directly or indirectly) + has the FS write lock. */ +static svn_error_t * +walk_digest_files(const char *fs_path, + const char *digest_path, + walk_digests_callback_t walk_digests_func, + void *walk_digests_baton, + svn_boolean_t have_write_lock, + apr_pool_t *pool) +{ + apr_hash_index_t *hi; + apr_hash_t *children; + apr_pool_t *subpool; + svn_lock_t *lock; + + /* First, send up any locks in the current digest file. */ + SVN_ERR(read_digest_file(&children, &lock, fs_path, digest_path, pool)); + + SVN_ERR(walk_digests_func(walk_digests_baton, fs_path, digest_path, + children, lock, + have_write_lock, pool)); + + /* Now, recurse on this thing's child entries (if any; bail otherwise). */ + if (! apr_hash_count(children)) + return SVN_NO_ERROR; + subpool = svn_pool_create(pool); + for (hi = apr_hash_first(pool, children); hi; hi = apr_hash_next(hi)) + { + const char *digest = svn__apr_hash_index_key(hi); + svn_pool_clear(subpool); + SVN_ERR(walk_digest_files + (fs_path, digest_path_from_digest(fs_path, digest, subpool), + walk_digests_func, walk_digests_baton, have_write_lock, subpool)); + } + svn_pool_destroy(subpool); + return SVN_NO_ERROR; +} + +/* A recursive function that calls GET_LOCKS_FUNC/GET_LOCKS_BATON for + all locks in and under PATH in FS. + HAVE_WRITE_LOCK should be true if the caller (directly or indirectly) + has the FS write lock. */ +static svn_error_t * +walk_locks(svn_fs_t *fs, + const char *digest_path, + svn_fs_get_locks_callback_t get_locks_func, + void *get_locks_baton, + svn_boolean_t have_write_lock, + apr_pool_t *pool) +{ + struct walk_locks_baton wlb; + + wlb.get_locks_func = get_locks_func; + wlb.get_locks_baton = get_locks_baton; + wlb.fs = fs; + SVN_ERR(walk_digest_files(fs->path, digest_path, locks_walker, &wlb, + have_write_lock, pool)); + return SVN_NO_ERROR; +} + + +/* Utility function: verify that a lock can be used. Interesting + errors returned from this function: + + SVN_ERR_FS_NO_USER: No username attached to FS. + SVN_ERR_FS_LOCK_OWNER_MISMATCH: FS's username doesn't match LOCK's owner. + SVN_ERR_FS_BAD_LOCK_TOKEN: FS doesn't hold matching lock-token for LOCK. + */ +static svn_error_t * +verify_lock(svn_fs_t *fs, + svn_lock_t *lock, + apr_pool_t *pool) +{ + if ((! fs->access_ctx) || (! fs->access_ctx->username)) + return svn_error_createf + (SVN_ERR_FS_NO_USER, NULL, + _("Cannot verify lock on path '%s'; no username available"), + lock->path); + + else if (strcmp(fs->access_ctx->username, lock->owner) != 0) + return svn_error_createf + (SVN_ERR_FS_LOCK_OWNER_MISMATCH, NULL, + _("User '%s' does not own lock on path '%s' (currently locked by '%s')"), + fs->access_ctx->username, lock->path, lock->owner); + + else if (svn_hash_gets(fs->access_ctx->lock_tokens, lock->token) == NULL) + return svn_error_createf + (SVN_ERR_FS_BAD_LOCK_TOKEN, NULL, + _("Cannot verify lock on path '%s'; no matching lock-token available"), + lock->path); + + return SVN_NO_ERROR; +} + + +/* This implements the svn_fs_get_locks_callback_t interface, where + BATON is just an svn_fs_t object. */ +static svn_error_t * +get_locks_callback(void *baton, + svn_lock_t *lock, + apr_pool_t *pool) +{ + return verify_lock(baton, lock, pool); +} + + +/* The main routine for lock enforcement, used throughout libsvn_fs_x. */ +svn_error_t * +svn_fs_x__allow_locked_operation(const char *path, + svn_fs_t *fs, + svn_boolean_t recurse, + svn_boolean_t have_write_lock, + apr_pool_t *pool) +{ + path = svn_fs__canonicalize_abspath(path, pool); + if (recurse) + { + /* Discover all locks at or below the path. */ + const char *digest_path; + SVN_ERR(digest_path_from_path(&digest_path, fs->path, path, pool)); + SVN_ERR(walk_locks(fs, digest_path, get_locks_callback, + fs, have_write_lock, pool)); + } + else + { + /* Discover and verify any lock attached to the path. */ + svn_lock_t *lock; + SVN_ERR(get_lock_helper(fs, &lock, path, have_write_lock, pool)); + if (lock) + SVN_ERR(verify_lock(fs, lock, pool)); + } + return SVN_NO_ERROR; +} + +/* Baton used for lock_body below. */ +struct lock_baton { + svn_lock_t **lock_p; + svn_fs_t *fs; + const char *path; + const char *token; + const char *comment; + svn_boolean_t is_dav_comment; + apr_time_t expiration_date; + svn_revnum_t current_rev; + svn_boolean_t steal_lock; + apr_pool_t *pool; +}; + + +/* This implements the svn_fs_x__with_write_lock() 'body' callback + type, and assumes that the write lock is held. + BATON is a 'struct lock_baton *'. */ +static svn_error_t * +lock_body(void *baton, apr_pool_t *pool) +{ + struct lock_baton *lb = baton; + svn_node_kind_t kind; + svn_lock_t *existing_lock; + svn_lock_t *lock; + svn_fs_root_t *root; + svn_revnum_t youngest; + const char *rev_0_path; + + /* Until we implement directory locks someday, we only allow locks + on files or non-existent paths. */ + /* Use fs->vtable->foo instead of svn_fs_foo to avoid circular + library dependencies, which are not portable. */ + SVN_ERR(lb->fs->vtable->youngest_rev(&youngest, lb->fs, pool)); + SVN_ERR(lb->fs->vtable->revision_root(&root, lb->fs, youngest, pool)); + SVN_ERR(svn_fs_x__check_path(&kind, root, lb->path, pool)); + if (kind == svn_node_dir) + return SVN_FS__ERR_NOT_FILE(lb->fs, lb->path); + + /* While our locking implementation easily supports the locking of + nonexistent paths, we deliberately choose not to allow such madness. */ + if (kind == svn_node_none) + { + if (SVN_IS_VALID_REVNUM(lb->current_rev)) + return svn_error_createf( + SVN_ERR_FS_OUT_OF_DATE, NULL, + _("Path '%s' doesn't exist in HEAD revision"), + lb->path); + else + return svn_error_createf( + SVN_ERR_FS_NOT_FOUND, NULL, + _("Path '%s' doesn't exist in HEAD revision"), + lb->path); + } + + /* We need to have a username attached to the fs. */ + if (!lb->fs->access_ctx || !lb->fs->access_ctx->username) + return SVN_FS__ERR_NO_USER(lb->fs); + + /* Is the caller attempting to lock an out-of-date working file? */ + if (SVN_IS_VALID_REVNUM(lb->current_rev)) + { + svn_revnum_t created_rev; + SVN_ERR(svn_fs_x__node_created_rev(&created_rev, root, lb->path, pool)); + + /* SVN_INVALID_REVNUM means the path doesn't exist. So + apparently somebody is trying to lock something in their + working copy, but somebody else has deleted the thing + from HEAD. That counts as being 'out of date'. */ + if (! SVN_IS_VALID_REVNUM(created_rev)) + return svn_error_createf + (SVN_ERR_FS_OUT_OF_DATE, NULL, + _("Path '%s' doesn't exist in HEAD revision"), lb->path); + + if (lb->current_rev < created_rev) + return svn_error_createf + (SVN_ERR_FS_OUT_OF_DATE, NULL, + _("Lock failed: newer version of '%s' exists"), lb->path); + } + + /* If the caller provided a TOKEN, we *really* need to see + if a lock already exists with that token, and if so, verify that + the lock's path matches PATH. Otherwise we run the risk of + breaking the 1-to-1 mapping of lock tokens to locked paths. */ + /* ### TODO: actually do this check. This is tough, because the + schema doesn't supply a lookup-by-token mechanism. */ + + /* Is the path already locked? + + Note that this next function call will automatically ignore any + errors about {the path not existing as a key, the path's token + not existing as a key, the lock just having been expired}. And + that's totally fine. Any of these three errors are perfectly + acceptable to ignore; it means that the path is now free and + clear for locking, because the fsx funcs just cleared out both + of the tables for us. */ + SVN_ERR(get_lock_helper(lb->fs, &existing_lock, lb->path, TRUE, pool)); + if (existing_lock) + { + if (! lb->steal_lock) + { + /* Sorry, the path is already locked. */ + return SVN_FS__ERR_PATH_ALREADY_LOCKED(lb->fs, existing_lock); + } + else + { + /* STEAL_LOCK was passed, so fs_username is "stealing" the + lock from lock->owner. Destroy the existing lock. */ + SVN_ERR(delete_lock(lb->fs, existing_lock, pool)); + } + } + + /* Create our new lock, and add it to the tables. + Ensure that the lock is created in the correct pool. */ + lock = svn_lock_create(lb->pool); + if (lb->token) + lock->token = apr_pstrdup(lb->pool, lb->token); + else + SVN_ERR(svn_fs_x__generate_lock_token(&(lock->token), lb->fs, lb->pool)); + lock->path = apr_pstrdup(lb->pool, lb->path); + lock->owner = apr_pstrdup(lb->pool, lb->fs->access_ctx->username); + lock->comment = apr_pstrdup(lb->pool, lb->comment); + lock->is_dav_comment = lb->is_dav_comment; + lock->creation_date = apr_time_now(); + lock->expiration_date = lb->expiration_date; + + rev_0_path = svn_fs_x__path_rev_absolute(lb->fs, 0, pool); + SVN_ERR(set_lock(lb->fs->path, lock, rev_0_path, pool)); + *lb->lock_p = lock; + + return SVN_NO_ERROR; +} + +/* Baton used for unlock_body below. */ +struct unlock_baton { + svn_fs_t *fs; + const char *path; + const char *token; + svn_boolean_t break_lock; +}; + +/* This implements the svn_fs_x__with_write_lock() 'body' callback + type, and assumes that the write lock is held. + BATON is a 'struct unlock_baton *'. */ +static svn_error_t * +unlock_body(void *baton, apr_pool_t *pool) +{ + struct unlock_baton *ub = baton; + svn_lock_t *lock; + + /* This could return SVN_ERR_FS_BAD_LOCK_TOKEN or SVN_ERR_FS_LOCK_EXPIRED. */ + SVN_ERR(get_lock(&lock, ub->fs, ub->path, TRUE, TRUE, pool)); + + /* Unless breaking the lock, we do some checks. */ + if (! ub->break_lock) + { + /* Sanity check: the incoming token should match lock->token. */ + if (strcmp(ub->token, lock->token) != 0) + return SVN_FS__ERR_NO_SUCH_LOCK(ub->fs, lock->path); + + /* There better be a username attached to the fs. */ + if (! (ub->fs->access_ctx && ub->fs->access_ctx->username)) + return SVN_FS__ERR_NO_USER(ub->fs); + + /* And that username better be the same as the lock's owner. */ + if (strcmp(ub->fs->access_ctx->username, lock->owner) != 0) + return SVN_FS__ERR_LOCK_OWNER_MISMATCH( + ub->fs, ub->fs->access_ctx->username, lock->owner); + } + + /* Remove lock and lock token files. */ + return delete_lock(ub->fs, lock, pool); +} + + +/*** Public API implementations ***/ + +svn_error_t * +svn_fs_x__lock(svn_lock_t **lock_p, + svn_fs_t *fs, + const char *path, + const char *token, + const char *comment, + svn_boolean_t is_dav_comment, + apr_time_t expiration_date, + svn_revnum_t current_rev, + svn_boolean_t steal_lock, + apr_pool_t *pool) +{ + struct lock_baton lb; + + SVN_ERR(svn_fs__check_fs(fs, TRUE)); + path = svn_fs__canonicalize_abspath(path, pool); + + lb.lock_p = lock_p; + lb.fs = fs; + lb.path = path; + lb.token = token; + lb.comment = comment; + lb.is_dav_comment = is_dav_comment; + lb.expiration_date = expiration_date; + lb.current_rev = current_rev; + lb.steal_lock = steal_lock; + lb.pool = pool; + + return svn_fs_x__with_write_lock(fs, lock_body, &lb, pool); +} + + +svn_error_t * +svn_fs_x__generate_lock_token(const char **token, + svn_fs_t *fs, + apr_pool_t *pool) +{ + SVN_ERR(svn_fs__check_fs(fs, TRUE)); + + /* Notice that 'fs' is currently unused. But perhaps someday, we'll + want to use the fs UUID + some incremented number? For now, we + generate a URI that matches the DAV RFC. We could change this to + some other URI scheme someday, if we wish. */ + *token = apr_pstrcat(pool, "opaquelocktoken:", + svn_uuid_generate(pool), (char *)NULL); + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__unlock(svn_fs_t *fs, + const char *path, + const char *token, + svn_boolean_t break_lock, + apr_pool_t *pool) +{ + struct unlock_baton ub; + + SVN_ERR(svn_fs__check_fs(fs, TRUE)); + path = svn_fs__canonicalize_abspath(path, pool); + + ub.fs = fs; + ub.path = path; + ub.token = token; + ub.break_lock = break_lock; + + return svn_fs_x__with_write_lock(fs, unlock_body, &ub, pool); +} + + +svn_error_t * +svn_fs_x__get_lock(svn_lock_t **lock_p, + svn_fs_t *fs, + const char *path, + apr_pool_t *pool) +{ + SVN_ERR(svn_fs__check_fs(fs, TRUE)); + path = svn_fs__canonicalize_abspath(path, pool); + return get_lock_helper(fs, lock_p, path, FALSE, pool); +} + + +/* Baton for get_locks_filter_func(). */ +typedef struct get_locks_filter_baton_t +{ + const char *path; + svn_depth_t requested_depth; + svn_fs_get_locks_callback_t get_locks_func; + void *get_locks_baton; + +} get_locks_filter_baton_t; + + +/* A wrapper for the GET_LOCKS_FUNC passed to svn_fs_x__get_locks() + which filters out locks on paths that aren't within + BATON->requested_depth of BATON->path before called + BATON->get_locks_func() with BATON->get_locks_baton. + + NOTE: See issue #3660 for details about how the FSX lock + management code is inconsistent. Until that inconsistency is + resolved, we take this filtering approach rather than honoring + depth requests closer to the crawling code. In other words, once + we decide how to resolve issue #3660, there might be a more + performant way to honor the depth passed to svn_fs_x__get_locks(). */ +static svn_error_t * +get_locks_filter_func(void *baton, + svn_lock_t *lock, + apr_pool_t *pool) +{ + get_locks_filter_baton_t *b = baton; + + /* Filter out unwanted paths. Since Subversion only allows + locks on files, we can treat depth=immediates the same as + depth=files for filtering purposes. Meaning, we'll keep + this lock if: + + a) its path is the very path we queried, or + b) we've asked for a fully recursive answer, or + c) we've asked for depth=files or depth=immediates, and this + lock is on an immediate child of our query path. + */ + if ((strcmp(b->path, lock->path) == 0) + || (b->requested_depth == svn_depth_infinity)) + { + SVN_ERR(b->get_locks_func(b->get_locks_baton, lock, pool)); + } + else if ((b->requested_depth == svn_depth_files) || + (b->requested_depth == svn_depth_immediates)) + { + const char *rel_uri = svn_fspath__skip_ancestor(b->path, lock->path); + if (rel_uri && (svn_path_component_count(rel_uri) == 1)) + SVN_ERR(b->get_locks_func(b->get_locks_baton, lock, pool)); + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__get_locks(svn_fs_t *fs, + const char *path, + svn_depth_t depth, + svn_fs_get_locks_callback_t get_locks_func, + void *get_locks_baton, + apr_pool_t *pool) +{ + const char *digest_path; + get_locks_filter_baton_t glfb; + + SVN_ERR(svn_fs__check_fs(fs, TRUE)); + path = svn_fs__canonicalize_abspath(path, pool); + + glfb.path = path; + glfb.requested_depth = depth; + glfb.get_locks_func = get_locks_func; + glfb.get_locks_baton = get_locks_baton; + + /* Get the top digest path in our tree of interest, and then walk it. */ + SVN_ERR(digest_path_from_path(&digest_path, fs->path, path, pool)); + SVN_ERR(walk_locks(fs, digest_path, get_locks_filter_func, &glfb, + FALSE, pool)); + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_fs_x/lock.h b/subversion/libsvn_fs_x/lock.h new file mode 100644 index 0000000..00b68b1 --- /dev/null +++ b/subversion/libsvn_fs_x/lock.h
@@ -0,0 +1,103 @@ +/* lock.h : internal interface to lock functions + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS_LOCK_H +#define SVN_LIBSVN_FS_LOCK_H + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + + + +/* These functions implement some of the calls in the FS loader + library's fs vtables. */ + +svn_error_t *svn_fs_x__lock(svn_lock_t **lock, + svn_fs_t *fs, + const char *path, + const char *token, + const char *comment, + svn_boolean_t is_dav_comment, + apr_time_t expiration_date, + svn_revnum_t current_rev, + svn_boolean_t steal_lock, + apr_pool_t *pool); + +svn_error_t *svn_fs_x__generate_lock_token(const char **token, + svn_fs_t *fs, + apr_pool_t *pool); + +svn_error_t *svn_fs_x__unlock(svn_fs_t *fs, + const char *path, + const char *token, + svn_boolean_t break_lock, + apr_pool_t *pool); + +svn_error_t *svn_fs_x__get_lock(svn_lock_t **lock, + svn_fs_t *fs, + const char *path, + apr_pool_t *pool); + +svn_error_t *svn_fs_x__get_locks(svn_fs_t *fs, + const char *path, + svn_depth_t depth, + svn_fs_get_locks_callback_t get_locks_func, + void *get_locks_baton, + apr_pool_t *pool); + + +/* Examine PATH for existing locks, and check whether they can be + used. Use POOL for temporary allocations. + + If no locks are present, return SVN_NO_ERROR. + + If PATH is locked (or contains locks "below" it, when RECURSE is + set), then verify that: + + 1. a username has been supplied to TRAIL->fs's access-context, + else return SVN_ERR_FS_NO_USER. + + 2. for every lock discovered, the current username in the access + context of TRAIL->fs matches the "owner" of the lock, else + return SVN_ERR_FS_LOCK_OWNER_MISMATCH. + + 3. for every lock discovered, a matching lock token has been + passed into TRAIL->fs's access-context, else return + SVN_ERR_FS_BAD_LOCK_TOKEN. + + If all three conditions are met, return SVN_NO_ERROR. + + If the caller (directly or indirectly) has the FS write lock, + HAVE_WRITE_LOCK should be true. +*/ +svn_error_t *svn_fs_x__allow_locked_operation(const char *path, + svn_fs_t *fs, + svn_boolean_t recurse, + svn_boolean_t have_write_lock, + apr_pool_t *pool); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* SVN_LIBSVN_FS_LOCK_H */
diff --git a/subversion/libsvn_fs_x/low_level.c b/subversion/libsvn_fs_x/low_level.c new file mode 100644 index 0000000..01489ce --- /dev/null +++ b/subversion/libsvn_fs_x/low_level.c
@@ -0,0 +1,1038 @@ +/* low_level.c --- low level r/w access to fs_x file structures + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "svn_private_config.h" +#include "svn_hash.h" +#include "svn_pools.h" +#include "svn_sorts.h" +#include "private/svn_string_private.h" + +#include "../libsvn_fs/fs-loader.h" + +#include "low_level.h" +#include "util.h" +#include "pack.h" +#include "cached_data.h" + +/* Headers used to describe node-revision in the revision file. */ +#define HEADER_ID "id" +#define HEADER_TYPE "type" +#define HEADER_COUNT "count" +#define HEADER_PROPS "props" +#define HEADER_TEXT "text" +#define HEADER_CPATH "cpath" +#define HEADER_PRED "pred" +#define HEADER_COPYFROM "copyfrom" +#define HEADER_COPYROOT "copyroot" +#define HEADER_FRESHTXNRT "is-fresh-txn-root" +#define HEADER_MINFO_HERE "minfo-here" +#define HEADER_MINFO_CNT "minfo-cnt" + +/* Kinds that a change can be. */ +#define ACTION_MODIFY "modify" +#define ACTION_ADD "add" +#define ACTION_DELETE "delete" +#define ACTION_REPLACE "replace" +#define ACTION_RESET "reset" + +/* True and False flags. */ +#define FLAG_TRUE "true" +#define FLAG_FALSE "false" + +/* Kinds of representation. */ +#define REP_DELTA "DELTA" + +/* An arbitrary maximum path length, so clients can't run us out of memory + * by giving us arbitrarily large paths. */ +#define FSX_MAX_PATH_LEN 4096 + +/* The 256 is an arbitrary size large enough to hold the node id and the + * various flags. */ +#define MAX_CHANGE_LINE_LEN FSX_MAX_PATH_LEN + 256 + +svn_error_t * +svn_fs_x__parse_revision_trailer(apr_off_t *root_offset, + apr_off_t *changes_offset, + svn_stringbuf_t *trailer, + svn_revnum_t rev) +{ + int i, num_bytes; + const char *str; + + /* This cast should be safe since the maximum amount read, 64, will + never be bigger than the size of an int. */ + num_bytes = (int) trailer->len; + + /* The last byte should be a newline. */ + if (trailer->len == 0 || trailer->data[trailer->len - 1] != '\n') + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Revision file (r%ld) lacks trailing newline"), + rev); + } + + /* Look for the next previous newline. */ + for (i = num_bytes - 2; i >= 0; i--) + { + if (trailer->data[i] == '\n') + break; + } + + if (i < 0) + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Final line in revision file (r%ld) longer " + "than 64 characters"), + rev); + } + + i++; + str = &trailer->data[i]; + + /* find the next space */ + for ( ; i < (num_bytes - 2) ; i++) + if (trailer->data[i] == ' ') + break; + + if (i == (num_bytes - 2)) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Final line in revision file r%ld missing space"), + rev); + + if (root_offset) + { + apr_int64_t val; + + trailer->data[i] = '\0'; + SVN_ERR(svn_cstring_atoi64(&val, str)); + *root_offset = (apr_off_t)val; + } + + i++; + str = &trailer->data[i]; + + /* find the next newline */ + for ( ; i < num_bytes; i++) + if (trailer->data[i] == '\n') + break; + + if (changes_offset) + { + apr_int64_t val; + + trailer->data[i] = '\0'; + SVN_ERR(svn_cstring_atoi64(&val, str)); + *changes_offset = (apr_off_t)val; + } + + return SVN_NO_ERROR; +} + +svn_stringbuf_t * +svn_fs_x__unparse_revision_trailer(apr_off_t root_offset, + apr_off_t changes_offset, + apr_pool_t *pool) +{ + return svn_stringbuf_createf(pool, + "%" APR_OFF_T_FMT " %" APR_OFF_T_FMT "\n", + root_offset, + changes_offset); +} + +/* Given a revision file FILE that has been pre-positioned at the + beginning of a Node-Rev header block, read in that header block and + store it in the apr_hash_t HEADERS. All allocations will be from + POOL. */ +static svn_error_t * +read_header_block(apr_hash_t **headers, + svn_stream_t *stream, + apr_pool_t *pool) +{ + *headers = apr_hash_make(pool); + + while (1) + { + svn_stringbuf_t *header_str; + const char *name, *value; + apr_size_t i = 0; + svn_boolean_t eof; + + SVN_ERR(svn_stream_readline(stream, &header_str, "\n", &eof, pool)); + + if (eof || header_str->len == 0) + break; /* end of header block */ + + while (header_str->data[i] != ':') + { + if (header_str->data[i] == '\0') + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Found malformed header '%s' in " + "revision file"), + header_str->data); + i++; + } + + /* Create a 'name' string and point to it. */ + header_str->data[i] = '\0'; + name = header_str->data; + + /* Skip over the NULL byte and the space following it. */ + i += 2; + + if (i > header_str->len) + { + /* Restore the original line for the error. */ + i -= 2; + header_str->data[i] = ':'; + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Found malformed header '%s' in " + "revision file"), + header_str->data); + } + + value = header_str->data + i; + + /* header_str is safely in our pool, so we can use bits of it as + key and value. */ + svn_hash_sets(*headers, name, value); + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__parse_representation(representation_t **rep_p, + svn_stringbuf_t *text, + apr_pool_t *pool) +{ + representation_t *rep; + char *str; + apr_int64_t val; + char *string = text->data; + svn_checksum_t *checksum; + + rep = apr_pcalloc(pool, sizeof(*rep)); + *rep_p = rep; + + str = svn_cstring_tokenize(" ", &string); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed text representation offset line in node-rev")); + + + rep->revision = SVN_STR_TO_REV(str); + + /* initialize transaction info (never stored) */ + svn_fs_x__id_txn_reset(&rep->txn_id); + + /* while in transactions, it is legal to simply write "-1" */ + str = svn_cstring_tokenize(" ", &string); + if (str == NULL) + { + if (rep->revision == SVN_INVALID_REVNUM) + return SVN_NO_ERROR; + + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed text representation offset line in node-rev")); + } + + SVN_ERR(svn_cstring_atoi64(&val, str)); + rep->item_index = (apr_off_t)val; + + str = svn_cstring_tokenize(" ", &string); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed text representation offset line in node-rev")); + + SVN_ERR(svn_cstring_atoi64(&val, str)); + rep->size = (svn_filesize_t)val; + + str = svn_cstring_tokenize(" ", &string); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed text representation offset line in node-rev")); + + SVN_ERR(svn_cstring_atoi64(&val, str)); + rep->expanded_size = (svn_filesize_t)val; + + /* Read in the MD5 hash. */ + str = svn_cstring_tokenize(" ", &string); + if ((str == NULL) || (strlen(str) != (APR_MD5_DIGESTSIZE * 2))) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed text representation offset line in node-rev")); + + SVN_ERR(svn_checksum_parse_hex(&checksum, svn_checksum_md5, str, pool)); + memcpy(rep->md5_digest, checksum->digest, sizeof(rep->md5_digest)); + + /* The remaining fields are only used for formats >= 4, so check that. */ + str = svn_cstring_tokenize(" ", &string); + if (str == NULL) + return SVN_NO_ERROR; + + /* Read the SHA1 hash. */ + if (strlen(str) != (APR_SHA1_DIGESTSIZE * 2)) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed text representation offset line in node-rev")); + + SVN_ERR(svn_checksum_parse_hex(&checksum, svn_checksum_sha1, str, pool)); + rep->has_sha1 = checksum != NULL; + memcpy(rep->sha1_digest, checksum->digest, sizeof(rep->sha1_digest)); + + /* Read the uniquifier. */ + str = svn_cstring_tokenize("/", &string); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed text representation offset line in node-rev")); + + SVN_ERR(svn_fs_x__id_txn_parse(&rep->uniquifier.txn_id, str)); + + str = svn_cstring_tokenize(" ", &string); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed text representation offset line in node-rev")); + + rep->uniquifier.number = svn__base36toui64(NULL, str); + + return SVN_NO_ERROR; +} + +/* Wrap read_rep_offsets_body(), extracting its TXN_ID from our NODEREV_ID, + and adding an error message. */ +static svn_error_t * +read_rep_offsets(representation_t **rep_p, + char *string, + const svn_fs_id_t *noderev_id, + apr_pool_t *pool) +{ + svn_error_t *err + = svn_fs_x__parse_representation(rep_p, + svn_stringbuf_create_wrap(string, pool), + pool); + if (err) + { + const svn_string_t *id_unparsed = svn_fs_x__id_unparse(noderev_id, pool); + const char *where; + where = apr_psprintf(pool, + _("While reading representation offsets " + "for node-revision '%s':"), + noderev_id ? id_unparsed->data : "(null)"); + + return svn_error_quick_wrap(err, where); + } + + if ((*rep_p)->revision == SVN_INVALID_REVNUM) + if (noderev_id) + (*rep_p)->txn_id = *svn_fs_x__id_txn_id(noderev_id); + + return SVN_NO_ERROR; +} + +static const char * +auto_escape_path(const char *path, + apr_pool_t *pool) +{ + apr_size_t len = strlen(path); + apr_size_t i; + const char esc = '\x1b'; + + for (i = 0; i < len; ++i) + if (path[i] < ' ') + { + svn_stringbuf_t *escaped = svn_stringbuf_create_ensure(2 * len, pool); + for (i = 0; i < len; ++i) + if (path[i] < ' ') + { + svn_stringbuf_appendbyte(escaped, esc); + svn_stringbuf_appendbyte(escaped, path[i] + 'A' - 1); + } + else + { + svn_stringbuf_appendbyte(escaped, path[i]); + } + + return escaped->data; + } + + return path; +} + +static const char * +auto_unescape_path(const char *path, + apr_pool_t *pool) +{ + const char esc = '\x1b'; + if (strchr(path, esc)) + { + apr_size_t len = strlen(path); + apr_size_t i; + + svn_stringbuf_t *unescaped = svn_stringbuf_create_ensure(len, pool); + for (i = 0; i < len; ++i) + if (path[i] == esc) + svn_stringbuf_appendbyte(unescaped, path[++i] + 1 - 'A'); + else + svn_stringbuf_appendbyte(unescaped, path[i]); + + return unescaped->data; + } + + return path; +} + +svn_error_t * +svn_fs_x__read_noderev(node_revision_t **noderev_p, + svn_stream_t *stream, + apr_pool_t *pool) +{ + apr_hash_t *headers; + node_revision_t *noderev; + char *value; + const char *noderev_id; + + SVN_ERR(read_header_block(&headers, stream, pool)); + + noderev = apr_pcalloc(pool, sizeof(*noderev)); + + /* Read the node-rev id. */ + value = svn_hash_gets(headers, HEADER_ID); + if (value == NULL) + /* ### More information: filename/offset coordinates */ + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Missing id field in node-rev")); + + SVN_ERR(svn_stream_close(stream)); + + noderev->id = svn_fs_x__id_parse(value, strlen(value), pool); + noderev_id = value; /* for error messages later */ + + /* Read the type. */ + value = svn_hash_gets(headers, HEADER_TYPE); + + if ((value == NULL) || + ( strcmp(value, SVN_FS_X__KIND_FILE) + && strcmp(value, SVN_FS_X__KIND_DIR))) + /* ### s/kind/type/ */ + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Missing kind field in node-rev '%s'"), + noderev_id); + + noderev->kind = (strcmp(value, SVN_FS_X__KIND_FILE) == 0) + ? svn_node_file + : svn_node_dir; + + /* Read the 'count' field. */ + value = svn_hash_gets(headers, HEADER_COUNT); + if (value) + SVN_ERR(svn_cstring_atoi(&noderev->predecessor_count, value)); + else + noderev->predecessor_count = 0; + + /* Get the properties location. */ + value = svn_hash_gets(headers, HEADER_PROPS); + if (value) + { + SVN_ERR(read_rep_offsets(&noderev->prop_rep, value, + noderev->id, pool)); + } + + /* Get the data location. */ + value = svn_hash_gets(headers, HEADER_TEXT); + if (value) + { + SVN_ERR(read_rep_offsets(&noderev->data_rep, value, + noderev->id, pool)); + } + + /* Get the created path. */ + value = svn_hash_gets(headers, HEADER_CPATH); + if (value == NULL) + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Missing cpath field in node-rev '%s'"), + noderev_id); + } + else + { + noderev->created_path = auto_unescape_path(value, pool); + } + + /* Get the predecessor ID. */ + value = svn_hash_gets(headers, HEADER_PRED); + if (value) + noderev->predecessor_id = svn_fs_x__id_parse(value, strlen(value), pool); + + /* Get the copyroot. */ + value = svn_hash_gets(headers, HEADER_COPYROOT); + if (value == NULL) + { + noderev->copyroot_path = noderev->created_path; + noderev->copyroot_rev = svn_fs_x__id_rev(noderev->id); + } + else + { + char *str; + + str = svn_cstring_tokenize(" ", &value); + if (str == NULL) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed copyroot line in node-rev '%s'"), + noderev_id); + + noderev->copyroot_rev = SVN_STR_TO_REV(str); + + if (*value == '\0') + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed copyroot line in node-rev '%s'"), + noderev_id); + noderev->copyroot_path = auto_unescape_path(value, pool); + } + + /* Get the copyfrom. */ + value = svn_hash_gets(headers, HEADER_COPYFROM); + if (value == NULL) + { + noderev->copyfrom_path = NULL; + noderev->copyfrom_rev = SVN_INVALID_REVNUM; + } + else + { + char *str = svn_cstring_tokenize(" ", &value); + if (str == NULL) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed copyfrom line in node-rev '%s'"), + noderev_id); + + noderev->copyfrom_rev = SVN_STR_TO_REV(str); + + if (*value == 0) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed copyfrom line in node-rev '%s'"), + noderev_id); + noderev->copyfrom_path = auto_unescape_path(value, pool); + } + + /* Get whether this is a fresh txn root. */ + value = svn_hash_gets(headers, HEADER_FRESHTXNRT); + noderev->is_fresh_txn_root = (value != NULL); + + /* Get the mergeinfo count. */ + value = svn_hash_gets(headers, HEADER_MINFO_CNT); + if (value) + SVN_ERR(svn_cstring_atoi64(&noderev->mergeinfo_count, value)); + else + noderev->mergeinfo_count = 0; + + /* Get whether *this* node has mergeinfo. */ + value = svn_hash_gets(headers, HEADER_MINFO_HERE); + noderev->has_mergeinfo = (value != NULL); + + *noderev_p = noderev; + + return SVN_NO_ERROR; +} + +/* Return a textual representation of the DIGEST of given KIND. + * If IS_NULL is TRUE, no digest is available. + * Use POOL for allocations. + */ +static const char * +format_digest(const unsigned char *digest, + svn_checksum_kind_t kind, + svn_boolean_t is_null, + apr_pool_t *pool) +{ + svn_checksum_t checksum; + checksum.digest = digest; + checksum.kind = kind; + + if (is_null) + return "(null)"; + + return svn_checksum_to_cstring_display(&checksum, pool); +} + +svn_stringbuf_t * +svn_fs_x__unparse_representation(representation_t *rep, + int format, + svn_boolean_t mutable_rep_truncated, + apr_pool_t *pool) +{ + char buffer[SVN_INT64_BUFFER_SIZE]; + if (svn_fs_x__id_txn_used(&rep->txn_id) && mutable_rep_truncated) + return svn_stringbuf_ncreate("-1", 2, pool); + + if (!rep->has_sha1) + return svn_stringbuf_createf + (pool, "%ld %" APR_OFF_T_FMT " %" SVN_FILESIZE_T_FMT + " %" SVN_FILESIZE_T_FMT " %s", + rep->revision, rep->item_index, rep->size, + rep->expanded_size, + format_digest(rep->md5_digest, svn_checksum_md5, FALSE, pool)); + + svn__ui64tobase36(buffer, rep->uniquifier.number); + return svn_stringbuf_createf + (pool, "%ld %" APR_OFF_T_FMT " %" SVN_FILESIZE_T_FMT + " %" SVN_FILESIZE_T_FMT " %s %s %s/%s", + rep->revision, rep->item_index, rep->size, + rep->expanded_size, + format_digest(rep->md5_digest, svn_checksum_md5, FALSE, pool), + format_digest(rep->sha1_digest, svn_checksum_sha1, + !rep->has_sha1, pool), + svn_fs_x__id_txn_unparse(&rep->uniquifier.txn_id, pool), + buffer); + +#undef DISPLAY_MAYBE_NULL_CHECKSUM + +} + + +svn_error_t * +svn_fs_x__write_noderev(svn_stream_t *outfile, + node_revision_t *noderev, + int format, + apr_pool_t *pool) +{ + SVN_ERR(svn_stream_printf(outfile, pool, HEADER_ID ": %s\n", + svn_fs_x__id_unparse(noderev->id, pool)->data)); + + SVN_ERR(svn_stream_printf(outfile, pool, HEADER_TYPE ": %s\n", + (noderev->kind == svn_node_file) ? + SVN_FS_X__KIND_FILE : SVN_FS_X__KIND_DIR)); + + if (noderev->predecessor_id) + SVN_ERR(svn_stream_printf(outfile, pool, HEADER_PRED ": %s\n", + svn_fs_x__id_unparse(noderev->predecessor_id, + pool)->data)); + + SVN_ERR(svn_stream_printf(outfile, pool, HEADER_COUNT ": %d\n", + noderev->predecessor_count)); + + if (noderev->data_rep) + SVN_ERR(svn_stream_printf(outfile, pool, HEADER_TEXT ": %s\n", + svn_fs_x__unparse_representation + (noderev->data_rep, + format, + noderev->kind == svn_node_dir, + pool)->data)); + + if (noderev->prop_rep) + SVN_ERR(svn_stream_printf(outfile, pool, HEADER_PROPS ": %s\n", + svn_fs_x__unparse_representation + (noderev->prop_rep, format, + TRUE, pool)->data)); + + SVN_ERR(svn_stream_printf(outfile, pool, HEADER_CPATH ": %s\n", + auto_escape_path(noderev->created_path, pool))); + + if (noderev->copyfrom_path) + SVN_ERR(svn_stream_printf(outfile, pool, HEADER_COPYFROM ": %ld" + " %s\n", + noderev->copyfrom_rev, + auto_escape_path(noderev->copyfrom_path, pool))); + + if ((noderev->copyroot_rev != svn_fs_x__id_rev(noderev->id)) || + (strcmp(noderev->copyroot_path, noderev->created_path) != 0)) + SVN_ERR(svn_stream_printf(outfile, pool, HEADER_COPYROOT ": %ld" + " %s\n", + noderev->copyroot_rev, + auto_escape_path(noderev->copyroot_path, pool))); + + if (noderev->is_fresh_txn_root) + SVN_ERR(svn_stream_puts(outfile, HEADER_FRESHTXNRT ": y\n")); + + if (noderev->mergeinfo_count > 0) + SVN_ERR(svn_stream_printf(outfile, pool, HEADER_MINFO_CNT ": %" + APR_INT64_T_FMT "\n", + noderev->mergeinfo_count)); + + if (noderev->has_mergeinfo) + SVN_ERR(svn_stream_puts(outfile, HEADER_MINFO_HERE ": y\n")); + + return svn_stream_puts(outfile, "\n"); +} + +svn_error_t * +svn_fs_x__read_rep_header(svn_fs_x__rep_header_t **header, + svn_stream_t *stream, + apr_pool_t *pool) +{ + svn_stringbuf_t *buffer; + char *str, *last_str; + apr_int64_t val; + svn_boolean_t eol = FALSE; + + SVN_ERR(svn_stream_readline(stream, &buffer, "\n", &eol, pool)); + + *header = apr_pcalloc(pool, sizeof(**header)); + (*header)->header_size = buffer->len + 1; + if (strcmp(buffer->data, REP_DELTA) == 0) + { + /* This is a delta against the empty stream. */ + (*header)->type = svn_fs_x__rep_self_delta; + return SVN_NO_ERROR; + } + + (*header)->type = svn_fs_x__rep_delta; + + /* We have hopefully a DELTA vs. a non-empty base revision. */ + last_str = buffer->data; + str = svn_cstring_tokenize(" ", &last_str); + if (! str || (strcmp(str, REP_DELTA) != 0)) + goto error; + + str = svn_cstring_tokenize(" ", &last_str); + if (! str) + goto error; + (*header)->base_revision = SVN_STR_TO_REV(str); + + str = svn_cstring_tokenize(" ", &last_str); + if (! str) + goto error; + SVN_ERR(svn_cstring_atoi64(&val, str)); + (*header)->base_item_index = (apr_off_t)val; + + str = svn_cstring_tokenize(" ", &last_str); + if (! str) + goto error; + SVN_ERR(svn_cstring_atoi64(&val, str)); + (*header)->base_length = (svn_filesize_t)val; + + return SVN_NO_ERROR; + + error: + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed representation header")); +} + +svn_error_t * +svn_fs_x__write_rep_header(svn_fs_x__rep_header_t *header, + svn_stream_t *stream, + apr_pool_t *pool) +{ + const char *text; + + switch (header->type) + { + case svn_fs_x__rep_self_delta: + text = REP_DELTA "\n"; + break; + + default: + text = apr_psprintf(pool, REP_DELTA " %ld %" APR_OFF_T_FMT " %" + SVN_FILESIZE_T_FMT "\n", + header->base_revision, header->base_item_index, + header->base_length); + } + + return svn_error_trace(svn_stream_puts(stream, text)); +} + +/* Read the next entry in the changes record from file FILE and store + the resulting change in *CHANGE_P. If there is no next record, + store NULL there. Perform all allocations from POOL. */ +static svn_error_t * +read_change(change_t **change_p, + svn_stream_t *stream, + apr_pool_t *pool) +{ + svn_stringbuf_t *line; + svn_boolean_t eof = TRUE; + change_t *change; + char *str, *last_str, *kind_str; + svn_fs_path_change2_t *info; + + /* Default return value. */ + *change_p = NULL; + + SVN_ERR(svn_stream_readline(stream, &line, "\n", &eof, pool)); + + /* Check for a blank line. */ + if (eof || (line->len == 0)) + return SVN_NO_ERROR; + + change = apr_pcalloc(pool, sizeof(*change)); + info = &change->info; + last_str = line->data; + + /* Get the node-id of the change. */ + str = svn_cstring_tokenize(" ", &last_str); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid changes line in rev-file")); + + info->node_rev_id = svn_fs_x__id_parse(str, strlen(str), pool); + if (info->node_rev_id == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid changes line in rev-file")); + + /* Get the change type. */ + str = svn_cstring_tokenize(" ", &last_str); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid changes line in rev-file")); + + /* Don't bother to check the format number before looking for + * node-kinds: just read them if you find them. */ + info->node_kind = svn_node_unknown; + kind_str = strchr(str, '-'); + if (kind_str) + { + /* Cap off the end of "str" (the action). */ + *kind_str = '\0'; + kind_str++; + if (strcmp(kind_str, SVN_FS_X__KIND_FILE) == 0) + info->node_kind = svn_node_file; + else if (strcmp(kind_str, SVN_FS_X__KIND_DIR) == 0) + info->node_kind = svn_node_dir; + else + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid changes line in rev-file")); + } + + if (strcmp(str, ACTION_MODIFY) == 0) + { + info->change_kind = svn_fs_path_change_modify; + } + else if (strcmp(str, ACTION_ADD) == 0) + { + info->change_kind = svn_fs_path_change_add; + } + else if (strcmp(str, ACTION_DELETE) == 0) + { + info->change_kind = svn_fs_path_change_delete; + } + else if (strcmp(str, ACTION_REPLACE) == 0) + { + info->change_kind = svn_fs_path_change_replace; + } + else if (strcmp(str, ACTION_RESET) == 0) + { + info->change_kind = svn_fs_path_change_reset; + } + else + { + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid change kind in rev file")); + } + + /* Get the text-mod flag. */ + str = svn_cstring_tokenize(" ", &last_str); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid changes line in rev-file")); + + if (strcmp(str, FLAG_TRUE) == 0) + { + info->text_mod = TRUE; + } + else if (strcmp(str, FLAG_FALSE) == 0) + { + info->text_mod = FALSE; + } + else + { + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid text-mod flag in rev-file")); + } + + /* Get the prop-mod flag. */ + str = svn_cstring_tokenize(" ", &last_str); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid changes line in rev-file")); + + if (strcmp(str, FLAG_TRUE) == 0) + { + info->prop_mod = TRUE; + } + else if (strcmp(str, FLAG_FALSE) == 0) + { + info->prop_mod = FALSE; + } + else + { + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid prop-mod flag in rev-file")); + } + + /* Get the changed path. */ + change->path.data = auto_unescape_path(apr_pstrmemdup(pool, last_str, + strlen(last_str)), + pool); + change->path.len = strlen(change->path.data); + + /* Read the next line, the copyfrom line. */ + SVN_ERR(svn_stream_readline(stream, &line, "\n", &eof, pool)); + info->copyfrom_known = TRUE; + if (eof || line->len == 0) + { + info->copyfrom_rev = SVN_INVALID_REVNUM; + info->copyfrom_path = NULL; + } + else + { + last_str = line->data; + str = svn_cstring_tokenize(" ", &last_str); + if (! str) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid changes line in rev-file")); + info->copyfrom_rev = SVN_STR_TO_REV(str); + + if (! last_str) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid changes line in rev-file")); + + info->copyfrom_path = auto_unescape_path(last_str, pool); + } + + *change_p = change; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__read_changes(apr_array_header_t **changes, + svn_stream_t *stream, + apr_pool_t *pool) +{ + change_t *change; + + /* pre-allocate enough room for most change lists + (will be auto-expanded as necessary) */ + *changes = apr_array_make(pool, 30, sizeof(change_t *)); + + SVN_ERR(read_change(&change, stream, pool)); + while (change) + { + APR_ARRAY_PUSH(*changes, change_t*) = change; + SVN_ERR(read_change(&change, stream, pool)); + } + + return SVN_NO_ERROR; +} + +/* Write a single change entry, path PATH, change CHANGE, and copyfrom + string COPYFROM, into the file specified by FILE. Only include the + node kind field if INCLUDE_NODE_KIND is true. All temporary + allocations are in POOL. */ +static svn_error_t * +write_change_entry(svn_stream_t *stream, + const char *path, + svn_fs_path_change2_t *change, + apr_pool_t *pool) +{ + const char *idstr, *buf; + const char *change_string = NULL; + const char *kind_string = ""; + + switch (change->change_kind) + { + case svn_fs_path_change_modify: + change_string = ACTION_MODIFY; + break; + case svn_fs_path_change_add: + change_string = ACTION_ADD; + break; + case svn_fs_path_change_delete: + change_string = ACTION_DELETE; + break; + case svn_fs_path_change_replace: + change_string = ACTION_REPLACE; + break; + case svn_fs_path_change_reset: + change_string = ACTION_RESET; + break; + default: + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid change type %d"), + change->change_kind); + } + + if (change->node_rev_id) + idstr = svn_fs_x__id_unparse(change->node_rev_id, pool)->data; + else + idstr = ACTION_RESET; + + SVN_ERR_ASSERT(change->node_kind == svn_node_dir + || change->node_kind == svn_node_file); + kind_string = apr_psprintf(pool, "-%s", + change->node_kind == svn_node_dir + ? SVN_FS_X__KIND_DIR + : SVN_FS_X__KIND_FILE); + buf = apr_psprintf(pool, "%s %s%s %s %s %s\n", + idstr, change_string, kind_string, + change->text_mod ? FLAG_TRUE : FLAG_FALSE, + change->prop_mod ? FLAG_TRUE : FLAG_FALSE, + auto_escape_path(path, pool)); + + SVN_ERR(svn_stream_puts(stream, buf)); + + if (SVN_IS_VALID_REVNUM(change->copyfrom_rev)) + { + buf = apr_psprintf(pool, "%ld %s", change->copyfrom_rev, + auto_escape_path(change->copyfrom_path, pool)); + SVN_ERR(svn_stream_puts(stream, buf)); + } + + return svn_error_trace(svn_stream_puts(stream, "\n")); +} + +svn_error_t * +svn_fs_x__write_changes(svn_stream_t *stream, + svn_fs_t *fs, + apr_hash_t *changes, + svn_boolean_t terminate_list, + apr_pool_t *pool) +{ + apr_pool_t *iterpool = svn_pool_create(pool); + apr_array_header_t *sorted_changed_paths; + int i; + + /* For the sake of the repository administrator sort the changes so + that the final file is deterministic and repeatable, however the + rest of the FSX code doesn't require any particular order here. */ + sorted_changed_paths = svn_sort__hash(changes, + svn_sort_compare_items_lexically, pool); + + /* Write all items to disk in the new order. */ + for (i = 0; i < sorted_changed_paths->nelts; ++i) + { + svn_fs_path_change2_t *change; + const char *path; + + svn_pool_clear(iterpool); + + change = APR_ARRAY_IDX(sorted_changed_paths, i, svn_sort__item_t).value; + path = APR_ARRAY_IDX(sorted_changed_paths, i, svn_sort__item_t).key; + + /* Write out the new entry into the final rev-file. */ + SVN_ERR(write_change_entry(stream, path, change, iterpool)); + } + + if (terminate_list) + svn_stream_puts(stream, "\n"); + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} +
diff --git a/subversion/libsvn_fs_x/low_level.h b/subversion/libsvn_fs_x/low_level.h new file mode 100644 index 0000000..4ef83ad --- /dev/null +++ b/subversion/libsvn_fs_x/low_level.h
@@ -0,0 +1,161 @@ +/* low_level.c --- low level r/w access to fs_x file structures + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "svn_fs.h" + +#include "fs_x.h" +#include "id.h" + +/* Kinds that a node-rev can be. */ +#define SVN_FS_X__KIND_FILE "file" +#define SVN_FS_X__KIND_DIR "dir" + +/* Given the last "few" bytes (should be at least 40) of revision REV in + * TRAILER, parse the last line and return the offset of the root noderev + * in *ROOT_OFFSET and the offset of the changed paths list in + * *CHANGES_OFFSET. Offsets are relative to the revision's start offset. + * ROOT_OFFSET and / or CHANGES_OFFSET may be NULL. + * + * Note that REV is only used to construct nicer error objects. + */ +svn_error_t * +svn_fs_x__parse_revision_trailer(apr_off_t *root_offset, + apr_off_t *changes_offset, + svn_stringbuf_t *trailer, + svn_revnum_t rev); + +/* Given the offset of the root noderev in ROOT_OFFSET and the offset of + * the changed paths list in CHANGES_OFFSET, return the corresponding + * revision's trailer. Allocate it in POOL. + */ +svn_stringbuf_t * +svn_fs_x__unparse_revision_trailer(apr_off_t root_offset, + apr_off_t changes_offset, + apr_pool_t *pool); + +/* Parse the description of a representation from TEXT and store it + into *REP_P. Allocate *REP_P in POOL. */ +svn_error_t * +svn_fs_x__parse_representation(representation_t **rep_p, + svn_stringbuf_t *text, + apr_pool_t *pool); + +/* Return a formatted string, compatible with filesystem format FORMAT, + that represents the location of representation REP. If + MUTABLE_REP_TRUNCATED is given, the rep is for props or dir contents, + and only a "-1" revision number will be given for a mutable rep. + If MAY_BE_CORRUPT is true, guard for NULL when constructing the string. + Perform the allocation from POOL. */ +svn_stringbuf_t * +svn_fs_x__unparse_representation(representation_t *rep, + int format, + svn_boolean_t mutable_rep_truncated, + apr_pool_t *pool); + +/* Read a node-revision from STREAM. Set *NODEREV to the new structure, + allocated in POOL. */ +svn_error_t * +svn_fs_x__read_noderev(node_revision_t **noderev, + svn_stream_t *stream, + apr_pool_t *pool); + +/* Write the node-revision NODEREV into the stream OUTFILE, compatible with + filesystem format FORMAT. Temporary allocations are from POOL. */ +svn_error_t * +svn_fs_x__write_noderev(svn_stream_t *outfile, + node_revision_t *noderev, + int format, + apr_pool_t *pool); + +/* This type enumerates all forms of representations that we support. */ +typedef enum svn_fs_x__rep_type_t +{ + /* this is a DELTA representation with no base representation */ + svn_fs_x__rep_self_delta, + + /* this is a DELTA representation against some base representation */ + svn_fs_x__rep_delta, + + /* this is a representation in a star-delta container */ + svn_fs_x__rep_container +} svn_fs_x__rep_type_t; + +/* This structure is used to hold the information stored in a representation + * header. */ +typedef struct svn_fs_x__rep_header_t +{ + /* type of the representation, i.e. whether self-DELTA etc. */ + svn_fs_x__rep_type_t type; + + /* if this rep is a delta against some other rep, that base rep can + * be found in this revision. Should be 0 if there is no base rep. */ + svn_revnum_t base_revision; + + /* if this rep is a delta against some other rep, that base rep can + * be found at this item index within the base rep's revision. Should + * be 0 if there is no base rep. */ + apr_off_t base_item_index; + + /* if this rep is a delta against some other rep, this is the (deltified) + * size of that base rep. Should be 0 if there is no base rep. */ + svn_filesize_t base_length; + + /* length of the textual representation of the header in the rep or pack + * file, including EOL. Only valid after reading it from disk. + * Should be 0 otherwise. */ + apr_size_t header_size; +} svn_fs_x__rep_header_t; + +/* Read the next line from file FILE and parse it as a text + representation entry. Return the parsed entry in *REP_ARGS_P. + Perform all allocations in POOL. */ +svn_error_t * +svn_fs_x__read_rep_header(svn_fs_x__rep_header_t **header, + svn_stream_t *stream, + apr_pool_t *pool); + +/* Write the representation HEADER to STREAM. Use POOL for allocations. */ +svn_error_t * +svn_fs_x__write_rep_header(svn_fs_x__rep_header_t *header, + svn_stream_t *stream, + apr_pool_t *pool); + +/* Read all the changes from STREAM and store them in *CHANGES. Do all + allocations in POOL. */ +svn_error_t * +svn_fs_x__read_changes(apr_array_header_t **changes, + svn_stream_t *stream, + apr_pool_t *pool); + +/* Write the changed path info from CHANGES in filesystem FS to the + output stream STREAM. You may call this function multiple time on + the same stream but the last call should set TERMINATE_LIST to write + an extra empty line that marks the end of the changed paths list. + Perform temporary allocations in POOL. + */ +svn_error_t * +svn_fs_x__write_changes(svn_stream_t *stream, + svn_fs_t *fs, + apr_hash_t *changes, + svn_boolean_t terminate_list, + apr_pool_t *pool); +
diff --git a/subversion/libsvn_fs_x/noderevs.c b/subversion/libsvn_fs_x/noderevs.c new file mode 100644 index 0000000..6f4f42f --- /dev/null +++ b/subversion/libsvn_fs_x/noderevs.c
@@ -0,0 +1,1015 @@ +/* noderevs.h --- FSX node revision container + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "svn_private_config.h" + +#include "private/svn_packed_data.h" +#include "private/svn_subr_private.h" +#include "private/svn_temp_serializer.h" + +#include "noderevs.h" +#include "string_table.h" +#include "temp_serializer.h" + +/* These flags will be used with the FLAGS field in binary_noderev_t. + */ + +/* (flags & NODEREV_KIND_MASK) extracts the noderev type */ +#define NODEREV_KIND_MASK 0x00007 + +/* the noderev has merge info */ +#define NODEREV_HAS_MINFO 0x00008 + +/* the noderev has copy-from-path and revision */ +#define NODEREV_HAS_COPYFROM 0x00010 + +/* the noderev has copy-root path and revision */ +#define NODEREV_HAS_COPYROOT 0x00020 + +/* the noderev has copy-root path and revision */ +#define NODEREV_HAS_CPATH 0x00040 + +/* Our internal representation of an id + * (basically, strip off the txn_id and the fs-agnostic header) + */ +typedef struct binary_id_t +{ + svn_fs_x__id_part_t node_id; + svn_fs_x__id_part_t copy_id; + svn_fs_x__id_part_t rev_id; +} binary_id_t; + +/* Our internal representation of an representation. + * We simply omit the uniquifier, which allows us to share instances of + * binary_representation_t and uniquify them in a shared_representation_t. + */ +typedef struct binary_representation_t +{ + /* Checksums digests for the contents produced by this representation. + If has_sha1 is FALSE, sha1_digest is not being used. */ + svn_boolean_t has_sha1; + unsigned char sha1_digest[APR_SHA1_DIGESTSIZE]; + unsigned char md5_digest[APR_MD5_DIGESTSIZE]; + + /* Revision where this representation is located. */ + svn_revnum_t revision; + + /* Item index with the the revision. */ + apr_uint64_t item_index; + + /* The size of the representation in bytes as seen in the revision + file. */ + svn_filesize_t size; + + /* The size of the fulltext of the representation. If this is 0, + * the fulltext size is equal to representation size in the rev file, */ + svn_filesize_t expanded_size; +} binary_representation_t; + +/* Add a uniquifier to binary_representation_t to distinguish between + * multiple uses of the same shared representation. + */ +typedef struct shared_representation_t +{ + /* For rep-sharing, we need a way of uniquifying node-revs which share the + same representation (see svn_fs_x__noderev_same_rep_key() ). So, we + store the original txn of the node rev (not the rep!), along with some + intra-node uniqification content. */ + struct + { + svn_fs_x__id_part_t txn_id; + apr_uint64_t number; + } uniquifier; + + /* Index+1 of the representation. + */ + int representation; +} shared_representation_t; + + +/* Our internal representation of a node_revision_t. + * + * We will store path strings in a string container and reference them + * from here. Similarly, IDs and representations are being stored in + * separate containers and then also referenced here. This eliminates + * the need to store the same IDs and representations more than once. + */ +typedef struct binary_noderev_t +{ + /* node type and presence indicators */ + apr_uint32_t flags; + + /* Index+1 of the node-id for this node-rev. */ + int id; + + /* Index+1 of the predecessor node revision id, or 0 if there is no + predecessor for this node revision */ + int predecessor_id; + + /* number of predecessors this node revision has (recursively), or + -1 if not known (for backward compatibility). */ + int predecessor_count; + + /* If this node-rev is a copy, what revision was it copied from? */ + svn_revnum_t copyfrom_rev; + + /* Helper for history tracing, root revision of the parent tree from + whence this node-rev was copied. */ + svn_revnum_t copyroot_rev; + + /* If this node-rev is a copy, this is the string index+1 of the path + from which that copy way made. 0, otherwise. */ + int copyfrom_path; + + /* String index+1 of the root of the parent tree from whence this node- + * rev was copied. */ + int copyroot_path; + + /* Index+1 of the representation key for this node's properties. + May be 0 if there are no properties. */ + int prop_rep; + + /* Index+1 of the representation for this node's data. + May be 0 if there is no data. */ + shared_representation_t data_rep; + + /* String index+1 of the path at which this node first came into + existence. */ + int created_path; + + /* Number of nodes with svn:mergeinfo properties that are + descendants of this node (including it itself) */ + apr_int64_t mergeinfo_count; + +} binary_noderev_t; + +/* The actual container object. Node revisions are concatenated into + * NODEREVS, referenced representations are stored in DATA_REPS / PROP_REPS + * and the ids in IDs. PATHS is the string table for all paths. + * + * During construction, BUILDER will be used instead of PATHS. IDS_DICT, + * DATA_REPS_DICT and PROP_REPS_DICT are also only used during construction + * and are NULL otherwise. + */ +struct svn_fs_x__noderevs_t +{ + /* The paths - either in 'builder' mode or finalized mode. + * The respective other pointer will be NULL. */ + string_table_builder_t *builder; + string_table_t *paths; + + /* During construction, maps a full binary_id_t to an index into IDS */ + apr_hash_t *ids_dict; + + /* During construction, maps a full binary_representation_t to an index + * into DATA_REPS. */ + apr_hash_t *data_reps_dict; + + /* During construction, maps a full binary_representation_t to an index + * into PROP_REPS. */ + apr_hash_t *prop_reps_dict; + + /* array of binary_id_t */ + apr_array_header_t *ids; + + /* arrays of binary_representation_t */ + apr_array_header_t *data_reps; + apr_array_header_t *prop_reps; + + /* array of binary_noderev_t. */ + apr_array_header_t *noderevs; +}; + +svn_fs_x__noderevs_t * +svn_fs_x__noderevs_create(apr_size_t initial_count, + apr_pool_t* pool) +{ + svn_fs_x__noderevs_t *noderevs = apr_palloc(pool, sizeof(*noderevs)); + + noderevs->builder = svn_fs_x__string_table_builder_create(pool); + noderevs->ids_dict = svn_hash__make(pool); + noderevs->data_reps_dict = svn_hash__make(pool); + noderevs->prop_reps_dict = svn_hash__make(pool); + noderevs->paths = NULL; + + noderevs->ids + = apr_array_make(pool, initial_count, sizeof(binary_id_t)); + noderevs->data_reps + = apr_array_make(pool, initial_count, sizeof(binary_representation_t)); + noderevs->prop_reps + = apr_array_make(pool, initial_count, sizeof(binary_representation_t)); + noderevs->noderevs + = apr_array_make(pool, initial_count, sizeof(binary_noderev_t)); + + return noderevs; +} + +/* Given the ID, return the index+1 into IDS that contains a binary_id + * for it. Returns 0 for NULL IDs. We use DICT to detect duplicates. + */ +static int +store_id(apr_array_header_t *ids, + apr_hash_t *dict, + const svn_fs_id_t *id) +{ + binary_id_t bin_id = { { 0 } }; + int idx; + + if (id == NULL) + return 0; + + bin_id.node_id = *svn_fs_x__id_node_id(id); + bin_id.copy_id = *svn_fs_x__id_copy_id(id); + bin_id.rev_id = *svn_fs_x__id_rev_item(id); + + idx = (int)(apr_uintptr_t)apr_hash_get(dict, &bin_id, sizeof(bin_id)); + if (idx == 0) + { + APR_ARRAY_PUSH(ids, binary_id_t) = bin_id; + idx = ids->nelts; + apr_hash_set(dict, ids->elts + (idx-1) * ids->elt_size, + ids->elt_size, (void*)(apr_uintptr_t)idx); + } + + return idx; +} + +/* Given the REP, return the index+1 into REPS that contains a copy of it. + * Returns 0 for NULL IDs. We use DICT to detect duplicates. + */ +static int +store_representation(apr_array_header_t *reps, + apr_hash_t *dict, + const representation_t *rep) +{ + binary_representation_t binary_rep = { 0 }; + int idx; + + if (rep == NULL) + return 0; + + binary_rep.has_sha1 = rep->has_sha1; + memcpy(binary_rep.sha1_digest, rep->sha1_digest, sizeof(rep->sha1_digest)); + memcpy(binary_rep.md5_digest, rep->md5_digest, sizeof(rep->md5_digest)); + binary_rep.revision = rep->revision; + binary_rep.item_index = rep->item_index; + binary_rep.size = rep->size; + binary_rep.expanded_size = rep->expanded_size; + + idx = (int)(apr_uintptr_t)apr_hash_get(dict, &binary_rep, + sizeof(binary_rep)); + if (idx == 0) + { + APR_ARRAY_PUSH(reps, binary_representation_t) = binary_rep; + idx = reps->nelts; + apr_hash_set(dict, reps->elts + (idx-1) * reps->elt_size, + reps->elt_size, (void*)(apr_uintptr_t)idx); + } + + return idx; +} + +apr_size_t +svn_fs_x__noderevs_add(svn_fs_x__noderevs_t *container, + node_revision_t *noderev) +{ + binary_noderev_t binary_noderev = { 0 }; + + binary_noderev.flags = (noderev->has_mergeinfo ? NODEREV_HAS_MINFO : 0) + | (noderev->copyfrom_path ? NODEREV_HAS_COPYFROM : 0) + | (noderev->copyroot_path ? NODEREV_HAS_COPYROOT : 0) + | (noderev->created_path ? NODEREV_HAS_CPATH : 0) + | (int)noderev->kind; + + binary_noderev.id + = store_id(container->ids, container->ids_dict, noderev->id); + binary_noderev.predecessor_id + = store_id(container->ids, container->ids_dict, noderev->predecessor_id); + + if (noderev->copyfrom_path) + { + binary_noderev.copyfrom_path + = svn_fs_x__string_table_builder_add(container->builder, + noderev->copyfrom_path, + 0); + binary_noderev.copyfrom_rev = noderev->copyfrom_rev; + } + + if (noderev->copyroot_path) + { + binary_noderev.copyroot_path + = svn_fs_x__string_table_builder_add(container->builder, + noderev->copyroot_path, + 0); + binary_noderev.copyroot_rev = noderev->copyroot_rev; + } + + binary_noderev.predecessor_count = noderev->predecessor_count; + binary_noderev.prop_rep = store_representation(container->prop_reps, + container->prop_reps_dict, + noderev->prop_rep); + if (noderev->data_rep) + { + binary_noderev.data_rep.representation + = store_representation(container->data_reps, + container->data_reps_dict, + noderev->data_rep); + binary_noderev.data_rep.uniquifier.txn_id + = noderev->data_rep->uniquifier.txn_id; + binary_noderev.data_rep.uniquifier.number + = noderev->data_rep->uniquifier.number; + } + + if (noderev->created_path) + binary_noderev.created_path + = svn_fs_x__string_table_builder_add(container->builder, + noderev->created_path, + 0); + + binary_noderev.mergeinfo_count = noderev->mergeinfo_count; + + APR_ARRAY_PUSH(container->noderevs, binary_noderev_t) = binary_noderev; + + return container->noderevs->nelts - 1; +} + +apr_size_t +svn_fs_x__noderevs_estimate_size(const svn_fs_x__noderevs_t *container) +{ + /* CONTAINER must be in 'builder' mode */ + if (container->builder == NULL) + return 0; + + /* string table code makes its own prediction, + * noderevs should be < 16 bytes each, + * ids < 10 bytes each, + * data representations < 40 bytes each, + * property representations < 30 bytes each, + * some static overhead should be assumed */ + return svn_fs_x__string_table_builder_estimate_size(container->builder) + + container->noderevs->nelts * 16 + + container->ids->nelts * 10 + + container->data_reps->nelts * 40 + + container->prop_reps->nelts * 30 + + 100; +} + +/* Create an svn_fs_id_t in *ID, allocated in POOL based on the id stored + * at index IDX in IDS. + */ +static svn_error_t * +get_id(const svn_fs_id_t **id, + const apr_array_header_t *ids, + int idx, + apr_pool_t *pool) +{ + binary_id_t *binary_id; + + /* handle NULL IDs */ + if (idx == 0) + { + *id = NULL; + return SVN_NO_ERROR; + } + + /* check for corrupted data */ + if (idx < 0 || idx > ids->nelts) + return svn_error_createf(SVN_ERR_FS_CONTAINER_INDEX, NULL, + _("Node revision ID index %d" + " exceeds container size %d"), + idx, ids->nelts); + + /* create a svn_fs_id_t from stored info */ + binary_id = &APR_ARRAY_IDX(ids, idx - 1, binary_id_t); + *id = svn_fs_x__id_rev_create(&binary_id->node_id, + &binary_id->copy_id, + &binary_id->rev_id, + pool); + + return SVN_NO_ERROR; +} + +/* Create a representation_t in *REP, allocated in POOL based on the + * representation stored at index IDX in REPS. + */ +static svn_error_t * +get_representation(representation_t **rep, + const apr_array_header_t *reps, + int idx, + apr_pool_t *pool) +{ + binary_representation_t *binary_rep; + + /* handle NULL representations */ + if (idx == 0) + { + *rep = NULL; + return SVN_NO_ERROR; + } + + /* check for corrupted data */ + if (idx < 0 || idx > reps->nelts) + return svn_error_createf(SVN_ERR_FS_CONTAINER_INDEX, NULL, + _("Node revision ID index %d" + " exceeds container size %d"), + idx, reps->nelts); + + /* no translation required. Just duplicate the info */ + binary_rep = &APR_ARRAY_IDX(reps, idx - 1, binary_representation_t); + + *rep = apr_pcalloc(pool, sizeof(**rep)); + (*rep)->has_sha1 = binary_rep->has_sha1; + memcpy((*rep)->sha1_digest, binary_rep->sha1_digest, + sizeof((*rep)->sha1_digest)); + memcpy((*rep)->md5_digest, binary_rep->md5_digest, + sizeof((*rep)->md5_digest)); + (*rep)->revision = binary_rep->revision; + (*rep)->item_index = binary_rep->item_index; + (*rep)->size = binary_rep->size; + (*rep)->expanded_size = binary_rep->expanded_size; + svn_fs_x__id_txn_reset(&(*rep)->txn_id); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__noderevs_get(node_revision_t **noderev_p, + const svn_fs_x__noderevs_t *container, + apr_size_t idx, + apr_pool_t *pool) +{ + node_revision_t *noderev; + binary_noderev_t *binary_noderev; + + /* CONTAINER must be in 'finalized' mode */ + SVN_ERR_ASSERT(container->builder == NULL); + SVN_ERR_ASSERT(container->paths); + + /* validate index */ + if (idx >= (apr_size_t)container->noderevs->nelts) + return svn_error_createf(SVN_ERR_FS_CONTAINER_INDEX, NULL, + _("Node revision index %" APR_SIZE_T_FMT + " exceeds container size %d"), + idx, container->noderevs->nelts); + + /* allocate result struct and fill it field by field */ + noderev = apr_pcalloc(pool, sizeof(*noderev)); + binary_noderev = &APR_ARRAY_IDX(container->noderevs, idx, binary_noderev_t); + + noderev->kind = (svn_node_kind_t)(binary_noderev->flags & NODEREV_KIND_MASK); + SVN_ERR(get_id(&noderev->id, container->ids, binary_noderev->id, pool)); + SVN_ERR(get_id(&noderev->predecessor_id, container->ids, + binary_noderev->predecessor_id, pool)); + + if (binary_noderev->flags & NODEREV_HAS_COPYFROM) + { + noderev->copyfrom_path + = svn_fs_x__string_table_get(container->paths, + binary_noderev->copyfrom_path, + NULL, + pool); + noderev->copyfrom_rev = binary_noderev->copyfrom_rev; + } + else + { + noderev->copyfrom_path = NULL; + noderev->copyfrom_rev = SVN_INVALID_REVNUM; + } + + if (binary_noderev->flags & NODEREV_HAS_COPYROOT) + { + noderev->copyroot_path + = svn_fs_x__string_table_get(container->paths, + binary_noderev->copyroot_path, + NULL, + pool); + noderev->copyroot_rev = binary_noderev->copyroot_rev; + } + else + { + noderev->copyroot_path = NULL; + noderev->copyroot_rev = 0; + } + + noderev->predecessor_count = binary_noderev->predecessor_count; + + SVN_ERR(get_representation(&noderev->prop_rep, container->prop_reps, + binary_noderev->prop_rep, pool)); + SVN_ERR(get_representation(&noderev->data_rep, container->data_reps, + binary_noderev->data_rep.representation, pool)); + if (noderev->data_rep) + { + noderev->data_rep->uniquifier.txn_id + = binary_noderev->data_rep.uniquifier.txn_id; + noderev->data_rep->uniquifier.number + = binary_noderev->data_rep.uniquifier.number; + } + + if (binary_noderev->flags & NODEREV_HAS_CPATH) + noderev->created_path + = svn_fs_x__string_table_get(container->paths, + binary_noderev->created_path, + NULL, + pool); + + noderev->mergeinfo_count = binary_noderev->mergeinfo_count; + + noderev->has_mergeinfo = (binary_noderev->flags & NODEREV_HAS_MINFO) ? 1 : 0; + *noderev_p = noderev; + + return SVN_NO_ERROR; +} + +/* Create and return a stream for representations in PARENT. + * Initialize the sub-streams for all fields, except checksums. + */ +static svn_packed__int_stream_t * +create_rep_stream(svn_packed__int_stream_t *parent) +{ + svn_packed__int_stream_t *stream + = svn_packed__create_int_substream(parent, FALSE, FALSE); + + /* sub-streams for members - except for checksums */ + /* has_sha1 */ + svn_packed__create_int_substream(stream, FALSE, FALSE); + + /* rev, item_index, size, expanded_size */ + svn_packed__create_int_substream(stream, TRUE, FALSE); + svn_packed__create_int_substream(stream, FALSE, FALSE); + svn_packed__create_int_substream(stream, FALSE, FALSE); + svn_packed__create_int_substream(stream, FALSE, FALSE); + + return stream; +} + +/* Serialize all representations in REP. Store checksums in DIGEST_STREAM, + * put all other fields into REP_STREAM. + */ +static void +write_reps(svn_packed__int_stream_t *rep_stream, + svn_packed__byte_stream_t *digest_stream, + apr_array_header_t *reps) +{ + int i; + for (i = 0; i < reps->nelts; ++i) + { + binary_representation_t *rep + = &APR_ARRAY_IDX(reps, i, binary_representation_t); + + svn_packed__add_uint(rep_stream, rep->has_sha1); + + svn_packed__add_uint(rep_stream, rep->revision); + svn_packed__add_uint(rep_stream, rep->item_index); + svn_packed__add_uint(rep_stream, rep->size); + svn_packed__add_uint(rep_stream, rep->expanded_size); + + svn_packed__add_bytes(digest_stream, + (const char *)rep->md5_digest, + sizeof(rep->md5_digest)); + if (rep->has_sha1) + svn_packed__add_bytes(digest_stream, + (const char *)rep->sha1_digest, + sizeof(rep->sha1_digest)); + } +} + +svn_error_t * +svn_fs_x__write_noderevs_container(svn_stream_t *stream, + const svn_fs_x__noderevs_t *container, + apr_pool_t *pool) +{ + int i; + + string_table_t *paths = container->paths + ? container->paths + : svn_fs_x__string_table_create(container->builder, + pool); + + svn_packed__data_root_t *root = svn_packed__data_create_root(pool); + + /* one common top-level stream for all arrays. One sub-stream */ + svn_packed__int_stream_t *structs_stream + = svn_packed__create_int_stream(root, FALSE, FALSE); + svn_packed__int_stream_t *ids_stream + = svn_packed__create_int_substream(structs_stream, FALSE, FALSE); + svn_packed__int_stream_t *data_reps_stream + = create_rep_stream(structs_stream); + svn_packed__int_stream_t *prop_reps_stream + = create_rep_stream(structs_stream); + svn_packed__int_stream_t *noderevs_stream + = svn_packed__create_int_substream(structs_stream, FALSE, FALSE); + svn_packed__byte_stream_t *digests_stream + = svn_packed__create_bytes_stream(root); + + /* structure the CHANGES_STREAM such we can extract much of the redundancy + * from the binary_change_t structs */ + for (i = 0; i < 3 * 2; ++i) + svn_packed__create_int_substream(ids_stream, TRUE, FALSE); + + svn_packed__create_int_substream(noderevs_stream, FALSE, FALSE); + for (i = 0; i < 14; ++i) + svn_packed__create_int_substream(noderevs_stream, TRUE, FALSE); + + /* serialize ids array */ + for (i = 0; i < container->ids->nelts; ++i) + { + binary_id_t *id = &APR_ARRAY_IDX(container->ids, i, binary_id_t); + + svn_packed__add_uint(ids_stream, id->node_id.revision); + svn_packed__add_uint(ids_stream, id->node_id.number); + svn_packed__add_uint(ids_stream, id->copy_id.revision); + svn_packed__add_uint(ids_stream, id->copy_id.number); + svn_packed__add_uint(ids_stream, id->rev_id.revision); + svn_packed__add_uint(ids_stream, id->rev_id.number); + } + + /* serialize rep arrays */ + write_reps(data_reps_stream, digests_stream, container->data_reps); + write_reps(prop_reps_stream, digests_stream, container->prop_reps); + + /* serialize noderevs array */ + for (i = 0; i < container->noderevs->nelts; ++i) + { + const binary_noderev_t *noderev + = &APR_ARRAY_IDX(container->noderevs, i, binary_noderev_t); + + svn_packed__add_uint(noderevs_stream, noderev->flags); + + svn_packed__add_uint(noderevs_stream, noderev->id); + svn_packed__add_uint(noderevs_stream, noderev->predecessor_id); + svn_packed__add_uint(noderevs_stream, noderev->predecessor_count); + + svn_packed__add_uint(noderevs_stream, noderev->copyfrom_path); + svn_packed__add_uint(noderevs_stream, noderev->copyfrom_rev); + svn_packed__add_uint(noderevs_stream, noderev->copyroot_path); + svn_packed__add_uint(noderevs_stream, noderev->copyroot_rev); + + svn_packed__add_uint(noderevs_stream, noderev->prop_rep); + svn_packed__add_uint(noderevs_stream, noderev->data_rep.representation); + + svn_packed__add_uint(noderevs_stream, + noderev->data_rep.uniquifier.txn_id.revision); + svn_packed__add_uint(noderevs_stream, + noderev->data_rep.uniquifier.txn_id.number); + svn_packed__add_uint(noderevs_stream, + noderev->data_rep.uniquifier.number); + + svn_packed__add_uint(noderevs_stream, noderev->created_path); + svn_packed__add_uint(noderevs_stream, noderev->mergeinfo_count); + } + + /* write to disk */ + SVN_ERR(svn_fs_x__write_string_table(stream, paths, pool)); + SVN_ERR(svn_packed__data_write(stream, root, pool)); + + return SVN_NO_ERROR; +} + +/* Allocate a representation_t array in POOL and return it in *REPS_P. + * Deserialize the data in REP_STREAM and DIGEST_STREAM and store the + * resulting representations into the *REPS_P. + */ +static svn_error_t * +read_reps(apr_array_header_t **reps_p, + svn_packed__int_stream_t *rep_stream, + svn_packed__byte_stream_t *digest_stream, + apr_pool_t *pool) +{ + apr_size_t i; + apr_size_t len; + const char *bytes; + + apr_size_t count + = svn_packed__int_count(svn_packed__first_int_substream(rep_stream)); + apr_array_header_t *reps + = apr_array_make(pool, count, sizeof(binary_representation_t)); + + for (i = 0; i < count; ++i) + { + binary_representation_t rep; + + rep.has_sha1 = svn_packed__get_uint(rep_stream); + + rep.revision = svn_packed__get_uint(rep_stream); + rep.item_index = svn_packed__get_uint(rep_stream); + rep.size = svn_packed__get_uint(rep_stream); + rep.expanded_size = svn_packed__get_uint(rep_stream); + + /* when extracting the checksums, beware of buffer under/overflows + caused by disk data corruption. */ + bytes = svn_packed__get_bytes(digest_stream, &len); + if (len != sizeof(rep.md5_digest)) + return svn_error_createf(SVN_ERR_FS_CONTAINER_INDEX, NULL, + _("Unexpected MD5 digest size %" + APR_SIZE_T_FMT), + len); + + memcpy(rep.md5_digest, bytes, sizeof(rep.md5_digest)); + if (rep.has_sha1) + { + bytes = svn_packed__get_bytes(digest_stream, &len); + if (len != sizeof(rep.sha1_digest)) + return svn_error_createf(SVN_ERR_FS_CONTAINER_INDEX, NULL, + _("Unexpected SHA1 digest size %" + APR_SIZE_T_FMT), + len); + + memcpy(rep.sha1_digest, bytes, sizeof(rep.sha1_digest)); + } + + APR_ARRAY_PUSH(reps, binary_representation_t) = rep; + } + + *reps_p = reps; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__read_noderevs_container(svn_fs_x__noderevs_t **container, + svn_stream_t *stream, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + apr_size_t i; + apr_size_t count; + + svn_fs_x__noderevs_t *noderevs + = apr_pcalloc(result_pool, sizeof(*noderevs)); + + svn_packed__data_root_t *root; + svn_packed__int_stream_t *structs_stream; + svn_packed__int_stream_t *ids_stream; + svn_packed__int_stream_t *data_reps_stream; + svn_packed__int_stream_t *prop_reps_stream; + svn_packed__int_stream_t *noderevs_stream; + svn_packed__byte_stream_t *digests_stream; + + /* read everything from disk */ + SVN_ERR(svn_fs_x__read_string_table(&noderevs->paths, stream, + result_pool, scratch_pool)); + SVN_ERR(svn_packed__data_read(&root, stream, result_pool, scratch_pool)); + + /* get streams */ + structs_stream = svn_packed__first_int_stream(root); + ids_stream = svn_packed__first_int_substream(structs_stream); + data_reps_stream = svn_packed__next_int_stream(ids_stream); + prop_reps_stream = svn_packed__next_int_stream(data_reps_stream); + noderevs_stream = svn_packed__next_int_stream(prop_reps_stream); + digests_stream = svn_packed__first_byte_stream(root); + + /* read ids array */ + count + = svn_packed__int_count(svn_packed__first_int_substream(ids_stream)); + noderevs->ids + = apr_array_make(result_pool, count, sizeof(binary_id_t)); + for (i = 0; i < count; ++i) + { + binary_id_t id; + + id.node_id.revision = svn_packed__get_uint(ids_stream); + id.node_id.number = svn_packed__get_uint(ids_stream); + id.copy_id.revision = svn_packed__get_uint(ids_stream); + id.copy_id.number = svn_packed__get_uint(ids_stream); + id.rev_id.revision = svn_packed__get_uint(ids_stream); + id.rev_id.number = svn_packed__get_uint(ids_stream); + + APR_ARRAY_PUSH(noderevs->ids, binary_id_t) = id; + } + + /* read rep arrays */ + SVN_ERR(read_reps(&noderevs->data_reps, data_reps_stream, digests_stream, + result_pool)); + SVN_ERR(read_reps(&noderevs->prop_reps, prop_reps_stream, digests_stream, + result_pool)); + + /* read noderevs array */ + count + = svn_packed__int_count(svn_packed__first_int_substream(noderevs_stream)); + noderevs->noderevs + = apr_array_make(result_pool, count, sizeof(binary_noderev_t)); + for (i = 0; i < count; ++i) + { + binary_noderev_t noderev; + + noderev.flags = svn_packed__get_uint(noderevs_stream); + + noderev.id = svn_packed__get_uint(noderevs_stream); + noderev.predecessor_id = svn_packed__get_uint(noderevs_stream); + noderev.predecessor_count = svn_packed__get_uint(noderevs_stream); + + noderev.copyfrom_path = svn_packed__get_uint(noderevs_stream); + noderev.copyfrom_rev = svn_packed__get_uint(noderevs_stream); + noderev.copyroot_path = svn_packed__get_uint(noderevs_stream); + noderev.copyroot_rev = svn_packed__get_uint(noderevs_stream); + + noderev.prop_rep = svn_packed__get_uint(noderevs_stream); + noderev.data_rep.representation = svn_packed__get_uint(noderevs_stream); + + noderev.data_rep.uniquifier.txn_id.revision + = svn_packed__get_uint(noderevs_stream); + noderev.data_rep.uniquifier.txn_id.number + = svn_packed__get_uint(noderevs_stream); + noderev.data_rep.uniquifier.number + = svn_packed__get_uint(noderevs_stream); + + noderev.created_path = svn_packed__get_uint(noderevs_stream); + noderev.mergeinfo_count = svn_packed__get_uint(noderevs_stream); + + APR_ARRAY_PUSH(noderevs->noderevs, binary_noderev_t) = noderev; + } + + *container = noderevs; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__serialize_noderevs_container(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + svn_fs_x__noderevs_t *noderevs = in; + svn_stringbuf_t *serialized; + apr_size_t size + = noderevs->ids->elt_size * noderevs->ids->nelts + + noderevs->data_reps->elt_size * noderevs->data_reps->nelts + + noderevs->prop_reps->elt_size * noderevs->prop_reps->nelts + + noderevs->noderevs->elt_size * noderevs->noderevs->nelts + + 10 * noderevs->noderevs->elt_size + + 100; + + /* serialize array header and all its elements */ + svn_temp_serializer__context_t *context + = svn_temp_serializer__init(noderevs, sizeof(*noderevs), size, pool); + + /* serialize sub-structures */ + svn_fs_x__serialize_string_table(context, &noderevs->paths); + svn_fs_x__serialize_apr_array(context, &noderevs->ids); + svn_fs_x__serialize_apr_array(context, &noderevs->data_reps); + svn_fs_x__serialize_apr_array(context, &noderevs->prop_reps); + svn_fs_x__serialize_apr_array(context, &noderevs->noderevs); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_noderevs_container(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + svn_fs_x__noderevs_t *noderevs = (svn_fs_x__noderevs_t *)data; + + /* de-serialize sub-structures */ + svn_fs_x__deserialize_string_table(noderevs, &noderevs->paths); + svn_fs_x__deserialize_apr_array(noderevs, &noderevs->ids, pool); + svn_fs_x__deserialize_apr_array(noderevs, &noderevs->data_reps, pool); + svn_fs_x__deserialize_apr_array(noderevs, &noderevs->prop_reps, pool); + svn_fs_x__deserialize_apr_array(noderevs, &noderevs->noderevs, pool); + + /* done */ + *out = noderevs; + + return SVN_NO_ERROR; +} + +/* Deserialize the cache serialized APR struct at *IN in BUFFER and write + * the result to OUT. Note that this will only resolve the pointers and + * not the array elements themselves. */ +static void +resolve_apr_array_header(apr_array_header_t *out, + const void *buffer, + apr_array_header_t * const *in) +{ + const apr_array_header_t *array + = svn_temp_deserializer__ptr(buffer, (const void *const *)in); + const char *elements + = svn_temp_deserializer__ptr(array, (const void *const *)&array->elts); + + *out = *array; + out->elts = (char *)elements; + out->pool = NULL; +} + +svn_error_t * +svn_fs_x__noderevs_get_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool) +{ + node_revision_t *noderev; + binary_noderev_t *binary_noderev; + + apr_array_header_t ids; + apr_array_header_t data_reps; + apr_array_header_t prop_reps; + apr_array_header_t noderevs; + + apr_uint32_t idx = *(apr_uint32_t *)baton; + const svn_fs_x__noderevs_t *container = data; + + /* Resolve all container pointers */ + const string_table_t *paths + = svn_temp_deserializer__ptr(container, + (const void *const *)&container->paths); + + resolve_apr_array_header(&ids, container, &container->ids); + resolve_apr_array_header(&data_reps, container, &container->data_reps); + resolve_apr_array_header(&prop_reps, container, &container->prop_reps); + resolve_apr_array_header(&noderevs, container, &container->noderevs); + + /* allocate result struct and fill it field by field */ + noderev = apr_pcalloc(pool, sizeof(*noderev)); + binary_noderev = &APR_ARRAY_IDX(&noderevs, idx, binary_noderev_t); + + noderev->kind = (svn_node_kind_t)(binary_noderev->flags & NODEREV_KIND_MASK); + SVN_ERR(get_id(&noderev->id, &ids, binary_noderev->id, pool)); + SVN_ERR(get_id(&noderev->predecessor_id, &ids, + binary_noderev->predecessor_id, pool)); + + if (binary_noderev->flags & NODEREV_HAS_COPYFROM) + { + noderev->copyfrom_path + = svn_fs_x__string_table_get_func(paths, + binary_noderev->copyfrom_path, + NULL, + pool); + noderev->copyfrom_rev = binary_noderev->copyfrom_rev; + } + else + { + noderev->copyfrom_path = NULL; + noderev->copyfrom_rev = SVN_INVALID_REVNUM; + } + + if (binary_noderev->flags & NODEREV_HAS_COPYROOT) + { + noderev->copyroot_path + = svn_fs_x__string_table_get_func(paths, + binary_noderev->copyroot_path, + NULL, + pool); + noderev->copyroot_rev = binary_noderev->copyroot_rev; + } + else + { + noderev->copyroot_path = NULL; + noderev->copyroot_rev = 0; + } + + noderev->predecessor_count = binary_noderev->predecessor_count; + + SVN_ERR(get_representation(&noderev->prop_rep, &prop_reps, + binary_noderev->prop_rep, pool)); + SVN_ERR(get_representation(&noderev->data_rep, &data_reps, + binary_noderev->data_rep.representation, pool)); + if (noderev->data_rep) + { + noderev->data_rep->uniquifier.txn_id + = binary_noderev->data_rep.uniquifier.txn_id; + noderev->data_rep->uniquifier.number + = binary_noderev->data_rep.uniquifier.number; + } + + if (binary_noderev->flags & NODEREV_HAS_CPATH) + noderev->created_path + = svn_fs_x__string_table_get_func(paths, + binary_noderev->created_path, + NULL, + pool); + + noderev->mergeinfo_count = binary_noderev->mergeinfo_count; + + noderev->has_mergeinfo = (binary_noderev->flags & NODEREV_HAS_MINFO) ? 1 : 0; + *out = noderev; + + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_fs_x/noderevs.h b/subversion/libsvn_fs_x/noderevs.h new file mode 100644 index 0000000..68e8048 --- /dev/null +++ b/subversion/libsvn_fs_x/noderevs.h
@@ -0,0 +1,130 @@ +/* noderevs.h --- FSX node revision container + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS__NODEREVS_H +#define SVN_LIBSVN_FS__NODEREVS_H + +#include "svn_io.h" +#include "fs.h" + +/* A collection of related noderevs tends to be widely redundant (similar + * paths, predecessor ID matching anothers ID, shared representations etc.) + * Also, the binary representation of a noderev can be much shorter than + * the ordinary textual variant. + * + * In its serialized form, the svn_fs_x__noderevs_t container extracts + * most of that redundancy and the run-time representation is also much + * smaller than sum of the respective node_revision_t objects. + * + * As with other containers, this one has two modes: 'construction', in + * which you may add data to it, and 'getter' in which there is only r/o + * access to the data. + */ + +/* An opaque collection of node revisions. + */ +typedef struct svn_fs_x__noderevs_t svn_fs_x__noderevs_t; + +/* Create and populate noderev containers. */ + +/* Create and return a new noderevs container with an initial capacity of + * INITIAL_COUNT node_revision_t objects. Allocate the result in POOL. + */ +svn_fs_x__noderevs_t * +svn_fs_x__noderevs_create(apr_size_t initial_count, + apr_pool_t *pool); + +/* Add NODEREV to the CONTAINER. Return the index that identifies the new + * item in this container. + */ +apr_size_t +svn_fs_x__noderevs_add(svn_fs_x__noderevs_t *container, + node_revision_t *noderev); + +/* Return a rough estimate in bytes for the serialized representation + * of CONTAINER. + */ +apr_size_t +svn_fs_x__noderevs_estimate_size(const svn_fs_x__noderevs_t *container); + +/* Read from noderev containers. */ + +/* From CONTAINER, extract the noderev with the given IDX. Allocate + * the result in POOL and return it in *NODEREV_P. + */ +svn_error_t * +svn_fs_x__noderevs_get(node_revision_t **noderev_p, + const svn_fs_x__noderevs_t *container, + apr_size_t idx, + apr_pool_t *pool); + +/* I/O interface. */ + +/* Write a serialized representation of CONTAINER to STREAM. Use POOL for + * temporary allocations. + */ +svn_error_t * +svn_fs_x__write_noderevs_container(svn_stream_t *stream, + const svn_fs_x__noderevs_t *container, + apr_pool_t *pool); + +/* Read a noderev container from its serialized representation in STREAM. + * Allocate the result in RESULT_POOL and return it in *CONTAINER. Use + * SCRATCH_POOL for temporary allocations. + */ +svn_error_t * +svn_fs_x__read_noderevs_container(svn_fs_x__noderevs_t **container, + svn_stream_t *stream, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + +/* Implements #svn_cache__serialize_func_t for svn_fs_x__noderevs_t + * objects. + */ +svn_error_t * +svn_fs_x__serialize_noderevs_container(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/* Implements #svn_cache__deserialize_func_t for svn_fs_x__noderevs_t + * objects. + */ +svn_error_t * +svn_fs_x__deserialize_noderevs_container(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/* Implements svn_cache__partial_getter_func_t for svn_fs_x__noderevs_t, + * setting *OUT to the node_revision_t selected by the apr_uint32_t index + * passed in as *BATON. This function is similar to svn_fs_x__noderevs_get + * but operates on the cache serialized representation of the container. + */ +svn_error_t * +svn_fs_x__noderevs_get_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool); + +#endif \ No newline at end of file
diff --git a/subversion/libsvn_fs_x/pack.c b/subversion/libsvn_fs_x/pack.c new file mode 100644 index 0000000..171a6cc --- /dev/null +++ b/subversion/libsvn_fs_x/pack.c
@@ -0,0 +1,2357 @@ +/* pack.c --- FSX shard packing functionality + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +#include <assert.h> + +#include "svn_pools.h" +#include "svn_dirent_uri.h" +#include "svn_sorts.h" +#include "private/svn_temp_serializer.h" +#include "private/svn_subr_private.h" +#include "private/svn_string_private.h" + +#include "fs_x.h" +#include "pack.h" +#include "util.h" +#include "revprops.h" +#include "transaction.h" +#include "index.h" +#include "low_level.h" +#include "cached_data.h" +#include "changes.h" +#include "noderevs.h" +#include "reps.h" + +#include "../libsvn_fs/fs-loader.h" + +#include "svn_private_config.h" +#include "temp_serializer.h" + +/* Format 7 packing logic: + * + * We pack files on a pack file basis (e.g. 1000 revs) without changing + * existing pack files nor the revision files outside the range to pack. + * + * First, we will scan the revision file indexes to determine the number + * of items to "place" (i.e. determine their optimal position within the + * future pack file). For each item, we will need a constant amount of + * memory to track it. A MAX_MEM parameter sets a limit to the number of + * items we may place in one go. That means, we may not be able to add + * all revisions at once. Instead, we will run the placement for a subset + * of revisions at a time. T very unlikely worst case will simply append + * all revision data with just a little reshuffling inside each revision. + * + * In a second step, we read all revisions in the selected range, build + * the item tracking information and copy the items themselves from the + * revision files to temporary files. The latter serve as buckets for a + * very coarse bucket presort: Separate change lists, file properties, + * directory properties and noderevs + representations from one another. + * + * The third step will determine an optimized placement for the items in + * each of the 4 buckets separately. The first three will simply order + * their items by revision, starting with the newest once. Placing rep + * and noderev items is a more elaborate process documented in the code. + * + * Step 4 copies the items from the temporary buckets into the final + * pack file and write the temporary index files. + * + * Finally, after the last range of revisions, create the final indexes. + */ + +/* Data structure describing a node change at PATH, REVISION. + * We will sort these instances by PATH and NODE_ID such that we can combine + * similar nodes in the same reps container and store containers in path + * major order. + */ +typedef struct path_order_t +{ + /* changed path */ + svn_prefix_string__t *path; + + /* node ID for this PATH in REVISION */ + svn_fs_x__id_part_t node_id; + + /* when this change happened */ + svn_revnum_t revision; + + /* length of the expanded representation content */ + apr_int64_t expanded_size; + + /* item ID of the noderev linked to the change. May be (0, 0). */ + svn_fs_x__id_part_t noderev_id; + + /* item ID of the representation containing the new data. May be (0, 0). */ + svn_fs_x__id_part_t rep_id; +} path_order_t; + +/* Represents a reference from item FROM to item TO. FROM may be a noderev + * or rep_id while TO is (currently) always a representation. We will sort + * them by TO which allows us to collect all dependent items. + */ +typedef struct reference_t +{ + svn_fs_x__id_part_t to; + svn_fs_x__id_part_t from; +} reference_t; + +/* This structure keeps track of all the temporary data and status that + * needs to be kept around during the creation of one pack file. After + * each revision range (in case we can't process all revs at once due to + * memory restrictions), parts of the data will get re-initialized. + */ +typedef struct pack_context_t +{ + /* file system that we operate on */ + svn_fs_t *fs; + + /* cancel function to invoke at regular intervals. May be NULL */ + svn_cancel_func_t cancel_func; + + /* baton to pass to CANCEL_FUNC */ + void *cancel_baton; + + /* first revision in the shard (and future pack file) */ + svn_revnum_t shard_rev; + + /* first revision in the range to process (>= SHARD_REV) */ + svn_revnum_t start_rev; + + /* first revision after the range to process (<= SHARD_END_REV) */ + svn_revnum_t end_rev; + + /* first revision after the current shard */ + svn_revnum_t shard_end_rev; + + /* log-to-phys proto index for the whole pack file */ + apr_file_t *proto_l2p_index; + + /* phys-to-log proto index for the whole pack file */ + apr_file_t *proto_p2l_index; + + /* full shard directory path (containing the unpacked revisions) */ + const char *shard_dir; + + /* full packed shard directory path (containing the pack file + indexes) */ + const char *pack_file_dir; + + /* full pack file path (including PACK_FILE_DIR) */ + const char *pack_file_path; + + /* current write position (i.e. file length) in the pack file */ + apr_off_t pack_offset; + + /* the pack file to ultimately write all data to */ + apr_file_t *pack_file; + + /* array of svn_fs_x__p2l_entry_t *, all referring to change lists. + * Will be filled in phase 2 and be cleared after each revision range. */ + apr_array_header_t *changes; + + /* temp file receiving all change list items (referenced by CHANGES). + * Will be filled in phase 2 and be cleared after each revision range. */ + apr_file_t *changes_file; + + /* array of svn_fs_x__p2l_entry_t *, all referring to file properties. + * Will be filled in phase 2 and be cleared after each revision range. */ + apr_array_header_t *file_props; + + /* temp file receiving all file prop items (referenced by FILE_PROPS). + * Will be filled in phase 2 and be cleared after each revision range.*/ + apr_file_t *file_props_file; + + /* array of svn_fs_x__p2l_entry_t *, all referring to directory properties. + * Will be filled in phase 2 and be cleared after each revision range. */ + apr_array_header_t *dir_props; + + /* temp file receiving all directory prop items (referenced by DIR_PROPS). + * Will be filled in phase 2 and be cleared after each revision range.*/ + apr_file_t *dir_props_file; + + /* container for all PATH members in PATH_ORDER. */ + svn_prefix_tree__t *paths; + + /* array of path_order_t *. Will be filled in phase 2 and be cleared + * after each revision range. Sorted by PATH, NODE_ID. */ + apr_array_header_t *path_order; + + /* array of reference_t *. Will be filled in phase 2 and be cleared + * after each revision range. It will be sorted by the TO members. */ + apr_array_header_t *references; + + /* array of svn_fs_x__p2l_entry_t*. Will be filled in phase 2 and be + * cleared after each revision range. During phase 3, we will set items + * to NULL that we already processed. */ + apr_array_header_t *reps; + + /* array of int, marking for each revision, the which offset their items + * begin in REPS. Will be filled in phase 2 and be cleared after + * each revision range. */ + apr_array_header_t *rev_offsets; + + /* temp file receiving all items referenced by REPS_INFOS. + * Will be filled in phase 2 and be cleared after each revision range.*/ + apr_file_t *reps_file; + + /* pool used for temporary data structures that will be cleaned up when + * the next range of revisions is being processed */ + apr_pool_t *info_pool; +} pack_context_t; + +/* Create and initialize a new pack context for packing shard SHARD_REV in + * SHARD_DIR into PACK_FILE_DIR within filesystem FS. Allocate it in POOL + * and return the structure in *CONTEXT. + * + * Limit the number of items being copied per iteration to MAX_ITEMS. + * Set CANCEL_FUNC and CANCEL_BATON as well. + */ +static svn_error_t * +initialize_pack_context(pack_context_t *context, + svn_fs_t *fs, + const char *pack_file_dir, + const char *shard_dir, + svn_revnum_t shard_rev, + apr_size_t max_items, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + const char *temp_dir; + apr_size_t max_revs = MIN(ffd->max_files_per_dir, (int)max_items); + + SVN_ERR_ASSERT(shard_rev % ffd->max_files_per_dir == 0); + + /* where we will place our various temp files */ + SVN_ERR(svn_io_temp_dir(&temp_dir, pool)); + + /* store parameters */ + context->fs = fs; + context->cancel_func = cancel_func; + context->cancel_baton = cancel_baton; + + context->shard_rev = shard_rev; + context->start_rev = shard_rev; + context->end_rev = shard_rev; + context->shard_end_rev = shard_rev + ffd->max_files_per_dir; + + /* Create the new directory and pack file. */ + context->shard_dir = shard_dir; + context->pack_file_dir = pack_file_dir; + context->pack_file_path + = svn_dirent_join(pack_file_dir, PATH_PACKED, pool); + SVN_ERR(svn_io_file_open(&context->pack_file, context->pack_file_path, + APR_WRITE | APR_BUFFERED | APR_BINARY | APR_EXCL + | APR_CREATE, APR_OS_DEFAULT, pool)); + + /* Proto index files */ + SVN_ERR(svn_fs_x__l2p_proto_index_open + (&context->proto_l2p_index, + svn_dirent_join(pack_file_dir, + PATH_INDEX PATH_EXT_L2P_INDEX, + pool), + pool)); + SVN_ERR(svn_fs_x__p2l_proto_index_open + (&context->proto_p2l_index, + svn_dirent_join(pack_file_dir, + PATH_INDEX PATH_EXT_P2L_INDEX, + pool), + pool)); + + /* item buckets: one item info array and one temp file per bucket */ + context->changes = apr_array_make(pool, max_items, + sizeof(svn_fs_x__p2l_entry_t *)); + SVN_ERR(svn_io_open_unique_file3(&context->changes_file, NULL, temp_dir, + svn_io_file_del_on_close, pool, pool)); + context->file_props = apr_array_make(pool, max_items, + sizeof(svn_fs_x__p2l_entry_t *)); + SVN_ERR(svn_io_open_unique_file3(&context->file_props_file, NULL, temp_dir, + svn_io_file_del_on_close, pool, pool)); + context->dir_props = apr_array_make(pool, max_items, + sizeof(svn_fs_x__p2l_entry_t *)); + SVN_ERR(svn_io_open_unique_file3(&context->dir_props_file, NULL, temp_dir, + svn_io_file_del_on_close, pool, pool)); + + /* noderev and representation item bucket */ + context->rev_offsets = apr_array_make(pool, max_revs, sizeof(int)); + context->path_order = apr_array_make(pool, max_items, sizeof(path_order_t *)); + context->references = apr_array_make(pool, max_items, sizeof(reference_t *)); + context->reps = apr_array_make(pool, max_items, + sizeof(svn_fs_x__p2l_entry_t *)); + SVN_ERR(svn_io_open_unique_file3(&context->reps_file, NULL, temp_dir, + svn_io_file_del_on_close, pool, pool)); + + /* the pool used for temp structures */ + context->info_pool = svn_pool_create(pool); + context->paths = svn_prefix_tree__create(context->info_pool); + + return SVN_NO_ERROR; +}; + +/* Clean up / free all revision range specific data and files in CONTEXT. + * Use POOL for temporary allocations. + */ +static svn_error_t * +reset_pack_context(pack_context_t *context, + apr_pool_t *pool) +{ + apr_array_clear(context->changes); + SVN_ERR(svn_io_file_trunc(context->changes_file, 0, pool)); + apr_array_clear(context->file_props); + SVN_ERR(svn_io_file_trunc(context->file_props_file, 0, pool)); + apr_array_clear(context->dir_props); + SVN_ERR(svn_io_file_trunc(context->dir_props_file, 0, pool)); + + apr_array_clear(context->rev_offsets); + apr_array_clear(context->path_order); + apr_array_clear(context->references); + apr_array_clear(context->reps); + SVN_ERR(svn_io_file_trunc(context->reps_file, 0, pool)); + + svn_pool_clear(context->info_pool); + + return SVN_NO_ERROR; +}; + +/* Call this after the last revision range. It will finalize all index files + * for CONTEXT and close any open files. Use POOL for temporary allocations. + */ +static svn_error_t * +close_pack_context(pack_context_t *context, + apr_pool_t *pool) +{ + const char *l2p_index_path + = apr_pstrcat(pool, context->pack_file_path, PATH_EXT_L2P_INDEX, NULL); + const char *p2l_index_path + = apr_pstrcat(pool, context->pack_file_path, PATH_EXT_P2L_INDEX, NULL); + const char *proto_l2p_index_path; + const char *proto_p2l_index_path; + + /* need the file names for the actual index creation call further down */ + SVN_ERR(svn_io_file_name_get(&proto_l2p_index_path, + context->proto_l2p_index, pool)); + SVN_ERR(svn_io_file_name_get(&proto_p2l_index_path, + context->proto_p2l_index, pool)); + + /* finalize proto index files */ + SVN_ERR(svn_io_file_close(context->proto_l2p_index, pool)); + SVN_ERR(svn_io_file_close(context->proto_p2l_index, pool)); + + /* Create the actual index files*/ + SVN_ERR(svn_fs_x__l2p_index_create(context->fs, l2p_index_path, + proto_l2p_index_path, + context->shard_rev, pool)); + SVN_ERR(svn_fs_x__p2l_index_create(context->fs, p2l_index_path, + proto_p2l_index_path, + context->shard_rev, pool)); + + /* remove proto index files */ + SVN_ERR(svn_io_remove_file2(proto_l2p_index_path, FALSE, pool)); + SVN_ERR(svn_io_remove_file2(proto_p2l_index_path, FALSE, pool)); + + SVN_ERR(svn_io_file_close(context->pack_file, pool)); + + return SVN_NO_ERROR; +}; + +/* Efficiently copy SIZE bytes from SOURCE to DEST. Invoke the CANCEL_FUNC + * from CONTEXT at regular intervals. Use POOL for allocations. + */ +static svn_error_t * +copy_file_data(pack_context_t *context, + apr_file_t *dest, + apr_file_t *source, + apr_off_t size, + apr_pool_t *pool) +{ + /* most non-representation items will be small. Minimize the buffer + * and infrastructure overhead in that case. */ + enum { STACK_BUFFER_SIZE = 1024 }; + + if (size < STACK_BUFFER_SIZE) + { + /* copy small data using a fixed-size buffer on stack */ + char buffer[STACK_BUFFER_SIZE]; + SVN_ERR(svn_io_file_read_full2(source, buffer, (apr_size_t)size, + NULL, NULL, pool)); + SVN_ERR(svn_io_file_write_full(dest, buffer, (apr_size_t)size, + NULL, pool)); + } + else + { + /* use streaming copies for larger data blocks. That may require + * the allocation of larger buffers and we should make sure that + * this extra memory is released asap. */ + fs_x_data_t *ffd = context->fs->fsap_data; + apr_pool_t *copypool = svn_pool_create(pool); + char *buffer = apr_palloc(copypool, ffd->block_size); + + while (size) + { + apr_size_t to_copy = (apr_size_t)(MIN(size, ffd->block_size)); + if (context->cancel_func) + SVN_ERR(context->cancel_func(context->cancel_baton)); + + SVN_ERR(svn_io_file_read_full2(source, buffer, to_copy, + NULL, NULL, pool)); + SVN_ERR(svn_io_file_write_full(dest, buffer, to_copy, + NULL, pool)); + + size -= to_copy; + } + + svn_pool_destroy(copypool); + } + + return SVN_NO_ERROR; +} + +/* Writes SIZE bytes, all 0, to DEST. Uses POOL for allocations. + */ +static svn_error_t * +write_null_bytes(apr_file_t *dest, + apr_off_t size, + apr_pool_t *pool) +{ + /* Have a collection of high-quality, easy to access NUL bytes handy. */ + enum { BUFFER_SIZE = 1024 }; + static const char buffer[BUFFER_SIZE] = { 0 }; + + /* copy SIZE of them into the file's buffer */ + while (size) + { + apr_size_t to_write = MIN(size, BUFFER_SIZE); + SVN_ERR(svn_io_file_write_full(dest, buffer, to_write, NULL, pool)); + size -= to_write; + } + + return SVN_NO_ERROR; +} + +/* Copy the "simple" item (changed paths list or property representation) + * from the current position in REV_FILE to TEMP_FILE using CONTEXT. Add + * a copy of ENTRY to ENTRIES but with an updated offset value that points + * to the copy destination in TEMP_FILE. Use POOL for allocations. + */ +static svn_error_t * +copy_item_to_temp(pack_context_t *context, + apr_array_header_t *entries, + apr_file_t *temp_file, + apr_file_t *rev_file, + svn_fs_x__p2l_entry_t *entry, + apr_pool_t *pool) +{ + svn_fs_x__p2l_entry_t *new_entry + = svn_fs_x__p2l_entry_dup(entry, context->info_pool); + new_entry->offset = 0; + SVN_ERR(svn_io_file_seek(temp_file, SEEK_CUR, &new_entry->offset, pool)); + APR_ARRAY_PUSH(entries, svn_fs_x__p2l_entry_t *) = new_entry; + + SVN_ERR(copy_file_data(context, temp_file, rev_file, entry->size, pool)); + + return SVN_NO_ERROR; +} + +/* Return the offset within CONTEXT->REPS_INFOS that corresponds to item + * ITEM_INDEX in REVISION. + */ +static int +get_item_array_index(pack_context_t *context, + svn_revnum_t revision, + apr_int64_t item_index) +{ + assert(revision >= context->start_rev); + return (int)item_index + APR_ARRAY_IDX(context->rev_offsets, + revision - context->start_rev, + int); +} + +/* Write INFO to the correct position in CONTEXT->REP_INFOS. The latter + * may need auto-expanding. Overwriting an array element is not allowed. + */ +static void +add_item_rep_mapping(pack_context_t *context, + svn_fs_x__p2l_entry_t *entry) +{ + int idx; + assert(entry->item_count == 1); + + /* index of INFO */ + idx = get_item_array_index(context, + entry->items[0].revision, + entry->items[0].number); + + /* make sure the index exists in the array */ + while (context->reps->nelts <= idx) + APR_ARRAY_PUSH(context->reps, void *) = NULL; + + /* set the element. If there is already an entry, there are probably + * two items claiming to be the same -> bail out */ + assert(!APR_ARRAY_IDX(context->reps, idx, void *)); + APR_ARRAY_IDX(context->reps, idx, void *) = entry; +} + +/* Return the P2L entry from CONTEXT->REPS for the given ID. If there is + * none (or not anymore), return NULL. If RESET has been specified, set + * the array entry to NULL after returning the entry. + */ +static svn_fs_x__p2l_entry_t * +get_item(pack_context_t *context, + const svn_fs_x__id_part_t *id, + svn_boolean_t reset) +{ + svn_fs_x__p2l_entry_t *result = NULL; + if (id->number && id->revision >= context->start_rev) + { + int idx = get_item_array_index(context, id->revision, id->number); + if (context->reps->nelts > idx) + { + result = APR_ARRAY_IDX(context->reps, idx, void *); + if (result && reset) + APR_ARRAY_IDX(context->reps, idx, void *) = NULL; + } + } + + return result; +} + +/* Copy representation item identified by ENTRY from the current position + * in REV_FILE into CONTEXT->REPS_FILE. Add all tracking into needed by + * our placement algorithm to CONTEXT. Use POOL for temporary allocations. + */ +static svn_error_t * +copy_rep_to_temp(pack_context_t *context, + apr_file_t *rev_file, + svn_fs_x__p2l_entry_t *entry, + apr_pool_t *pool) +{ + svn_fs_x__rep_header_t *rep_header; + svn_stream_t *stream; + apr_off_t source_offset = entry->offset; + + /* create a copy of ENTRY, make it point to the copy destination and + * store it in CONTEXT */ + entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool); + entry->offset = 0; + SVN_ERR(svn_io_file_seek(context->reps_file, SEEK_CUR, &entry->offset, + pool)); + add_item_rep_mapping(context, entry); + + /* read & parse the representation header */ + stream = svn_stream_from_aprfile2(rev_file, TRUE, pool); + SVN_ERR(svn_fs_x__read_rep_header(&rep_header, stream, pool)); + svn_stream_close(stream); + + /* if the representation is a delta against some other rep, link the two */ + if ( rep_header->type == svn_fs_x__rep_delta + && rep_header->base_revision >= context->start_rev) + { + reference_t *reference = apr_pcalloc(context->info_pool, + sizeof(*reference)); + reference->from = entry->items[0]; + reference->to.revision = rep_header->base_revision; + reference->to.number = rep_header->base_item_index; + APR_ARRAY_PUSH(context->references, reference_t *) = reference; + } + + /* copy the whole rep (including header!) to our temp file */ + SVN_ERR(svn_io_file_seek(rev_file, SEEK_SET, &source_offset, pool)); + SVN_ERR(copy_file_data(context, context->reps_file, rev_file, entry->size, + pool)); + + return SVN_NO_ERROR; +} + +/* Directories first, dirs / files sorted by name in reverse lexical order. + * This maximizes the chance of two items being located close to one another + * in *all* pack files independent of their change order. It also groups + * multi-project repos nicely according to their sub-projects. The reverse + * order aspect gives "trunk" preference over "tags" and "branches", so + * trunk-related items are more likely to be contiguous. + */ +static int +compare_dir_entries(const svn_sort__item_t *a, + const svn_sort__item_t *b) +{ + const svn_fs_dirent_t *lhs = (const svn_fs_dirent_t *) a->value; + const svn_fs_dirent_t *rhs = (const svn_fs_dirent_t *) b->value; + + if (lhs->kind != rhs->kind) + return lhs->kind == svn_node_dir ? -1 : 1; + + return 0 - strcmp(lhs->name, rhs->name); +} + +apr_array_header_t * +svn_fs_x__order_dir_entries(svn_fs_t *fs, + apr_hash_t *directory, + apr_pool_t *pool) +{ + apr_array_header_t *ordered + = svn_sort__hash(directory, compare_dir_entries, pool); + + apr_array_header_t *result + = apr_array_make(pool, ordered->nelts, sizeof(svn_fs_dirent_t *)); + + int i; + for (i = 0; i < ordered->nelts; ++i) + APR_ARRAY_PUSH(result, svn_fs_dirent_t *) + = APR_ARRAY_IDX(ordered, i, svn_sort__item_t).value; + + return result; +} + +/* Copy node revision item identified by ENTRY from the current position + * in REV_FILE into CONTEXT->REPS_FILE. Add all tracking into needed by + * our placement algorithm to CONTEXT. Use POOL for temporary allocations. + */ +static svn_error_t * +copy_node_to_temp(pack_context_t *context, + apr_file_t *rev_file, + svn_fs_x__p2l_entry_t *entry, + apr_pool_t *pool) +{ + path_order_t *path_order = apr_pcalloc(context->info_pool, + sizeof(*path_order)); + node_revision_t *noderev; + svn_stream_t *stream; + apr_off_t source_offset = entry->offset; + + /* read & parse noderev */ + stream = svn_stream_from_aprfile2(rev_file, TRUE, pool); + SVN_ERR(svn_fs_x__read_noderev(&noderev, stream, pool)); + svn_stream_close(stream); + + /* create a copy of ENTRY, make it point to the copy destination and + * store it in CONTEXT */ + entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool); + entry->offset = 0; + SVN_ERR(svn_io_file_seek(context->reps_file, SEEK_CUR, + &entry->offset, pool)); + add_item_rep_mapping(context, entry); + + /* copy the noderev to our temp file */ + SVN_ERR(svn_io_file_seek(rev_file, SEEK_SET, &source_offset, pool)); + SVN_ERR(copy_file_data(context, context->reps_file, rev_file, entry->size, + pool)); + + /* if the node has a data representation, make that the node's "base". + * This will (often) cause the noderev to be placed right in front of + * its data representation. */ + + if (noderev->data_rep && noderev->data_rep->revision >= context->start_rev) + { + reference_t *reference = apr_pcalloc(context->info_pool, + sizeof(*reference)); + reference->from = entry->items[0]; + reference->to.revision = noderev->data_rep->revision; + reference->to.number = noderev->data_rep->item_index; + APR_ARRAY_PUSH(context->references, reference_t *) = reference; + + path_order->rep_id = reference->to; + path_order->expanded_size = noderev->data_rep->expanded_size; + } + + path_order->path = svn_prefix_string__create(context->paths, + noderev->created_path); + path_order->node_id = *svn_fs_x__id_node_id(noderev->id); + path_order->revision = svn_fs_x__id_rev(noderev->id); + path_order->noderev_id = *svn_fs_x__id_rev_item(noderev->id); + APR_ARRAY_PUSH(context->path_order, path_order_t *) = path_order; + + return SVN_NO_ERROR; +} + +/* implements compare_fn_t. Place LHS before RHS, if the latter is older. + */ +static int +compare_p2l_info(const svn_fs_x__p2l_entry_t * const * lhs, + const svn_fs_x__p2l_entry_t * const * rhs) +{ + assert(*lhs != *rhs); + if ((*lhs)->item_count == 0) + return (*lhs)->item_count == 0 ? 0 : -1; + if ((*lhs)->item_count == 0) + return 1; + + if ((*lhs)->items[0].revision == (*rhs)->items[0].revision) + return (*lhs)->items[0].number > (*rhs)->items[0].number ? -1 : 1; + + return (*lhs)->items[0].revision > (*rhs)->items[0].revision ? -1 : 1; +} + +/* Sort svn_fs_x__p2l_entry_t * array ENTRIES by age. Place the latest + * items first. + */ +static void +sort_items(apr_array_header_t *entries) +{ + qsort(entries->elts, entries->nelts, entries->elt_size, + (int (*)(const void *, const void *))compare_p2l_info); +} + +/* Decorator for svn_fs_x__p2l_entry_t that associates it with a sorted + * variant of its ITEMS array. + */ +typedef struct sub_item_ordered_t +{ + /* ENTRY that got wrapped */ + svn_fs_x__p2l_entry_t *entry; + + /* Array of pointers into ENTRY->ITEMS, sorted by their revision member + * _descending_ order. May be NULL if ENTRY->ITEM_COUNT < 2. */ + svn_fs_x__id_part_t **order; +} sub_item_ordered_t; + +/* implements compare_fn_t. Place LHS before RHS, if the latter is younger. + * Used to sort sub_item_ordered_t::order + */ +static int +compare_sub_items(const svn_fs_x__id_part_t * const * lhs, + const svn_fs_x__id_part_t * const * rhs) +{ + return (*lhs)->revision < (*rhs)->revision + ? 1 + : ((*lhs)->revision > (*rhs)->revision ? -1 : 0); +} + +/* implements compare_fn_t. Place LHS before RHS, if the latter belongs to + * a newer revision. + */ +static int +compare_p2l_info_rev(const sub_item_ordered_t * lhs, + const sub_item_ordered_t * rhs) +{ + svn_fs_x__id_part_t *lhs_part; + svn_fs_x__id_part_t *rhs_part; + + assert(lhs != rhs); + if (lhs->entry->item_count == 0) + return rhs->entry->item_count == 0 ? 0 : -1; + if (rhs->entry->item_count == 0) + return 1; + + lhs_part = lhs->order ? lhs->order[lhs->entry->item_count - 1] + : &lhs->entry->items[0]; + rhs_part = rhs->order ? rhs->order[rhs->entry->item_count - 1] + : &rhs->entry->items[0]; + + if (lhs_part->revision == rhs_part->revision) + return 0; + + return lhs_part->revision < rhs_part->revision ? -1 : 1; +} + +/* implements compare_fn_t. Sort descending by PATH, NODE_ID and REVISION. + */ +static int +compare_path_order(const path_order_t * const * lhs_p, + const path_order_t * const * rhs_p) +{ + const path_order_t * lhs = *lhs_p; + const path_order_t * rhs = *rhs_p; + + /* reverse lexicographic order on path and node (i.e. latest first) */ + int diff = svn_prefix_string__compare(rhs->path, lhs->path); + if (diff) + return diff; + + /* reverse order on node (i.e. latest first) */ + diff = svn_fs_x__id_part_compare(&rhs->node_id, &lhs->node_id); + if (diff) + return diff; + + /* reverse order on revision (i.e. latest first) */ + if (lhs->revision != rhs->revision) + return lhs->revision < rhs->revision ? 1 : -1; + + return 0; +} + +/* implements compare_fn_t. Sort ascending by TO, FROM. + */ +static int +compare_references(const reference_t * const * lhs_p, + const reference_t * const * rhs_p) +{ + const reference_t * lhs = *lhs_p; + const reference_t * rhs = *rhs_p; + + int diff = svn_fs_x__id_part_compare(&lhs->to, &rhs->to); + return diff ? diff : svn_fs_x__id_part_compare(&lhs->from, &rhs->from); +} + +/* Order the data collected in CONTEXT such that we can place them in the + * desired order. + */ +static void +sort_reps(pack_context_t *context) +{ + qsort(context->path_order->elts, context->path_order->nelts, + context->path_order->elt_size, + (int (*)(const void *, const void *))compare_path_order); + qsort(context->references->elts, context->references->nelts, + context->references->elt_size, + (int (*)(const void *, const void *))compare_references); +} + +/* Return the remaining unused bytes in the current block in CONTEXT's + * pack file. + */ +static apr_ssize_t +get_block_left(pack_context_t *context) +{ + fs_x_data_t *ffd = context->fs->fsap_data; + return ffd->block_size - (context->pack_offset % ffd->block_size); +} + +/* To prevent items from overlapping a block boundary, we will usually + * put them into the next block and top up the old one with NUL bytes. + * Pad CONTEXT's pack file to the end of the current block, if that padding + * is short enough. Use POOL for allocations. + */ +static svn_error_t * +auto_pad_block(pack_context_t *context, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = context->fs->fsap_data; + + /* This is the maximum number of bytes "wasted" that way per block. + * Larger items will cross the block boundaries. */ + const apr_off_t max_padding = MAX(ffd->block_size / 50, 512); + + /* Is wasted space small enough to align the current item to the next + * block? */ + apr_off_t padding = get_block_left(context); + + if (padding < max_padding) + { + /* Yes. To up with NUL bytes and don't forget to create + * an P2L index entry marking this section as unused. */ + svn_fs_x__p2l_entry_t null_entry; + + null_entry.offset = context->pack_offset; + null_entry.size = padding; + null_entry.type = SVN_FS_X__ITEM_TYPE_UNUSED; + null_entry.item_count = 0; + null_entry.items = NULL; + + SVN_ERR(write_null_bytes(context->pack_file, padding, pool)); + SVN_ERR(svn_fs_x__p2l_proto_index_add_entry + (context->proto_p2l_index, &null_entry, pool)); + context->pack_offset += padding; + } + + return SVN_NO_ERROR; +} + +/* Return the index of the first entry in CONTEXT->REFERENCES that + * references ITEM->ITEMS[0] if such entries exist. All matching items + * will be consecutive. + */ +static int +find_first_reference(pack_context_t *context, + svn_fs_x__p2l_entry_t *item) +{ + int lower = 0; + int upper = context->references->nelts - 1; + + while (lower <= upper) + { + int current = lower + (upper - lower) / 2; + reference_t *reference + = APR_ARRAY_IDX(context->references, current, reference_t *); + + if (svn_fs_x__id_part_compare(&reference->to, item->items) < 0) + lower = current + 1; + else + upper = current - 1; + } + + return lower; +} + +/* Check whether entry number IDX in CONTEXT->REFERENCES references ITEM. + */ +static svn_boolean_t +is_reference_match(pack_context_t *context, + int idx, + svn_fs_x__p2l_entry_t *item) +{ + reference_t *reference; + if (context->references->nelts <= idx) + return FALSE; + + reference = APR_ARRAY_IDX(context->references, idx, reference_t *); + return svn_fs_x__id_part_eq(&reference->to, item->items); +} + +/* Starting at IDX in CONTEXT->PATH_ORDER, select all representations and + * noderevs that should be placed into the same container, respectively. + * Append the path_order_t * elements encountered in SELECTED, the + * svn_fs_x__p2l_entry_t * of the representations that should be placed + * into the same reps container will be appended to REP_PARTS and the + * svn_fs_x__p2l_entry_t * of the noderevs referencing those reps will + * be appended to NODE_PARTS. + * + * Remove all returned items from the CONTEXT->REPS container and prevent + * them from being placed a second time later on. That also means that the + * caller has to place all items returned. + */ +static svn_error_t * +select_reps(pack_context_t *context, + int idx, + apr_array_header_t *selected, + apr_array_header_t *node_parts, + apr_array_header_t *rep_parts) +{ + apr_array_header_t *path_order = context->path_order; + path_order_t *start_path = APR_ARRAY_IDX(path_order, idx, path_order_t *); + + svn_fs_x__p2l_entry_t *node_part; + svn_fs_x__p2l_entry_t *rep_part; + svn_fs_x__p2l_entry_t *depending; + int i, k; + + /* collect all path_order records as well as rep and noderev items + * that occupy the same path with the same node. */ + for (; idx < path_order->nelts; ++idx) + { + path_order_t *current_path + = APR_ARRAY_IDX(path_order, idx, path_order_t *); + + if (!svn_fs_x__id_part_eq(&start_path->node_id, + ¤t_path->node_id)) + break; + + APR_ARRAY_IDX(path_order, idx, path_order_t *) = NULL; + node_part = get_item(context, ¤t_path->noderev_id, TRUE); + rep_part = get_item(context, ¤t_path->rep_id, TRUE); + + if (node_part && rep_part) + APR_ARRAY_PUSH(selected, path_order_t *) = current_path; + + if (node_part) + APR_ARRAY_PUSH(node_parts, svn_fs_x__p2l_entry_t *) = node_part; + if (rep_part) + APR_ARRAY_PUSH(rep_parts, svn_fs_x__p2l_entry_t *) = rep_part; + } + + /* collect depending reps and noderevs that reference any of the collected + * reps */ + for (i = 0; i < rep_parts->nelts; ++i) + { + rep_part = APR_ARRAY_IDX(rep_parts, i, svn_fs_x__p2l_entry_t*); + for (k = find_first_reference(context, rep_part); + is_reference_match(context, k, rep_part); + ++k) + { + reference_t *reference + = APR_ARRAY_IDX(context->references, k, reference_t *); + + depending = get_item(context, &reference->from, TRUE); + if (!depending) + continue; + + if (depending->type == SVN_FS_X__ITEM_TYPE_NODEREV) + APR_ARRAY_PUSH(node_parts, svn_fs_x__p2l_entry_t *) = depending; + else + APR_ARRAY_PUSH(rep_parts, svn_fs_x__p2l_entry_t *) = depending; + } + } + + return SVN_NO_ERROR; +} + +/* Return TRUE, if all path_order_t * in SELECTED reference contents that is + * not longer than LIMIT. + */ +static svn_boolean_t +reps_fit_into_containers(apr_array_header_t *selected, + apr_uint64_t limit) +{ + int i; + for (i = 0; i < selected->nelts; ++i) + if (APR_ARRAY_IDX(selected, i, path_order_t *)->expanded_size > limit) + return FALSE; + + return TRUE; +} + +/* Write the *CONTAINER containing the noderevs described by the + * svn_fs_x__p2l_entry_t * in ITEMS to the pack file on CONTEXT. + * Append a P2L entry for the container to CONTAINER->REPS. + * Afterwards, clear ITEMS and re-allocate *CONTAINER in CONTAINER_POOL + * so the caller may fill them again. + * Use SCRATCH_POOL for temporary allocations. + */ +static svn_error_t * +write_nodes_container(pack_context_t *context, + svn_fs_x__noderevs_t **container, + apr_array_header_t *items, + apr_pool_t *container_pool, + apr_pool_t *scratch_pool) +{ + int i; + apr_off_t offset = 0; + svn_fs_x__p2l_entry_t *container_entry; + svn_stream_t *pack_stream; + + if (items->nelts == 0) + return SVN_NO_ERROR; + + /* serialize container */ + pack_stream = svn_stream_from_aprfile2(context->pack_file, TRUE, + scratch_pool); + + SVN_ERR(svn_fs_x__write_noderevs_container(pack_stream, *container, + scratch_pool)); + SVN_ERR(svn_io_file_seek(context->pack_file, APR_CUR, &offset, + scratch_pool)); + + /* replace first noderev item in ENTRIES with the container + and set all others to NULL */ + container_entry = apr_palloc(context->info_pool, sizeof(*container_entry)); + container_entry->offset = context->pack_offset; + container_entry->size = offset - container_entry->offset; + container_entry->type = SVN_FS_X__ITEM_TYPE_NODEREVS_CONT; + container_entry->item_count = items->nelts; + container_entry->items = apr_palloc(context->info_pool, + sizeof(svn_fs_x__id_part_t) * container_entry->item_count); + + for (i = 0; i < items->nelts; ++i) + container_entry->items[i] + = APR_ARRAY_IDX(items, i, svn_fs_x__p2l_entry_t *)->items[0]; + + context->pack_offset = offset; + APR_ARRAY_PUSH(context->reps, svn_fs_x__p2l_entry_t *) + = container_entry; + + /* Write P2L index for copied items, i.e. the 1 container */ + SVN_ERR(svn_fs_x__p2l_proto_index_add_entry + (context->proto_p2l_index, container_entry, scratch_pool)); + + svn_pool_clear(container_pool); + *container = svn_fs_x__noderevs_create(16, container_pool); + apr_array_clear(items); + + return SVN_NO_ERROR; +} + +/* Read the noderevs given by the svn_fs_x__p2l_entry_t * in NODE_PARTS + * from TEMP_FILE and add them to *CONTAINER and NODES_IN_CONTAINER. + * Whenever the container grows bigger than the current block in CONTEXT, + * write the data to disk and continue in the next block. + * + * Use CONTAINER_POOL to re-allocate the *CONTAINER as necessary and + * SCRATCH_POOL to temporary allocations. + */ +static svn_error_t * +store_nodes(pack_context_t *context, + apr_file_t *temp_file, + apr_array_header_t *node_parts, + svn_fs_x__noderevs_t **container, + apr_array_header_t *nodes_in_container, + apr_pool_t *container_pool, + apr_pool_t *scratch_pool) +{ + int i; + + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + svn_stream_t *stream + = svn_stream_from_aprfile2(temp_file, TRUE, scratch_pool); + + /* number of bytes in the current block not being spent on fixed-size + items (i.e. those not put into the container). */ + apr_size_t capacity_left = get_block_left(context); + + /* Estimated noderev container size */ + apr_size_t last_container_size = 0, container_size = 0; + + /* Estimate extra capacity we will gain from container compression. */ + apr_size_t pack_savings = 0; + for (i = 0; i < node_parts->nelts; ++i) + { + node_revision_t *noderev; + svn_fs_x__p2l_entry_t *entry + = APR_ARRAY_IDX(node_parts, i, svn_fs_x__p2l_entry_t *); + + /* if we reached the limit, check whether we saved some space + through the container. */ + if (capacity_left + pack_savings < container_size + entry->size) + container_size = svn_fs_x__noderevs_estimate_size(*container); + + /* If necessary and the container is large enough, try harder + by actually serializing the container and determine current + savings due to compression. */ + if ( capacity_left + pack_savings < container_size + entry->size + && container_size > last_container_size + 2000) + { + svn_stringbuf_t *serialized + = svn_stringbuf_create_ensure(container_size, iterpool); + svn_stream_t *temp_stream + = svn_stream_from_stringbuf(serialized, iterpool); + + SVN_ERR(svn_fs_x__write_noderevs_container(temp_stream, *container, + iterpool)); + SVN_ERR(svn_stream_close(temp_stream)); + + last_container_size = container_size; + pack_savings = container_size - serialized->len; + } + + /* still doesn't fit? -> block is full. Flush */ + if ( capacity_left + pack_savings < container_size + entry->size + && nodes_in_container->nelts < 2) + { + SVN_ERR(auto_pad_block(context, iterpool)); + capacity_left = get_block_left(context); + } + + /* still doesn't fit? -> block is full. Flush */ + if (capacity_left + pack_savings < container_size + entry->size) + { + SVN_ERR(write_nodes_container(context, container, + nodes_in_container, container_pool, + iterpool)); + + capacity_left = get_block_left(context); + pack_savings = 0; + container_size = 0; + } + + /* item will fit into the block. */ + SVN_ERR(svn_io_file_seek(temp_file, APR_SET, &entry->offset, iterpool)); + SVN_ERR(svn_fs_x__read_noderev(&noderev, stream, iterpool)); + svn_fs_x__noderevs_add(*container, noderev); + + container_size += entry->size; + APR_ARRAY_PUSH(nodes_in_container, svn_fs_x__p2l_entry_t *) = entry; + + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + + +/* Finalize CONTAINER and write it to CONTEXT's pack file. + * Append an P2L entry containing the given SUB_ITEMS to NEW_ENTRIES. + * Use POOL for temporary allocations. + */ +static svn_error_t * +write_reps_container(pack_context_t *context, + svn_fs_x__reps_builder_t *container, + apr_array_header_t *sub_items, + apr_array_header_t *new_entries, + apr_pool_t *pool) +{ + apr_off_t offset = 0; + svn_fs_x__p2l_entry_t container_entry; + + svn_stream_t *pack_stream + = svn_stream_from_aprfile2(context->pack_file, TRUE, pool); + + SVN_ERR(svn_fs_x__write_reps_container(pack_stream, container, pool)); + SVN_ERR(svn_io_file_seek(context->pack_file, SEEK_CUR, &offset, pool)); + + container_entry.offset = context->pack_offset; + container_entry.size = offset - container_entry.offset; + container_entry.type = SVN_FS_X__ITEM_TYPE_REPS_CONT; + container_entry.item_count = sub_items->nelts; + container_entry.items = (svn_fs_x__id_part_t *)sub_items->elts; + + context->pack_offset = offset; + APR_ARRAY_PUSH(new_entries, svn_fs_x__p2l_entry_t *) + = svn_fs_x__p2l_entry_dup(&container_entry, context->info_pool); + + SVN_ERR(svn_fs_x__p2l_proto_index_add_entry + (context->proto_p2l_index, &container_entry, pool)); + + return SVN_NO_ERROR; +} + +/* Read the (property) representations identified by svn_fs_x__p2l_entry_t + * elements in ENTRIES from TEMP_FILE, aggregate them and write them into + * CONTEXT->PACK_FILE. Use POOL for temporary allocations. + */ +static svn_error_t * +write_reps_containers(pack_context_t *context, + apr_array_header_t *entries, + apr_file_t *temp_file, + apr_array_header_t *new_entries, + apr_pool_t *pool) +{ + apr_pool_t *iterpool = svn_pool_create(pool); + apr_pool_t *container_pool = svn_pool_create(pool); + int i; + + apr_ssize_t block_left = get_block_left(context); + + svn_fs_x__reps_builder_t *container + = svn_fs_x__reps_builder_create(context->fs, container_pool); + apr_array_header_t *sub_items + = apr_array_make(pool, 64, sizeof(svn_fs_x__id_part_t)); + svn_stream_t *temp_stream + = svn_stream_from_aprfile2(temp_file, TRUE, pool); + + /* copy all items in strict order */ + for (i = entries->nelts-1; i >= 0; --i) + { + representation_t representation = { 0 }; + svn_stringbuf_t *contents; + svn_stream_t *stream; + apr_size_t list_index; + svn_fs_x__p2l_entry_t *entry + = APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t *); + + if ((block_left < entry->size) && sub_items->nelts) + { + block_left = get_block_left(context) + - svn_fs_x__reps_estimate_size(container); + } + + if ((block_left < entry->size) && sub_items->nelts) + { + SVN_ERR(write_reps_container(context, container, sub_items, + new_entries, iterpool)); + + apr_array_clear(sub_items); + svn_pool_clear(container_pool); + container = svn_fs_x__reps_builder_create(context->fs, + container_pool); + block_left = get_block_left(context); + } + + /* still enough space in current block? */ + if (block_left < entry->size) + { + SVN_ERR(auto_pad_block(context, iterpool)); + block_left = get_block_left(context); + } + + assert(entry->item_count == 1); + representation.revision = entry->items[0].revision; + representation.item_index = entry->items[0].number; + svn_fs_x__id_txn_reset(&representation.txn_id); + + /* select the change list in the source file, parse it and add it to + * the container */ + SVN_ERR(svn_io_file_seek(temp_file, SEEK_SET, &entry->offset, + iterpool)); + SVN_ERR(svn_fs_x__get_representation_length(&representation.size, + &representation.expanded_size, + context->fs, temp_file, + temp_stream, entry, iterpool)); + SVN_ERR(svn_fs_x__get_contents(&stream, context->fs, &representation, + iterpool)); + contents = svn_stringbuf_create_ensure(representation.expanded_size, + iterpool); + contents->len = representation.expanded_size; + + /* The representation is immutable. Read it normally. */ + SVN_ERR(svn_stream_read(stream, contents->data, &contents->len)); + SVN_ERR(svn_stream_close(stream)); + + list_index = svn_fs_x__reps_add(container, + svn_stringbuf__morph_into_string(contents)); + SVN_ERR_ASSERT(list_index == sub_items->nelts); + block_left -= entry->size; + + APR_ARRAY_PUSH(sub_items, svn_fs_x__id_part_t) = entry->items[0]; + + svn_pool_clear(iterpool); + } + + if (sub_items->nelts) + SVN_ERR(write_reps_container(context, container, sub_items, + new_entries, iterpool)); + + svn_pool_destroy(iterpool); + svn_pool_destroy(container_pool); + + return SVN_NO_ERROR; +} + +/* Return TRUE if the estimated size of the NODES_IN_CONTAINER plus the + * representations given as svn_fs_x__p2l_entry_t * in ENTRIES may exceed + * the space left in the current block. + */ +static svn_boolean_t +should_flush_nodes_container(pack_context_t *context, + svn_fs_x__noderevs_t *nodes_container, + apr_array_header_t *entries) +{ + apr_ssize_t block_left = get_block_left(context); + apr_ssize_t rep_sum = 0; + apr_ssize_t container_size + = svn_fs_x__noderevs_estimate_size(nodes_container); + + int i; + for (i = 0; i < entries->nelts; ++i) + { + svn_fs_x__p2l_entry_t *entry + = APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t *); + rep_sum += entry->size; + } + + return block_left < rep_sum + container_size; +} + +/* Read the contents of the first COUNT non-NULL, non-empty items in ITEMS + * from TEMP_FILE and write them to CONTEXT->PACK_FILE. + * Use POOL for allocations. + */ +static svn_error_t * +store_items(pack_context_t *context, + apr_file_t *temp_file, + apr_array_header_t *items, + int count, + apr_pool_t *pool) +{ + int i; + apr_pool_t *iterpool = svn_pool_create(pool); + + /* copy all items in strict order */ + for (i = 0; i < count; ++i) + { + svn_fs_x__p2l_entry_t *entry + = APR_ARRAY_IDX(items, i, svn_fs_x__p2l_entry_t *); + if (!entry + || entry->type == SVN_FS_X__ITEM_TYPE_UNUSED + || entry->item_count == 0) + continue; + + /* select the item in the source file and copy it into the target + * pack file */ + SVN_ERR(svn_io_file_seek(temp_file, SEEK_SET, &entry->offset, + iterpool)); + SVN_ERR(copy_file_data(context, context->pack_file, temp_file, + entry->size, iterpool)); + + /* write index entry and update current position */ + entry->offset = context->pack_offset; + context->pack_offset += entry->size; + + SVN_ERR(svn_fs_x__p2l_proto_index_add_entry + (context->proto_p2l_index, entry, iterpool)); + + APR_ARRAY_PUSH(context->reps, svn_fs_x__p2l_entry_t *) = entry; + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Copy (append) the items identified by svn_fs_x__p2l_entry_t * elements + * in ENTRIES strictly in order from TEMP_FILE into CONTEXT->PACK_FILE. + * Use POOL for temporary allocations. + */ +static svn_error_t * +copy_reps_from_temp(pack_context_t *context, + apr_file_t *temp_file, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = context->fs->fsap_data; + + apr_pool_t *iterpool = svn_pool_create(pool); + apr_pool_t *container_pool = svn_pool_create(pool); + apr_array_header_t *path_order = context->path_order; + apr_array_header_t *reps = context->reps; + apr_array_header_t *selected = apr_array_make(pool, 16, + path_order->elt_size); + apr_array_header_t *node_parts = apr_array_make(pool, 16, + reps->elt_size); + apr_array_header_t *rep_parts = apr_array_make(pool, 16, + reps->elt_size); + apr_array_header_t *nodes_in_container = apr_array_make(pool, 16, + reps->elt_size); + int i, k; + int initial_reps_count = reps->nelts; + + /* 1 container for all noderevs in the current block. We will try to + * not write it to disk until the current block fills up, i.e. aim for + * a single noderevs container per block. */ + svn_fs_x__noderevs_t *nodes_container + = svn_fs_x__noderevs_create(16, container_pool); + + /* copy items in path order. Create block-sized containers. */ + for (i = 0; i < path_order->nelts; ++i) + { + if (APR_ARRAY_IDX(path_order, i, path_order_t *) == NULL) + continue; + + /* Collect reps to combine and all noderevs referencing them */ + SVN_ERR(select_reps(context, i, selected, node_parts, rep_parts)); + + /* store the noderevs container in front of the reps */ + SVN_ERR(store_nodes(context, temp_file, node_parts, &nodes_container, + nodes_in_container, container_pool, iterpool)); + + /* actually flush the noderevs to disk if the reps container is likely + * to fill the block, i.e. no further noderevs will be added to the + * nodes container. */ + if (should_flush_nodes_container(context, nodes_container, node_parts)) + SVN_ERR(write_nodes_container(context, &nodes_container, + nodes_in_container, container_pool, + iterpool)); + + /* if all reps are short enough put them into one container. + * Otherwise, just store all containers here. */ + if (reps_fit_into_containers(selected, 2 * ffd->block_size)) + SVN_ERR(write_reps_containers(context, rep_parts, temp_file, + context->reps, iterpool)); + else + SVN_ERR(store_items(context, temp_file, rep_parts, rep_parts->nelts, + iterpool)); + + /* processed all items */ + apr_array_clear(selected); + apr_array_clear(node_parts); + apr_array_clear(rep_parts); + + svn_pool_clear(iterpool); + } + + /* flush noderevs container to disk */ + if (nodes_in_container->nelts) + SVN_ERR(write_nodes_container(context, &nodes_container, + nodes_in_container, container_pool, + iterpool)); + + /* copy all items in strict order */ + SVN_ERR(store_items(context, temp_file, reps, initial_reps_count, pool)); + + /* vaccum ENTRIES array: eliminate NULL entries */ + for (i = 0, k = 0; i < reps->nelts; ++i) + { + svn_fs_x__p2l_entry_t *entry + = APR_ARRAY_IDX(reps, i, svn_fs_x__p2l_entry_t *); + if (entry) + { + APR_ARRAY_IDX(reps, k, svn_fs_x__p2l_entry_t *) = entry; + ++k; + } + } + reps->nelts = k; + + svn_pool_destroy(iterpool); + svn_pool_destroy(container_pool); + + return SVN_NO_ERROR; +} + +/* Finalize CONTAINER and write it to CONTEXT's pack file. + * Append an P2L entry containing the given SUB_ITEMS to NEW_ENTRIES. + * Use POOL for temporary allocations. + */ +static svn_error_t * +write_changes_container(pack_context_t *context, + svn_fs_x__changes_t *container, + apr_array_header_t *sub_items, + apr_array_header_t *new_entries, + apr_pool_t *pool) +{ + apr_off_t offset = 0; + svn_fs_x__p2l_entry_t container_entry; + + svn_stream_t *pack_stream + = svn_stream_from_aprfile2(context->pack_file, TRUE, pool); + + SVN_ERR(svn_fs_x__write_changes_container(pack_stream, + container, + pool)); + SVN_ERR(svn_io_file_seek(context->pack_file, SEEK_CUR, &offset, pool)); + + container_entry.offset = context->pack_offset; + container_entry.size = offset - container_entry.offset; + container_entry.type = SVN_FS_X__ITEM_TYPE_CHANGES_CONT; + container_entry.item_count = sub_items->nelts; + container_entry.items = (svn_fs_x__id_part_t *)sub_items->elts; + + context->pack_offset = offset; + APR_ARRAY_PUSH(new_entries, svn_fs_x__p2l_entry_t *) + = svn_fs_x__p2l_entry_dup(&container_entry, context->info_pool); + + SVN_ERR(svn_fs_x__p2l_proto_index_add_entry + (context->proto_p2l_index, &container_entry, pool)); + + return SVN_NO_ERROR; +} + +/* Read the change lists identified by svn_fs_x__p2l_entry_t * elements + * in ENTRIES strictly in from TEMP_FILE, aggregate them and write them + * into CONTEXT->PACK_FILE. Use POOL for temporary allocations. + */ +static svn_error_t * +write_changes_containers(pack_context_t *context, + apr_array_header_t *entries, + apr_file_t *temp_file, + apr_pool_t *pool) +{ + apr_pool_t *iterpool = svn_pool_create(pool); + apr_pool_t *container_pool = svn_pool_create(pool); + int i; + + apr_ssize_t block_left = get_block_left(context); + apr_ssize_t estimated_addition = 0; + + svn_fs_x__changes_t *container + = svn_fs_x__changes_create(1000, container_pool); + apr_array_header_t *sub_items + = apr_array_make(pool, 64, sizeof(svn_fs_x__id_part_t)); + apr_array_header_t *new_entries + = apr_array_make(context->info_pool, 16, entries->elt_size); + svn_stream_t *temp_stream + = svn_stream_from_aprfile2(temp_file, TRUE, pool); + + /* copy all items in strict order */ + for (i = entries->nelts-1; i >= 0; --i) + { + apr_array_header_t *changes; + apr_size_t list_index; + svn_fs_x__p2l_entry_t *entry + = APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t *); + + /* zip compression alone will significantly reduce the size of large + * change lists. So, we will probably need even less than this estimate. + */ + apr_ssize_t estimated_size = (entry->size / 5) + 250; + + /* If necessary and enough data has been added to the container since + * the last test, try harder by actually serializing the container and + * determine current savings due to compression. */ + if (block_left < estimated_size && estimated_addition > 2000) + { + svn_stringbuf_t *serialized + = svn_stringbuf_create_ensure(get_block_left(context), iterpool); + svn_stream_t *memory_stream + = svn_stream_from_stringbuf(serialized, iterpool); + + SVN_ERR(svn_fs_x__write_changes_container(memory_stream, + container, iterpool)); + SVN_ERR(svn_stream_close(temp_stream)); + + block_left = get_block_left(context) - serialized->len; + estimated_addition = 0; + } + + if ((block_left < estimated_size) && sub_items->nelts) + { + SVN_ERR(write_changes_container(context, container, sub_items, + new_entries, iterpool)); + + apr_array_clear(sub_items); + svn_pool_clear(container_pool); + container = svn_fs_x__changes_create(1000, container_pool); + block_left = get_block_left(context); + estimated_addition = 0; + } + + /* still enough space in current block? */ + if (block_left < estimated_size) + { + SVN_ERR(auto_pad_block(context, iterpool)); + block_left = get_block_left(context); + } + + /* select the change list in the source file, parse it and add it to + * the container */ + SVN_ERR(svn_io_file_seek(temp_file, SEEK_SET, &entry->offset, + iterpool)); + SVN_ERR(svn_fs_x__read_changes(&changes, temp_stream, iterpool)); + SVN_ERR(svn_fs_x__changes_append_list(&list_index, container, changes)); + SVN_ERR_ASSERT(list_index == sub_items->nelts); + block_left -= estimated_size; + estimated_addition += estimated_size; + + APR_ARRAY_PUSH(sub_items, svn_fs_x__id_part_t) = entry->items[0]; + + svn_pool_clear(iterpool); + } + + if (sub_items->nelts) + SVN_ERR(write_changes_container(context, container, sub_items, + new_entries, iterpool)); + + *entries = *new_entries; + svn_pool_destroy(iterpool); + svn_pool_destroy(container_pool); + + return SVN_NO_ERROR; +} + +/* Read the (property) representations identified by svn_fs_x__p2l_entry_t + * elements in ENTRIES from TEMP_FILE, aggregate them and write them into + * CONTEXT->PACK_FILE. Use POOL for temporary allocations. + */ +static svn_error_t * +write_property_containers(pack_context_t *context, + apr_array_header_t *entries, + apr_file_t *temp_file, + apr_pool_t *pool) +{ + apr_array_header_t *new_entries + = apr_array_make(context->info_pool, 16, entries->elt_size); + + SVN_ERR(write_reps_containers(context, entries, temp_file, new_entries, + pool)); + + *entries = *new_entries; + + return SVN_NO_ERROR; +} + +/* Append all entries of svn_fs_x__p2l_entry_t * array TO_APPEND to + * svn_fs_x__p2l_entry_t * array DEST. + */ +static void +append_entries(apr_array_header_t *dest, + apr_array_header_t *to_append) +{ + int i; + for (i = 0; i < to_append->nelts; ++i) + APR_ARRAY_PUSH(dest, svn_fs_x__p2l_entry_t *) + = APR_ARRAY_IDX(to_append, i, svn_fs_x__p2l_entry_t *); +} + +/* Write the log-to-phys proto index file for CONTEXT and use POOL for + * temporary allocations. All items in all buckets must have been placed + * by now. + */ +static svn_error_t * +write_l2p_index(pack_context_t *context, + apr_pool_t *pool) +{ + apr_pool_t *iterpool = svn_pool_create(pool); + svn_revnum_t prev_rev = SVN_INVALID_REVNUM; + int i; + apr_uint32_t k; + svn_priority_queue__t *queue; + apr_size_t count = 0; + apr_array_header_t *sub_item_orders; + + /* lump all items into one bucket. As target, use the bucket that + * probably has the most entries already. */ + append_entries(context->reps, context->changes); + append_entries(context->reps, context->file_props); + append_entries(context->reps, context->dir_props); + + /* wrap P2L entries such that we have access to the sub-items in revision + order. The ENTRY_COUNT member will point to the next item to read+1. */ + sub_item_orders + = apr_array_make(pool, context->reps->nelts, sizeof(sub_item_ordered_t)); + sub_item_orders->nelts = context->reps->nelts; + + for (i = 0; i < context->reps->nelts; ++i) + { + svn_fs_x__p2l_entry_t *entry + = APR_ARRAY_IDX(context->reps, i, svn_fs_x__p2l_entry_t *); + sub_item_ordered_t *ordered + = &APR_ARRAY_IDX(sub_item_orders, i, sub_item_ordered_t); + + /* skip unused regions (e.g. padding) */ + if (entry->item_count == 0) + continue; + + assert(entry); + ordered->entry = entry; + count += entry->item_count; + + if (entry->item_count > 1) + { + ordered->order + = apr_palloc(pool, sizeof(*ordered->order) * entry->item_count); + for (k = 0; k < entry->item_count; ++k) + ordered->order[k] = &entry->items[k]; + + qsort(ordered->order, entry->item_count, sizeof(*ordered->order), + (int (*)(const void *, const void *))compare_sub_items); + } + } + + /* we need to write the index in ascending revision order */ + queue = svn_priority_queue__create + (sub_item_orders, + (int (*)(const void *, const void *))compare_p2l_info_rev); + + /* write index entries */ + for (i = 0; i < count; ++i) + { + svn_fs_x__id_part_t *sub_item; + sub_item_ordered_t *ordered = svn_priority_queue__peek(queue); + + if (ordered->entry->item_count > 0) + { + /* if there is only one item, we skip the overhead of having an + extra array for the item order */ + sub_item = ordered->order + ? ordered->order[ordered->entry->item_count - 1] + : &ordered->entry->items[0]; + + /* next revision? */ + if (prev_rev != sub_item->revision) + { + prev_rev = sub_item->revision; + SVN_ERR(svn_fs_x__l2p_proto_index_add_revision + (context->proto_l2p_index, iterpool)); + } + + /* add entry */ + SVN_ERR(svn_fs_x__l2p_proto_index_add_entry + (context->proto_l2p_index, ordered->entry->offset, + (apr_uint32_t)(sub_item - ordered->entry->items), + sub_item->number, iterpool)); + + /* make ITEM_COUNT point the next sub-item to use+1 */ + --ordered->entry->item_count; + } + + /* process remaining sub-items (if any) of that container later */ + if (ordered->entry->item_count) + svn_priority_queue__update(queue); + else + svn_priority_queue__pop(queue); + + /* keep memory usage in check */ + if (i % 256 == 0) + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Pack the current revision range of CONTEXT, i.e. this covers phases 2 + * to 4. Use POOL for allocations. + */ +static svn_error_t * +pack_range(pack_context_t *context, + apr_pool_t *pool) +{ + apr_pool_t *revpool = svn_pool_create(pool); + apr_pool_t *iterpool = svn_pool_create(pool); + + /* Phase 2: Copy items into various buckets and build tracking info */ + svn_revnum_t revision; + for (revision = context->start_rev; revision < context->end_rev; ++revision) + { + apr_off_t offset = 0; + apr_finfo_t finfo; + apr_file_t *rev_file; + + /* Get the size of the file. */ + const char *path = svn_dirent_join(context->shard_dir, + apr_psprintf(revpool, "%ld", + revision), + revpool); + SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, revpool)); + + SVN_ERR(svn_io_file_open(&rev_file, path, + APR_READ | APR_BUFFERED | APR_BINARY, + APR_OS_DEFAULT, revpool)); + + /* store the indirect array index */ + APR_ARRAY_PUSH(context->rev_offsets, int) = context->reps->nelts; + + /* read the phys-to-log index file until we covered the whole rev file. + * That index contains enough info to build both target indexes from it. */ + while (offset < finfo.size) + { + /* read one cluster */ + int i; + apr_array_header_t *entries; + SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, context->fs, + revision, offset, + iterpool)); + + for (i = 0; i < entries->nelts; ++i) + { + svn_fs_x__p2l_entry_t *entry + = &APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t); + + /* skip first entry if that was duplicated due crossing a + cluster boundary */ + if (offset > entry->offset) + continue; + + /* process entry while inside the rev file */ + offset = entry->offset; + if (offset < finfo.size) + { + SVN_ERR(svn_io_file_seek(rev_file, SEEK_SET, &offset, + iterpool)); + + if (entry->type == SVN_FS_X__ITEM_TYPE_CHANGES) + SVN_ERR(copy_item_to_temp(context, + context->changes, + context->changes_file, + rev_file, entry, iterpool)); + else if (entry->type == SVN_FS_X__ITEM_TYPE_FILE_PROPS) + SVN_ERR(copy_item_to_temp(context, + context->file_props, + context->file_props_file, + rev_file, entry, iterpool)); + else if (entry->type == SVN_FS_X__ITEM_TYPE_DIR_PROPS) + SVN_ERR(copy_item_to_temp(context, + context->dir_props, + context->dir_props_file, + rev_file, entry, iterpool)); + else if ( entry->type == SVN_FS_X__ITEM_TYPE_FILE_REP + || entry->type == SVN_FS_X__ITEM_TYPE_DIR_REP) + SVN_ERR(copy_rep_to_temp(context, rev_file, entry, + iterpool)); + else if (entry->type == SVN_FS_X__ITEM_TYPE_NODEREV) + SVN_ERR(copy_node_to_temp(context, rev_file, entry, + iterpool)); + else + SVN_ERR_ASSERT(entry->type == SVN_FS_X__ITEM_TYPE_UNUSED); + + offset += entry->size; + } + } + + if (context->cancel_func) + SVN_ERR(context->cancel_func(context->cancel_baton)); + + svn_pool_clear(iterpool); + } + + svn_pool_clear(revpool); + } + + svn_pool_destroy(iterpool); + + /* phase 3: placement. + * Use "newest first" placement for simple items. */ + sort_items(context->changes); + sort_items(context->file_props); + sort_items(context->dir_props); + + /* follow dependencies recursively for noderevs and data representations */ + sort_reps(context); + + /* phase 4: copy bucket data to pack file. Write P2L index. */ + SVN_ERR(write_changes_containers(context, context->changes, + context->changes_file, revpool)); + svn_pool_clear(revpool); + SVN_ERR(write_property_containers(context, context->file_props, + context->file_props_file, revpool)); + svn_pool_clear(revpool); + SVN_ERR(write_property_containers(context, context->dir_props, + context->dir_props_file, revpool)); + svn_pool_clear(revpool); + SVN_ERR(copy_reps_from_temp(context, context->reps_file, revpool)); + svn_pool_clear(revpool); + + /* write L2P index as well (now that we know all target offsets) */ + SVN_ERR(write_l2p_index(context, revpool)); + + svn_pool_destroy(revpool); + + return SVN_NO_ERROR; +} + +/* Append CONTEXT->START_REV to the context's pack file with no re-ordering. + * This function will only be used for very large revisions (>>100k changes). + * Use POOL for temporary allocations. + */ +static svn_error_t * +append_revision(pack_context_t *context, + apr_pool_t *pool) +{ + apr_off_t offset = 0; + apr_pool_t *iterpool = svn_pool_create(pool); + apr_file_t *rev_file; + apr_finfo_t finfo; + + /* Get the size of the file. */ + const char *path = svn_dirent_join(context->shard_dir, + apr_psprintf(iterpool, "%ld", + context->start_rev), + pool); + SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, pool)); + + /* Copy all the bits from the rev file to the end of the pack file. */ + SVN_ERR(svn_io_file_open(&rev_file, path, + APR_READ | APR_BUFFERED | APR_BINARY, + APR_OS_DEFAULT, pool)); + SVN_ERR(copy_file_data(context, context->pack_file, rev_file, finfo.size, + iterpool)); + + /* mark the start of a new revision */ + SVN_ERR(svn_fs_x__l2p_proto_index_add_revision(context->proto_l2p_index, + pool)); + + /* read the phys-to-log index file until we covered the whole rev file. + * That index contains enough info to build both target indexes from it. */ + while (offset < finfo.size) + { + /* read one cluster */ + int i; + apr_array_header_t *entries; + SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, context->fs, + context->start_rev, offset, + iterpool)); + + for (i = 0; i < entries->nelts; ++i) + { + svn_fs_x__p2l_entry_t *entry + = &APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t); + + /* skip first entry if that was duplicated due crossing a + cluster boundary */ + if (offset > entry->offset) + continue; + + /* process entry while inside the rev file */ + offset = entry->offset; + if (offset < finfo.size) + { + /* there should be true containers */ + SVN_ERR_ASSERT(entry->item_count == 1); + + entry->offset += context->pack_offset; + offset += entry->size; + SVN_ERR(svn_fs_x__l2p_proto_index_add_entry + (context->proto_l2p_index, entry->offset, 0, + entry->items[0].number, iterpool)); + SVN_ERR(svn_fs_x__p2l_proto_index_add_entry + (context->proto_p2l_index, entry, iterpool)); + } + } + + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + context->pack_offset += finfo.size; + + return SVN_NO_ERROR; +} + +/* Format 7 packing logic. + * + * Pack the revision shard starting at SHARD_REV in filesystem FS from + * SHARD_DIR into the PACK_FILE_DIR, using POOL for allocations. Limit + * the extra memory consumption to MAX_MEM bytes. CANCEL_FUNC and + * CANCEL_BATON are what you think they are. + */ +static svn_error_t * +pack_log_addressed(svn_fs_t *fs, + const char *pack_file_dir, + const char *shard_dir, + svn_revnum_t shard_rev, + apr_size_t max_mem, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + enum + { + /* estimated amount of memory used to represent one item in memory + * during rev file packing */ + PER_ITEM_MEM = APR_ALIGN_DEFAULT(sizeof(path_order_t)) + + APR_ALIGN_DEFAULT(2 *sizeof(void*)) + + APR_ALIGN_DEFAULT(sizeof(reference_t)) + + APR_ALIGN_DEFAULT(sizeof(svn_fs_x__p2l_entry_t)) + + 6 * sizeof(void*) + }; + + apr_size_t max_items = max_mem / PER_ITEM_MEM; + apr_array_header_t *max_ids; + pack_context_t context = { 0 }; + int i; + apr_size_t item_count = 0; + apr_pool_t *iterpool = svn_pool_create(pool); + + /* set up a pack context */ + SVN_ERR(initialize_pack_context(&context, fs, pack_file_dir, shard_dir, + shard_rev, max_items, cancel_func, + cancel_baton, pool)); + + /* phase 1: determine the size of the revisions to pack */ + SVN_ERR(svn_fs_x__l2p_get_max_ids(&max_ids, fs, shard_rev, + context.shard_end_rev - shard_rev, + pool)); + + /* pack revisions in ranges that don't exceed MAX_MEM */ + for (i = 0; i < max_ids->nelts; ++i) + if (APR_ARRAY_IDX(max_ids, i, apr_uint64_t) + item_count <= max_items) + { + context.end_rev++; + } + else + { + /* some unpacked revisions before this one? */ + if (context.start_rev < context.end_rev) + { + /* pack them intelligently (might be just 1 rev but still ...) */ + SVN_ERR(pack_range(&context, iterpool)); + SVN_ERR(reset_pack_context(&context, iterpool)); + item_count = 0; + } + + /* next revision range is to start with the current revision */ + context.start_rev = i + context.shard_rev; + context.end_rev = context.start_rev + 1; + + /* if this is a very large revision, we must place it as is */ + if (APR_ARRAY_IDX(max_ids, i, apr_uint64_t) > max_items) + { + SVN_ERR(append_revision(&context, iterpool)); + context.start_rev++; + } + else + item_count += (apr_size_t)APR_ARRAY_IDX(max_ids, i, apr_uint64_t); + + svn_pool_clear(iterpool); + } + + /* non-empty revision range at the end? */ + if (context.start_rev < context.end_rev) + SVN_ERR(pack_range(&context, iterpool)); + + /* last phase: finalize indexes and clean up */ + SVN_ERR(reset_pack_context(&context, iterpool)); + SVN_ERR(close_pack_context(&context, iterpool)); + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file. + Use POOL for temporary allocations. */ +svn_error_t * +svn_fs_x__get_packed_offset(apr_off_t *rev_offset, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_stream_t *manifest_stream; + svn_boolean_t is_cached; + svn_revnum_t shard; + apr_int64_t shard_pos; + apr_array_header_t *manifest; + apr_pool_t *iterpool; + + shard = rev / ffd->max_files_per_dir; + + /* position of the shard within the manifest */ + shard_pos = rev % ffd->max_files_per_dir; + + /* fetch exactly that element into *rev_offset, if the manifest is found + in the cache */ + SVN_ERR(svn_cache__get_partial((void **) rev_offset, &is_cached, + ffd->packed_offset_cache, &shard, + svn_fs_x__get_sharded_offset, &shard_pos, + pool)); + + if (is_cached) + return SVN_NO_ERROR; + + /* Open the manifest file. */ + SVN_ERR(svn_stream_open_readonly(&manifest_stream, + svn_fs_x__path_rev_packed(fs, rev, PATH_MANIFEST, pool), + pool, pool)); + + /* While we're here, let's just read the entire manifest file into an array, + so we can cache the entire thing. */ + iterpool = svn_pool_create(pool); + manifest = apr_array_make(pool, ffd->max_files_per_dir, sizeof(apr_off_t)); + while (1) + { + svn_boolean_t eof; + apr_int64_t val; + + svn_pool_clear(iterpool); + SVN_ERR(svn_fs_x__read_number_from_stream(&val, &eof, manifest_stream, + iterpool)); + if (eof) + break; + + APR_ARRAY_PUSH(manifest, apr_off_t) = (apr_off_t)val; + } + svn_pool_destroy(iterpool); + + *rev_offset = APR_ARRAY_IDX(manifest, rev % ffd->max_files_per_dir, + apr_off_t); + + /* Close up shop and cache the array. */ + SVN_ERR(svn_stream_close(manifest_stream)); + return svn_cache__set(ffd->packed_offset_cache, &shard, manifest, pool); +} + +/* In filesystem FS, pack the revision SHARD containing exactly + * MAX_FILES_PER_DIR revisions from SHARD_PATH into the PACK_FILE_DIR, + * using POOL for allocations. Try to limit the amount of temporary + * memory needed to MAX_MEM bytes. CANCEL_FUNC and CANCEL_BATON are what + * you think they are. + * + * If for some reason we detect a partial packing already performed, we + * remove the pack file and start again. + * + * The actual packing will be done in a format-specific sub-function. + */ +static svn_error_t * +pack_rev_shard(svn_fs_t *fs, + const char *pack_file_dir, + const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + apr_size_t max_mem, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + const char *pack_file_path; + svn_revnum_t shard_rev = (svn_revnum_t) (shard * max_files_per_dir); + + /* Some useful paths. */ + pack_file_path = svn_dirent_join(pack_file_dir, PATH_PACKED, pool); + + /* Remove any existing pack file for this shard, since it is incomplete. */ + SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton, + pool)); + + /* Create the new directory and pack file. */ + SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, pool)); + + /* Index information files */ + SVN_ERR(pack_log_addressed(fs, pack_file_dir, shard_path, shard_rev, + max_mem, cancel_func, cancel_baton, pool)); + + SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, pool)); + SVN_ERR(svn_io_set_file_read_only(pack_file_path, FALSE, pool)); + + return SVN_NO_ERROR; +} + +/* In the file system at FS_PATH, pack the SHARD in REVS_DIR and + * REVPROPS_DIR containing exactly MAX_FILES_PER_DIR revisions, using POOL + * for allocations. REVPROPS_DIR will be NULL if revprop packing is not + * supported. COMPRESSION_LEVEL and MAX_PACK_SIZE will be ignored in that + * case. + * + * CANCEL_FUNC and CANCEL_BATON are what you think they are; similarly + * NOTIFY_FUNC and NOTIFY_BATON. + * + * If for some reason we detect a partial packing already performed, we + * remove the pack file and start again. + */ +static svn_error_t * +pack_shard(const char *revs_dir, + const char *revsprops_dir, + svn_fs_t *fs, + apr_int64_t shard, + int max_files_per_dir, + apr_off_t max_pack_size, + int compression_level, + svn_fs_pack_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + const char *rev_shard_path, *rev_pack_file_dir; + const char *revprops_shard_path, *revprops_pack_file_dir; + + /* Notify caller we're starting to pack this shard. */ + if (notify_func) + SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start, + pool)); + + /* Some useful paths. */ + rev_pack_file_dir = svn_dirent_join(revs_dir, + apr_psprintf(pool, + "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, + shard), + pool); + rev_shard_path = svn_dirent_join(revs_dir, + apr_psprintf(pool, "%" APR_INT64_T_FMT, shard), + pool); + + /* pack the revision content */ + SVN_ERR(pack_rev_shard(fs, rev_pack_file_dir, rev_shard_path, + shard, max_files_per_dir, 64 * 1024 * 1024, + cancel_func, cancel_baton, pool)); + + /* if enabled, pack the revprops in an equivalent way */ + if (revsprops_dir) + { + revprops_pack_file_dir = svn_dirent_join(revsprops_dir, + apr_psprintf(pool, + "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, + shard), + pool); + revprops_shard_path = svn_dirent_join(revsprops_dir, + apr_psprintf(pool, "%" APR_INT64_T_FMT, shard), + pool); + + SVN_ERR(svn_fs_x__pack_revprops_shard(revprops_pack_file_dir, + revprops_shard_path, + shard, max_files_per_dir, + (int)(0.9 * max_pack_size), + compression_level, + cancel_func, cancel_baton, + pool)); + } + + /* Update the min-unpacked-rev file to reflect our newly packed shard. */ + SVN_ERR(svn_fs_x__write_revnum_file(fs, + (svn_revnum_t)((shard + 1) * max_files_per_dir), + pool)); + ffd->min_unpacked_rev = (svn_revnum_t)((shard + 1) * max_files_per_dir); + + /* Finally, remove the existing shard directories. + * For revprops, clean up older obsolete shards as well as they might + * have been left over from an interrupted FS upgrade. */ + SVN_ERR(svn_io_remove_dir2(rev_shard_path, TRUE, + cancel_func, cancel_baton, pool)); + if (revsprops_dir) + { + svn_node_kind_t kind = svn_node_dir; + apr_int64_t to_cleanup = shard; + do + { + SVN_ERR(svn_fs_x__delete_revprops_shard(revprops_shard_path, + to_cleanup, + max_files_per_dir, + cancel_func, cancel_baton, + pool)); + + /* If the previous shard exists, clean it up as well. + Don't try to clean up shard 0 as it we can't tell quickly + whether it actually needs cleaning up. */ + revprops_shard_path = svn_dirent_join(revsprops_dir, + apr_psprintf(pool, "%" APR_INT64_T_FMT, --to_cleanup), + pool); + SVN_ERR(svn_io_check_path(revprops_shard_path, &kind, pool)); + } + while (kind == svn_node_dir && to_cleanup > 0); + } + + /* Notify caller we're starting to pack this shard. */ + if (notify_func) + SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_end, + pool)); + + return SVN_NO_ERROR; +} + +struct pack_baton +{ + svn_fs_t *fs; + svn_fs_pack_notify_t notify_func; + void *notify_baton; + svn_cancel_func_t cancel_func; + void *cancel_baton; +}; + + +/* The work-horse for svn_fs_x__pack, called with the FS write lock. + This implements the svn_fs_x__with_write_lock() 'body' callback + type. BATON is a 'struct pack_baton *'. + + WARNING: if you add a call to this function, please note: + The code currently assumes that any piece of code running with + the write-lock set can rely on the ffd->min_unpacked_rev and + ffd->min_unpacked_revprop caches to be up-to-date (and, by + extension, on not having to use a retry when calling + svn_fs_x__path_rev_absolute() and friends). If you add a call + to this function, consider whether you have to call + update_min_unpacked_rev(). + See this thread: http://thread.gmane.org/1291206765.3782.3309.camel@edith + */ +static svn_error_t * +pack_body(void *baton, + apr_pool_t *pool) +{ + struct pack_baton *pb = baton; + fs_x_data_t *ffd = pb->fs->fsap_data; + apr_int64_t completed_shards; + apr_int64_t i; + svn_revnum_t youngest; + apr_pool_t *iterpool; + const char *rev_data_path; + const char *revprops_data_path = NULL; + + /* If we aren't using sharding, we can't do any packing, so quit. */ + SVN_ERR(svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, pb->fs, + pool)); + + SVN_ERR(svn_fs_x__youngest_rev(&youngest, pb->fs, pool)); + completed_shards = (youngest + 1) / ffd->max_files_per_dir; + + /* See if we've already completed all possible shards thus far. */ + if (ffd->min_unpacked_rev == (completed_shards * ffd->max_files_per_dir)) + return SVN_NO_ERROR; + + rev_data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, pool); + revprops_data_path = svn_dirent_join(pb->fs->path, PATH_REVPROPS_DIR, + pool); + + iterpool = svn_pool_create(pool); + for (i = ffd->min_unpacked_rev / ffd->max_files_per_dir; + i < completed_shards; + i++) + { + svn_pool_clear(iterpool); + + if (pb->cancel_func) + SVN_ERR(pb->cancel_func(pb->cancel_baton)); + + SVN_ERR(pack_shard(rev_data_path, revprops_data_path, + pb->fs, i, ffd->max_files_per_dir, + ffd->revprop_pack_size, + ffd->compress_packed_revprops + ? SVN__COMPRESSION_ZLIB_DEFAULT + : SVN__COMPRESSION_NONE, + pb->notify_func, pb->notify_baton, + pb->cancel_func, pb->cancel_baton, iterpool)); + } + + svn_pool_destroy(iterpool); + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__pack(svn_fs_t *fs, + svn_fs_pack_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + struct pack_baton pb = { 0 }; + pb.fs = fs; + pb.notify_func = notify_func; + pb.notify_baton = notify_baton; + pb.cancel_func = cancel_func; + pb.cancel_baton = cancel_baton; + return svn_fs_x__with_write_lock(fs, pack_body, &pb, pool); +}
diff --git a/subversion/libsvn_fs_x/pack.h b/subversion/libsvn_fs_x/pack.h new file mode 100644 index 0000000..abb1c84 --- /dev/null +++ b/subversion/libsvn_fs_x/pack.h
@@ -0,0 +1,62 @@ +/* pack.h : interface FSX pack functionality + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS__PACK_H +#define SVN_LIBSVN_FS__PACK_H + +#include "fs.h" + +/* Possibly pack the repository at PATH. This just take full shards, and + combines all the revision files into a single one, with a manifest header. + Use optional CANCEL_FUNC/CANCEL_BATON for cancellation support. + + Existing filesystem references need not change. */ +svn_error_t * +svn_fs_x__pack(svn_fs_t *fs, + svn_fs_pack_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool); + +/** + * For the packed revision @a rev in @a fs, determine the offset within + * the revision pack file and return it in @a rev_offset. Use @a pool for + * allocations. + */ +svn_error_t * +svn_fs_x__get_packed_offset(apr_off_t *rev_offset, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +/* Return the svn_dir_entry_t* objects of DIRECTORY in an APR array + * allocated in POOL with entries added in storage (on-disk) order. + * FS format will be used to pick the optimal ordering strategy. + */ +apr_array_header_t * +svn_fs_x__order_dir_entries(svn_fs_t *fs, + apr_hash_t *directory, + apr_pool_t *pool); + + +#endif
diff --git a/subversion/libsvn_fs_x/recovery.c b/subversion/libsvn_fs_x/recovery.c new file mode 100644 index 0000000..efd3f86 --- /dev/null +++ b/subversion/libsvn_fs_x/recovery.c
@@ -0,0 +1,238 @@ +/* recovery.c --- FSX recovery functionality +* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "recovery.h" + +#include "svn_hash.h" +#include "svn_pools.h" +#include "private/svn_string_private.h" + +#include "low_level.h" +#include "rep-cache.h" +#include "revprops.h" +#include "transaction.h" +#include "util.h" +#include "cached_data.h" +#include "index.h" + +#include "../libsvn_fs/fs-loader.h" + +#include "svn_private_config.h" + +/* Part of the recovery procedure. Return the largest revision *REV in + filesystem FS. Use POOL for temporary allocation. */ +static svn_error_t * +recover_get_largest_revision(svn_fs_t *fs, svn_revnum_t *rev, apr_pool_t *pool) +{ + /* Discovering the largest revision in the filesystem would be an + expensive operation if we did a readdir() or searched linearly, + so we'll do a form of binary search. left is a revision that we + know exists, right a revision that we know does not exist. */ + apr_pool_t *iterpool; + svn_revnum_t left, right = 1; + + iterpool = svn_pool_create(pool); + /* Keep doubling right, until we find a revision that doesn't exist. */ + while (1) + { + svn_error_t *err; + apr_file_t *file; + + err = svn_fs_x__open_pack_or_rev_file(&file, fs, right, iterpool); + svn_pool_clear(iterpool); + + if (err && err->apr_err == SVN_ERR_FS_NO_SUCH_REVISION) + { + svn_error_clear(err); + break; + } + else + SVN_ERR(err); + + right <<= 1; + } + + left = right >> 1; + + /* We know that left exists and right doesn't. Do a normal bsearch to find + the last revision. */ + while (left + 1 < right) + { + svn_revnum_t probe = left + ((right - left) / 2); + svn_error_t *err; + apr_file_t *file; + + err = svn_fs_x__open_pack_or_rev_file(&file, fs, probe, iterpool); + svn_pool_clear(iterpool); + + if (err && err->apr_err == SVN_ERR_FS_NO_SUCH_REVISION) + { + svn_error_clear(err); + right = probe; + } + else + { + SVN_ERR(err); + left = probe; + } + } + + svn_pool_destroy(iterpool); + + /* left is now the largest revision that exists. */ + *rev = left; + return SVN_NO_ERROR; +} + +/* Baton used for recover_body below. */ +struct recover_baton { + svn_fs_t *fs; + svn_cancel_func_t cancel_func; + void *cancel_baton; +}; + +/* The work-horse for svn_fs_x__recover, called with the FS + write lock. This implements the svn_fs_x__with_write_lock() + 'body' callback type. BATON is a 'struct recover_baton *'. */ +static svn_error_t * +recover_body(void *baton, apr_pool_t *pool) +{ + struct recover_baton *b = baton; + svn_fs_t *fs = b->fs; + fs_x_data_t *ffd = fs->fsap_data; + svn_revnum_t max_rev; + svn_revnum_t youngest_rev; + svn_node_kind_t youngest_revprops_kind; + + /* Lose potentially corrupted data in temp files */ + SVN_ERR(svn_fs_x__cleanup_revprop_namespace(fs)); + + /* We need to know the largest revision in the filesystem. */ + SVN_ERR(recover_get_largest_revision(fs, &max_rev, pool)); + + /* Get the expected youngest revision */ + SVN_ERR(svn_fs_x__youngest_rev(&youngest_rev, fs, pool)); + + /* Policy note: + + Since the revprops file is written after the revs file, the true + maximum available revision is the youngest one for which both are + present. That's probably the same as the max_rev we just found, + but if it's not, we could, in theory, repeatedly decrement + max_rev until we find a revision that has both a revs and + revprops file, then write db/current with that. + + But we choose not to. If a repository is so corrupt that it's + missing at least one revprops file, we shouldn't assume that the + youngest revision for which both the revs and revprops files are + present is healthy. In other words, we're willing to recover + from a missing or out-of-date db/current file, because db/current + is truly redundant -- it's basically a cache so we don't have to + find max_rev each time, albeit a cache with unusual semantics, + since it also officially defines when a revision goes live. But + if we're missing more than the cache, it's time to back out and + let the admin reconstruct things by hand: correctness at that + point may depend on external things like checking a commit email + list, looking in particular working copies, etc. + + This policy matches well with a typical naive backup scenario. + Say you're rsyncing your FSX repository nightly to the same + location. Once revs and revprops are written, you've got the + maximum rev; if the backup should bomb before db/current is + written, then db/current could stay arbitrarily out-of-date, but + we can still recover. It's a small window, but we might as well + do what we can. */ + + /* Even if db/current were missing, it would be created with 0 by + get_youngest(), so this conditional remains valid. */ + if (youngest_rev > max_rev) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Expected current rev to be <= %ld " + "but found %ld"), max_rev, youngest_rev); + + /* Before setting current, verify that there is a revprops file + for the youngest revision. (Issue #2992) */ + SVN_ERR(svn_io_check_path(svn_fs_x__path_revprops(fs, max_rev, pool), + &youngest_revprops_kind, pool)); + if (youngest_revprops_kind == svn_node_none) + { + svn_boolean_t missing = TRUE; + if (!svn_fs_x__packed_revprop_available(&missing, fs, max_rev, pool)) + { + if (missing) + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Revision %ld has a revs file but no " + "revprops file"), + max_rev); + } + else + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Revision %ld has a revs file but the " + "revprops file is inaccessible"), + max_rev); + } + } + } + else if (youngest_revprops_kind != svn_node_file) + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Revision %ld has a non-file where its " + "revprops file should be"), + max_rev); + } + + /* Prune younger-than-(newfound-youngest) revisions from the rep + cache if sharing is enabled taking care not to create the cache + if it does not exist. */ + if (ffd->rep_sharing_allowed) + { + svn_boolean_t rep_cache_exists; + + SVN_ERR(svn_fs_x__exists_rep_cache(&rep_cache_exists, fs, pool)); + if (rep_cache_exists) + SVN_ERR(svn_fs_x__del_rep_reference(fs, max_rev, pool)); + } + + /* Now store the discovered youngest revision, and the next IDs if + relevant, in a new 'current' file. */ + return svn_fs_x__write_current(fs, max_rev, pool); +} + +/* This implements the fs_library_vtable_t.recover() API. */ +svn_error_t * +svn_fs_x__recover(svn_fs_t *fs, + svn_cancel_func_t cancel_func, void *cancel_baton, + apr_pool_t *pool) +{ + struct recover_baton b; + + /* We have no way to take out an exclusive lock in FSX, so we're + restricted as to the types of recovery we can do. Luckily, + we just want to recreate the 'current' file, and we can do that just + by blocking other writers. */ + b.fs = fs; + b.cancel_func = cancel_func; + b.cancel_baton = cancel_baton; + return svn_fs_x__with_write_lock(fs, recover_body, &b, pool); +}
diff --git a/subversion/libsvn_fs_x/recovery.h b/subversion/libsvn_fs_x/recovery.h new file mode 100644 index 0000000..d4bbdb8 --- /dev/null +++ b/subversion/libsvn_fs_x/recovery.h
@@ -0,0 +1,36 @@ +/* recovery.h : interface to the FSX recovery functionality + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS__RECOVERY_H +#define SVN_LIBSVN_FS__RECOVERY_H + +#include "fs.h" + +/* Recover the fsx associated with filesystem FS. + Use optional CANCEL_FUNC/CANCEL_BATON for cancellation support. + Use POOL for temporary allocations. */ +svn_error_t *svn_fs_x__recover(svn_fs_t *fs, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool); + +#endif
diff --git a/subversion/libsvn_fs_x/rep-cache-db.sql b/subversion/libsvn_fs_x/rep-cache-db.sql new file mode 100644 index 0000000..8a95d34 --- /dev/null +++ b/subversion/libsvn_fs_x/rep-cache-db.sql
@@ -0,0 +1,67 @@ +/* rep-cache-db.sql -- schema for use in rep-caching + * This is intended for use with SQLite 3 + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +-- STMT_CREATE_SCHEMA +PRAGMA PAGE_SIZE = 4096; + +/* A table mapping representation hashes to locations in a rev file. */ +CREATE TABLE rep_cache ( + hash TEXT NOT NULL PRIMARY KEY, + revision INTEGER NOT NULL, + offset INTEGER NOT NULL, + size INTEGER NOT NULL, + expanded_size INTEGER NOT NULL + ); + +PRAGMA USER_VERSION = 1; + + +-- STMT_GET_REP +SELECT revision, offset, size, expanded_size +FROM rep_cache +WHERE hash = ?1 + +-- STMT_SET_REP +INSERT OR FAIL INTO rep_cache (hash, revision, offset, size, expanded_size) +VALUES (?1, ?2, ?3, ?4, ?5) + +-- STMT_GET_REPS_FOR_RANGE +SELECT hash, revision, offset, size, expanded_size +FROM rep_cache +WHERE revision >= ?1 AND revision <= ?2 + +-- STMT_GET_MAX_REV +SELECT MAX(revision) +FROM rep_cache + +-- STMT_DEL_REPS_YOUNGER_THAN_REV +DELETE FROM rep_cache +WHERE revision > ?1 + +/* An INSERT takes an SQLite reserved lock that prevents other writes + but doesn't block reads. The incomplete transaction means that no + permanent change is made to the database and the transaction is + removed when the database is closed. */ +-- STMT_LOCK_REP +BEGIN TRANSACTION; +INSERT INTO rep_cache VALUES ('dummy', 0, 0, 0, 0)
diff --git a/subversion/libsvn_fs_x/rep-cache.c b/subversion/libsvn_fs_x/rep-cache.c new file mode 100644 index 0000000..ba92c42 --- /dev/null +++ b/subversion/libsvn_fs_x/rep-cache.c
@@ -0,0 +1,386 @@ +/* rep-sharing.c --- the rep-sharing cache for fsx + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "svn_pools.h" + +#include "svn_private_config.h" + +#include "fs_x.h" +#include "fs.h" +#include "rep-cache.h" +#include "../libsvn_fs/fs-loader.h" + +#include "svn_path.h" + +#include "private/svn_sqlite.h" + +#include "rep-cache-db.h" + +/* A few magic values */ +#define REP_CACHE_SCHEMA_FORMAT 1 + +REP_CACHE_DB_SQL_DECLARE_STATEMENTS(statements); + + + +/** Helper functions. **/ +static APR_INLINE const char * +path_rep_cache_db(const char *fs_path, + apr_pool_t *result_pool) +{ + return svn_dirent_join(fs_path, REP_CACHE_DB_NAME, result_pool); +} + +/* Check that REP refers to a revision that exists in FS. */ +static svn_error_t * +rep_has_been_born(representation_t *rep, + svn_fs_t *fs, + apr_pool_t *pool) +{ + SVN_ERR_ASSERT(rep); + + SVN_ERR(svn_fs_x__ensure_revision_exists(rep->revision, fs, pool)); + + return SVN_NO_ERROR; +} + + + +/** Library-private API's. **/ + +/* Body of svn_fs_x__open_rep_cache(). + Implements svn_atomic__init_once().init_func. + */ +static svn_error_t * +open_rep_cache(void *baton, + apr_pool_t *pool) +{ + svn_fs_t *fs = baton; + fs_x_data_t *ffd = fs->fsap_data; + svn_sqlite__db_t *sdb; + const char *db_path; + int version; + + /* Open (or create) the sqlite database. It will be automatically + closed when fs->pool is destoyed. */ + db_path = path_rep_cache_db(fs->path, pool); + SVN_ERR(svn_sqlite__open(&sdb, db_path, + svn_sqlite__mode_rwcreate, statements, + 0, NULL, + fs->pool, pool)); + + SVN_ERR(svn_sqlite__read_schema_version(&version, sdb, pool)); + if (version < REP_CACHE_SCHEMA_FORMAT) + { + /* Must be 0 -- an uninitialized (no schema) database. Create + the schema. Results in schema version of 1. */ + SVN_ERR(svn_sqlite__exec_statements(sdb, STMT_CREATE_SCHEMA)); + } + + /* This is used as a flag that the database is available so don't + set it earlier. */ + ffd->rep_cache_db = sdb; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__open_rep_cache(svn_fs_t *fs, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_error_t *err = svn_atomic__init_once(&ffd->rep_cache_db_opened, + open_rep_cache, fs, pool); + return svn_error_quick_wrap(err, _("Couldn't open rep-cache database")); +} + +svn_error_t * +svn_fs_x__exists_rep_cache(svn_boolean_t *exists, + svn_fs_t *fs, apr_pool_t *pool) +{ + svn_node_kind_t kind; + + SVN_ERR(svn_io_check_path(path_rep_cache_db(fs->path, pool), + &kind, pool)); + + *exists = (kind != svn_node_none); + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__walk_rep_reference(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t end, + svn_error_t *(*walker)(representation_t *, + void *, + svn_fs_t *, + apr_pool_t *), + void *walker_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_sqlite__stmt_t *stmt; + svn_boolean_t have_row; + int iterations = 0; + + apr_pool_t *iterpool = svn_pool_create(pool); + + if (! ffd->rep_cache_db) + SVN_ERR(svn_fs_x__open_rep_cache(fs, pool)); + + /* Check global invariants. */ + if (start == 0) + { + svn_revnum_t max; + + SVN_ERR(svn_sqlite__get_statement(&stmt, ffd->rep_cache_db, + STMT_GET_MAX_REV)); + SVN_ERR(svn_sqlite__step(&have_row, stmt)); + max = svn_sqlite__column_revnum(stmt, 0); + SVN_ERR(svn_sqlite__reset(stmt)); + if (SVN_IS_VALID_REVNUM(max)) /* The rep-cache could be empty. */ + SVN_ERR(svn_fs_x__ensure_revision_exists(max, fs, iterpool)); + } + + SVN_ERR(svn_sqlite__get_statement(&stmt, ffd->rep_cache_db, + STMT_GET_REPS_FOR_RANGE)); + SVN_ERR(svn_sqlite__bindf(stmt, "rr", + start, end)); + + /* Walk the cache entries. */ + SVN_ERR(svn_sqlite__step(&have_row, stmt)); + while (have_row) + { + representation_t *rep; + const char *sha1_digest; + svn_error_t *err; + svn_checksum_t *checksum; + + /* Clear ITERPOOL occasionally. */ + if (iterations++ % 16 == 0) + svn_pool_clear(iterpool); + + /* Check for cancellation. */ + if (cancel_func) + { + err = cancel_func(cancel_baton); + if (err) + return svn_error_compose_create(err, svn_sqlite__reset(stmt)); + } + + /* Construct a representation_t. */ + rep = apr_pcalloc(iterpool, sizeof(*rep)); + svn_fs_x__id_txn_reset(&rep->txn_id); + sha1_digest = svn_sqlite__column_text(stmt, 0, iterpool); + err = svn_checksum_parse_hex(&checksum, svn_checksum_sha1, + sha1_digest, iterpool); + if (err) + return svn_error_compose_create(err, svn_sqlite__reset(stmt)); + + rep->has_sha1 = TRUE; + memcpy(rep->sha1_digest, checksum->digest, sizeof(rep->sha1_digest)); + rep->revision = svn_sqlite__column_revnum(stmt, 1); + rep->item_index = svn_sqlite__column_int64(stmt, 2); + rep->size = svn_sqlite__column_int64(stmt, 3); + rep->expanded_size = svn_sqlite__column_int64(stmt, 4); + + /* Walk. */ + err = walker(rep, walker_baton, fs, iterpool); + if (err) + return svn_error_compose_create(err, svn_sqlite__reset(stmt)); + + SVN_ERR(svn_sqlite__step(&have_row, stmt)); + } + + SVN_ERR(svn_sqlite__reset(stmt)); + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + + +/* This function's caller ignores most errors it returns. + If you extend this function, check the callsite to see if you have + to make it not-ignore additional error codes. */ +svn_error_t * +svn_fs_x__get_rep_reference(representation_t **rep, + svn_fs_t *fs, + svn_checksum_t *checksum, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_sqlite__stmt_t *stmt; + svn_boolean_t have_row; + + SVN_ERR_ASSERT(ffd->rep_sharing_allowed); + if (! ffd->rep_cache_db) + SVN_ERR(svn_fs_x__open_rep_cache(fs, pool)); + + /* We only allow SHA1 checksums in this table. */ + if (checksum->kind != svn_checksum_sha1) + return svn_error_create(SVN_ERR_BAD_CHECKSUM_KIND, NULL, + _("Only SHA1 checksums can be used as keys in the " + "rep_cache table.\n")); + + SVN_ERR(svn_sqlite__get_statement(&stmt, ffd->rep_cache_db, STMT_GET_REP)); + SVN_ERR(svn_sqlite__bindf(stmt, "s", + svn_checksum_to_cstring(checksum, pool))); + + SVN_ERR(svn_sqlite__step(&have_row, stmt)); + if (have_row) + { + *rep = apr_pcalloc(pool, sizeof(**rep)); + svn_fs_x__id_txn_reset(&(*rep)->txn_id); + memcpy((*rep)->sha1_digest, checksum->digest, + sizeof((*rep)->sha1_digest)); + (*rep)->has_sha1 = TRUE; + (*rep)->revision = svn_sqlite__column_revnum(stmt, 0); + (*rep)->item_index = svn_sqlite__column_int64(stmt, 1); + (*rep)->size = svn_sqlite__column_int64(stmt, 2); + (*rep)->expanded_size = svn_sqlite__column_int64(stmt, 3); + } + else + *rep = NULL; + + SVN_ERR(svn_sqlite__reset(stmt)); + + if (*rep) + SVN_ERR(rep_has_been_born(*rep, fs, pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__set_rep_reference(svn_fs_t *fs, + representation_t *rep, + svn_boolean_t reject_dup, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_sqlite__stmt_t *stmt; + svn_error_t *err; + svn_checksum_t checksum; + checksum.kind = svn_checksum_sha1; + checksum.digest = rep->sha1_digest; + + SVN_ERR_ASSERT(ffd->rep_sharing_allowed); + if (! ffd->rep_cache_db) + SVN_ERR(svn_fs_x__open_rep_cache(fs, pool)); + + /* We only allow SHA1 checksums in this table. */ + if (! rep->has_sha1) + return svn_error_create(SVN_ERR_BAD_CHECKSUM_KIND, NULL, + _("Only SHA1 checksums can be used as keys in the " + "rep_cache table.\n")); + + SVN_ERR(svn_sqlite__get_statement(&stmt, ffd->rep_cache_db, STMT_SET_REP)); + SVN_ERR(svn_sqlite__bindf(stmt, "siiii", + svn_checksum_to_cstring(&checksum, pool), + (apr_int64_t) rep->revision, + (apr_int64_t) rep->item_index, + (apr_int64_t) rep->size, + (apr_int64_t) rep->expanded_size)); + + err = svn_sqlite__insert(NULL, stmt); + if (err) + { + representation_t *old_rep; + + if (err->apr_err != SVN_ERR_SQLITE_CONSTRAINT) + return svn_error_trace(err); + + svn_error_clear(err); + + /* Constraint failed so the mapping for SHA1_CHECKSUM->REP + should exist. If so, and the value is the same one we were + about to write, that's cool -- just do nothing. If, however, + the value is *different*, that's a red flag! */ + SVN_ERR(svn_fs_x__get_rep_reference(&old_rep, fs, &checksum, pool)); + + if (old_rep) + { + if (reject_dup && ((old_rep->revision != rep->revision) + || (old_rep->item_index != rep->item_index) + || (old_rep->size != rep->size) + || (old_rep->expanded_size != rep->expanded_size))) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + apr_psprintf(pool, + _("Representation key for checksum '%%s' exists " + "in filesystem '%%s' with a different value " + "(%%ld,%%%s,%%%s,%%%s) than what we were about " + "to store (%%ld,%%%s,%%%s,%%%s)"), + APR_OFF_T_FMT, SVN_FILESIZE_T_FMT, + SVN_FILESIZE_T_FMT, APR_OFF_T_FMT, + SVN_FILESIZE_T_FMT, SVN_FILESIZE_T_FMT), + svn_checksum_to_cstring_display(&checksum, pool), + fs->path, old_rep->revision, old_rep->item_index, + old_rep->size, old_rep->expanded_size, rep->revision, + rep->item_index, rep->size, rep->expanded_size); + else + return SVN_NO_ERROR; + } + else + { + /* Something really odd at this point, we failed to insert the + checksum AND failed to read an existing checksum. Do we need + to flag this? */ + } + } + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__del_rep_reference(svn_fs_t *fs, + svn_revnum_t youngest, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_sqlite__stmt_t *stmt; + + if (! ffd->rep_cache_db) + SVN_ERR(svn_fs_x__open_rep_cache(fs, pool)); + + SVN_ERR(svn_sqlite__get_statement(&stmt, ffd->rep_cache_db, + STMT_DEL_REPS_YOUNGER_THAN_REV)); + SVN_ERR(svn_sqlite__bindf(stmt, "r", youngest)); + SVN_ERR(svn_sqlite__step_done(stmt)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__lock_rep_cache(svn_fs_t *fs, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + + if (! ffd->rep_cache_db) + SVN_ERR(svn_fs_x__open_rep_cache(fs, pool)); + + SVN_ERR(svn_sqlite__exec_statements(ffd->rep_cache_db, STMT_LOCK_REP)); + + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_fs_x/rep-cache.h b/subversion/libsvn_fs_x/rep-cache.h new file mode 100644 index 0000000..66a4345 --- /dev/null +++ b/subversion/libsvn_fs_x/rep-cache.h
@@ -0,0 +1,102 @@ +/* rep-cache.h : interface to rep cache db functions + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS_X_REP_CACHE_H +#define SVN_LIBSVN_FS_X_REP_CACHE_H + +#include "svn_error.h" + +#include "fs.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + + +#define REP_CACHE_DB_NAME "rep-cache.db" + +/* Open and create, if needed, the rep cache database associated with FS. + Use POOL for temporary allocations. */ +svn_error_t * +svn_fs_x__open_rep_cache(svn_fs_t *fs, + apr_pool_t *pool); + +/* Set *EXISTS to TRUE iff the rep-cache DB file exists. */ +svn_error_t * +svn_fs_x__exists_rep_cache(svn_boolean_t *exists, + svn_fs_t *fs, + apr_pool_t *pool); + +/* Iterate all representations currently in FS's cache. */ +svn_error_t * +svn_fs_x__walk_rep_reference(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t end, + svn_error_t *(*walker)(representation_t *rep, + void *walker_baton, + svn_fs_t *fs, + apr_pool_t *scratch_pool), + void *walker_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool); + +/* Return the representation REP in FS which has fulltext CHECKSUM. + REP is allocated in POOL. If the rep cache database has not been + opened, just set *REP to NULL. */ +svn_error_t * +svn_fs_x__get_rep_reference(representation_t **rep, + svn_fs_t *fs, + svn_checksum_t *checksum, + apr_pool_t *pool); + +/* Set the representation REP in FS, using REP->CHECKSUM. + Use POOL for temporary allocations. + + If the rep cache database has not been opened, this may be a no op. + + If REJECT_DUP is TRUE, return an error if there is an existing + match for REP->CHECKSUM. */ +svn_error_t * +svn_fs_x__set_rep_reference(svn_fs_t *fs, + representation_t *rep, + svn_boolean_t reject_dup, + apr_pool_t *pool); + +/* Delete from the cache all reps corresponding to revisions younger + than YOUNGEST. */ +svn_error_t * +svn_fs_x__del_rep_reference(svn_fs_t *fs, + svn_revnum_t youngest, + apr_pool_t *pool); + +/* Start a transaction to take an SQLite reserved lock that prevents + other writes. */ +svn_error_t * +svn_fs_x__lock_rep_cache(svn_fs_t *fs, + apr_pool_t *pool); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* SVN_LIBSVN_FS_X_REP_CACHE_H */
diff --git a/subversion/libsvn_fs_x/reps.c b/subversion/libsvn_fs_x/reps.c new file mode 100644 index 0000000..12a9203 --- /dev/null +++ b/subversion/libsvn_fs_x/reps.c
@@ -0,0 +1,916 @@ +/* reps.c --- FSX representation container + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "reps.h" + +#include "svn_sorts.h" +#include "private/svn_string_private.h" +#include "private/svn_packed_data.h" +#include "private/svn_temp_serializer.h" + +#include "cached_data.h" + +/* Length of the text chunks we hash and match. The algorithm will find + * most matches with a length of 2 * MATCH_BLOCKSIZE and only specific + * ones that are shorter than MATCH_BLOCKSIZE. + * + * This should be a power of two and must be a multiple of 8. + * Good choices are 32, 64 and 128. + */ +#define MATCH_BLOCKSIZE 64 + +/* value of unused hash buckets */ +#define NO_OFFSET ((apr_uint32_t)(-1)) + +/* Byte strings are described by a series of copy instructions that each + * do one of the following + * + * - copy a given number of bytes from the text corpus starting at a + * given offset + * - reference other instruction and specify how many of instructions of + * that sequence shall be executed (i.e. a sub-sequence) + * - copy a number of bytes from the base representation buffer starting + * at a given offset + */ + +/* The contents of a fulltext / representation is defined by its first + * instruction and the number of instructions to execute. + */ +typedef struct rep_t +{ + apr_uint32_t first_instruction; + apr_uint32_t instruction_count; +} rep_t; + +/* A single instruction. The instruction type is being encoded in OFFSET. + */ +typedef struct instruction_t +{ + /* Instruction type and offset. + * - offset < 0 + * reference to instruction sub-sequence starting with + * container->instructions[-offset]. + * - 0 <= offset < container->base_text_len + * reference to the base text corpus; + * start copy at offset + * - offset >= container->base_text_len + * reference to the text corpus; + * start copy at offset-container->base_text_len + */ + apr_int32_t offset; + + /* Number of bytes to copy / instructions to execute + */ + apr_uint32_t count; +} instruction_t; + +/* Describe a base fulltext. + */ +typedef struct base_t +{ + /* Revision */ + svn_revnum_t revision; + + /* Item within that revision */ + apr_uint64_t item_index; + + /* Priority with which to use this base over others */ + int priority; + + /* Index into builder->representations that identifies the copy + * instructions for this base. */ + apr_uint32_t rep; +} base_t; + +/* Yet another hash data structure. This one tries to be more cache + * friendly by putting the first byte of each hashed sequence in a + * common array. This array will often fit into L1 or L2 at least and + * give a 99% accurate test for a match without giving false negatives. + */ +typedef struct hash_t +{ + /* for used entries i, prefixes[i] == text[offsets[i]]; 0 otherwise. + * This allows for a quick check without resolving the double + * indirection. */ + char *prefixes; + + /* for used entries i, offsets[i] is start offset in the text corpus; + * NO_OFFSET otherwise. + */ + apr_uint32_t *offsets; + + /* to be used later for optimizations. */ + apr_uint32_t *last_matches; + + /* number of buckets in this hash, i.e. elements in each array above. + * Must be 1 << (8 * sizeof(hash_key_t) - shift) */ + apr_size_t size; + + /* number of buckets actually in use. Must be <= size. */ + apr_size_t used; + + /* number of bits to shift right to map a hash_key_t to a bucket index */ + apr_size_t shift; + + /* pool to use when growing the hash */ + apr_pool_t *pool; +} hash_t; + +/* Hash key type. 32 bits for pseudo-Adler32 hash sums. + */ +typedef apr_uint32_t hash_key_t; + +/* Constructor data structure. + */ +struct svn_fs_x__reps_builder_t +{ + /* file system to read base representations from */ + svn_fs_t *fs; + + /* text corpus */ + svn_stringbuf_t *text; + + /* text block hash */ + hash_t hash; + + /* array of base_t objects describing all bases defined so far */ + apr_array_header_t *bases; + + /* array of rep_t objects describing all fulltexts (including bases) + * added so far */ + apr_array_header_t *reps; + + /* array of instruction_t objects describing all instructions */ + apr_array_header_t *instructions; + + /* number of bytes in the text corpus that belongs to bases */ + apr_size_t base_text_len; +}; + +/* R/o container. + */ +struct svn_fs_x__reps_t +{ + /* text corpus */ + const char *text; + + /* length of the text corpus in bytes */ + apr_size_t text_len; + + /* bases used */ + const base_t *bases; + + /* number of bases used */ + apr_size_t base_count; + + /* fulltext i can be reconstructed by executing instructions + * first_instructions[i] .. first_instructions[i+1]-1 + * (this array has one extra element at the end) + */ + const apr_uint32_t *first_instructions; + + /* number of fulltexts (no bases) */ + apr_size_t rep_count; + + /* instructions */ + const instruction_t *instructions; + + /* total number of instructions */ + apr_size_t instruction_count; + + /* offsets > 0 but smaller that this are considered base references */ + apr_size_t base_text_len; +}; + +/* describe a section in the extractor's result string that is not filled + * yet (but already exists). + */ +typedef struct missing_t +{ + /* start offset within the result string */ + apr_uint32_t start; + + /* number of bytes to write */ + apr_uint32_t count; + + /* index into extractor->bases selecting the base representation to + * copy from */ + apr_uint32_t base; + + /* copy source offset within that base representation */ + apr_uint32_t offset; +} missing_t; + +/* Fulltext extractor data structure. + */ +struct svn_fs_x__rep_extractor_t +{ + /* filesystem to read the bases from */ + svn_fs_t *fs; + + /* fulltext being constructed */ + svn_stringbuf_t *result; + + /* bases (base_t) yet to process (not used ATM) */ + apr_array_header_t *bases; + + /* missing sections (missing_t) in result->data that need to be filled, + * yet */ + apr_array_header_t *missing; + + /* pool to use for allocating the above arrays */ + apr_pool_t *pool; +}; + +/* Given the ADLER32 checksum for a certain range of MATCH_BLOCKSIZE + * bytes, return the checksum for the range excluding the first byte + * C_OUT and appending C_IN. + */ +static hash_key_t +hash_key_replace(hash_key_t adler32, const char c_out, const char c_in) +{ + adler32 -= (MATCH_BLOCKSIZE * 0x10000u * ((unsigned char) c_out)); + + adler32 -= (unsigned char)c_out; + adler32 += (unsigned char)c_in; + + return adler32 + adler32 * 0x10000; +} + +/* Calculate an pseudo-adler32 checksum for MATCH_BLOCKSIZE bytes starting + at DATA. Return the checksum value. */ +static hash_key_t +hash_key(const char *data) +{ + const unsigned char *input = (const unsigned char *)data; + const unsigned char *last = input + MATCH_BLOCKSIZE; + + hash_key_t s1 = 0; + hash_key_t s2 = 0; + + for (; input < last; input += 8) + { + s1 += input[0]; s2 += s1; + s1 += input[1]; s2 += s1; + s1 += input[2]; s2 += s1; + s1 += input[3]; s2 += s1; + s1 += input[4]; s2 += s1; + s1 += input[5]; s2 += s1; + s1 += input[6]; s2 += s1; + s1 += input[7]; s2 += s1; + } + + return s2 * 0x10000 + s1; +} + +/* Map the ADLER32 key to a bucket index in HASH and return that index. + */ +static apr_size_t +hash_to_index(hash_t *hash, hash_key_t adler32) +{ + return (adler32 * 0xd1f3da69) >> hash->shift; +} + +/* Allocate and initialized SIZE buckets in POOL. Assign them to HASH. + */ +static void +allocate_hash_members(hash_t *hash, + apr_size_t size, + apr_pool_t *pool) +{ + apr_size_t i; + + hash->pool = pool; + hash->size = size; + + hash->prefixes = apr_pcalloc(pool, size); + hash->last_matches = apr_pcalloc(pool, sizeof(*hash->last_matches) * size); + hash->offsets = apr_palloc(pool, sizeof(*hash->offsets) * size); + + for (i = 0; i < size; ++i) + hash->offsets[i] = NO_OFFSET; +} + +/* Initialize the HASH data structure with 2**TWOPOWER buckets allocated + * in POOL. + */ +static void +init_hash(hash_t *hash, + apr_size_t twoPower, + apr_pool_t *pool) +{ + hash->used = 0; + hash->shift = sizeof(hash_key_t) * 8 - twoPower; + + allocate_hash_members(hash, 1 << twoPower, pool); +} + +/* Make HASH have at least MIN_SIZE buckets but at least double the number + * of buckets in HASH by rehashing it based TEXT. + */ +static void +grow_hash(hash_t *hash, + svn_stringbuf_t *text, + apr_size_t min_size) +{ + hash_t copy; + apr_size_t i; + + /* determine the new hash size */ + apr_size_t new_size = hash->size * 2; + apr_size_t new_shift = hash->shift - 1; + while (new_size < min_size) + { + new_size *= 2; + --new_shift; + } + + /* allocate new hash */ + allocate_hash_members(©, new_size, hash->pool); + copy.used = 0; + copy.shift = new_shift; + + /* copy / translate data */ + for (i = 0; i < hash->size; ++i) + { + apr_uint32_t offset = hash->offsets[i]; + if (offset != NO_OFFSET) + { + hash_key_t key = hash_key(text->data + offset); + size_t idx = hash_to_index(©, key); + + if (copy.offsets[idx] == NO_OFFSET) + copy.used++; + + copy.prefixes[idx] = hash->prefixes[i]; + copy.offsets[idx] = offset; + copy.last_matches[idx] = hash->last_matches[i]; + } + } + + *hash = copy; +} + +svn_fs_x__reps_builder_t * +svn_fs_x__reps_builder_create(svn_fs_t *fs, + apr_pool_t *pool) +{ + svn_fs_x__reps_builder_t *result = apr_pcalloc(pool, sizeof(*result)); + + result->fs = fs; + result->text = svn_stringbuf_create_empty(pool); + init_hash(&result->hash, 4, pool); + + result->bases = apr_array_make(pool, 0, sizeof(base_t)); + result->reps = apr_array_make(pool, 0, sizeof(rep_t)); + result->instructions = apr_array_make(pool, 0, sizeof(instruction_t)); + + return result; +} + +svn_error_t * +svn_fs_x__reps_add_base(svn_fs_x__reps_builder_t *builder, + representation_t *rep, + int priority, + apr_pool_t *scratch_pool) +{ + base_t base; + apr_size_t text_start_offset = builder->text->len; + + svn_stream_t *stream; + svn_string_t *contents; + SVN_ERR(svn_fs_x__get_contents(&stream, builder->fs, rep, scratch_pool)); + SVN_ERR(svn_string_from_stream(&contents, stream, scratch_pool, + scratch_pool)); + + base.revision = rep->revision; + base.item_index = rep->item_index; + base.priority = priority; + base.rep = (apr_uint32_t)svn_fs_x__reps_add(builder, contents); + + APR_ARRAY_PUSH(builder->bases, base_t) = base; + builder->base_text_len += builder->text->len - text_start_offset; + + return SVN_NO_ERROR; +} + +/* Add LEN bytes from DATA to BUILDER's text corpus. Also, add a copy + * operation for that text fragment. + */ +static void +add_new_text(svn_fs_x__reps_builder_t *builder, + const char *data, + apr_size_t len) +{ + instruction_t instruction; + apr_size_t offset; + apr_size_t buckets_required; + + if (len == 0) + return; + + /* new instruction */ + instruction.offset = builder->text->len; + instruction.count = len; + APR_ARRAY_PUSH(builder->instructions, instruction_t) = instruction; + + /* add to text corpus */ + svn_stringbuf_appendbytes(builder->text, data, len); + + /* expand the hash upfront to minimize the chances of collisions */ + buckets_required = builder->hash.used + len / MATCH_BLOCKSIZE; + if (buckets_required * 3 >= builder->hash.size * 2) + grow_hash(&builder->hash, builder->text, 2 * buckets_required); + + /* add hash entries for the new sequence */ + for (offset = instruction.offset; + offset + MATCH_BLOCKSIZE <= builder->text->len; + offset += MATCH_BLOCKSIZE) + { + hash_key_t key = hash_key(builder->text->data + offset); + size_t idx = hash_to_index(&builder->hash, key); + + /* Don't replace hash entries that stem from the current text. + * This makes early matches more likely. */ + if (builder->hash.offsets[idx] == NO_OFFSET) + ++builder->hash.used; + else if (builder->hash.offsets[idx] >= instruction.offset) + continue; + + builder->hash.offsets[idx] = offset; + builder->hash.prefixes[idx] = builder->text->data[offset]; + } +} + +apr_size_t +svn_fs_x__reps_add(svn_fs_x__reps_builder_t *builder, + const svn_string_t *contents) +{ + rep_t rep; + const char *current = contents->data; + const char *processed = current; + const char *end = current + contents->len; + const char *last_to_test = end - MATCH_BLOCKSIZE - 1; + + rep.first_instruction = (apr_uint32_t)builder->instructions->nelts; + + while (current < last_to_test) + { + hash_key_t key = hash_key(current); + size_t offset; + size_t idx; + + /* search for the next matching sequence */ + + for (; current < last_to_test; ++current) + { + idx = hash_to_index(&builder->hash, key); + if (builder->hash.prefixes[idx] == current[0]) + { + offset = builder->hash.offsets[idx]; + if ( (offset != NO_OFFSET) + && (memcmp(&builder->text->data[offset], current, + MATCH_BLOCKSIZE) == 0)) + break; + } + key = hash_key_replace(key, current[0], current[MATCH_BLOCKSIZE]); + } + + /* found it? */ + + if (current < last_to_test) + { + instruction_t instruction; + + /* extend the match */ + + size_t prefix_match + = svn_cstring__reverse_match_length(current, + builder->text->data + offset, + MIN(offset, current - processed)); + size_t postfix_match + = svn_cstring__match_length(current + MATCH_BLOCKSIZE, + builder->text->data + offset + MATCH_BLOCKSIZE, + MIN(builder->text->len - offset - MATCH_BLOCKSIZE, + end - current - MATCH_BLOCKSIZE)); + + /* non-matched section */ + + size_t new_copy = (current - processed) - prefix_match; + if (new_copy) + add_new_text(builder, processed, new_copy); + + /* add instruction for matching section */ + + instruction.offset = offset - prefix_match; + instruction.count = prefix_match + postfix_match + MATCH_BLOCKSIZE; + APR_ARRAY_PUSH(builder->instructions, instruction_t) = instruction; + + processed = current + MATCH_BLOCKSIZE + postfix_match; + current = processed; + } + } + + add_new_text(builder, processed, end - processed); + rep.instruction_count = (apr_uint32_t)builder->instructions->nelts + - rep.first_instruction; + APR_ARRAY_PUSH(builder->reps, rep_t) = rep; + + return (apr_size_t)(builder->reps->nelts - 1); +} + +apr_size_t +svn_fs_x__reps_estimate_size(const svn_fs_x__reps_builder_t *builder) +{ + /* approx: size of the text exclusive to us @ 50% compression rate + * + 2 bytes per instruction + * + 2 bytes per representation + * + 8 bytes per base representation + * + 1:8 inefficiency in using the base representations + * + 100 bytes static overhead + */ + return (builder->text->len - builder->base_text_len) / 2 + + builder->instructions->nelts * 2 + + builder->reps->nelts * 2 + + builder->bases->nelts * 8 + + builder->base_text_len / 8 + + 100; +} + +/* Execute COUNT instructions starting at INSTRUCTION_IDX in CONTAINER + * and fill the parts of EXTRACTOR->RESULT that we can from this container. + * Record the remainder in EXTRACTOR->MISSING. + * + * This function will recurse for instructions that reference other + * instruction sequences. COUNT refers to the top-level instructions only. + */ +static void +get_text(svn_fs_x__rep_extractor_t *extractor, + const svn_fs_x__reps_t *container, + apr_size_t instruction_idx, + apr_size_t count) +{ + const instruction_t *instruction; + const char *offset_0 = container->text - container->base_text_len; + + for (instruction = container->instructions + instruction_idx; + instruction < container->instructions + instruction_idx + count; + instruction++) + if (instruction->offset < 0) + { + /* instruction sub-sequence */ + get_text(extractor, container, -instruction->offset, + instruction->count); + } + else if (instruction->offset >= container->base_text_len) + { + /* direct copy instruction */ + svn_stringbuf_appendbytes(extractor->result, + offset_0 + instruction->offset, + instruction->count); + } + else + { + /* a section that we need to fill from some external base rep. */ + missing_t missing; + missing.base = 0; + missing.start = extractor->result->len; + missing.count = instruction->count; + missing.offset = instruction->offset; + svn_stringbuf_appendfill(extractor->result, 0, instruction->count); + + if (extractor->missing == NULL) + extractor->missing = apr_array_make(extractor->pool, 1, + sizeof(missing)); + + APR_ARRAY_PUSH(extractor->missing, missing_t) = missing; + } +} + +svn_error_t * +svn_fs_x__reps_get(svn_fs_x__rep_extractor_t **extractor, + svn_fs_t *fs, + const svn_fs_x__reps_t *container, + apr_size_t idx, + apr_pool_t *pool) +{ + apr_uint32_t first = container->first_instructions[idx]; + apr_uint32_t last = container->first_instructions[idx + 1]; + + /* create the extractor object */ + svn_fs_x__rep_extractor_t *result = apr_pcalloc(pool, sizeof(*result)); + result->fs = fs; + result->result = svn_stringbuf_create_empty(pool); + result->pool = pool; + + /* fill all the bits of the result that we can, i.e. all but bits comming + * from base representations */ + get_text(result, container, first, last - first); + *extractor = result; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__extractor_drive(svn_stringbuf_t **contents, + svn_fs_x__rep_extractor_t *extractor, + apr_size_t start_offset, + apr_size_t size, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + /* we don't support base reps right now */ + SVN_ERR_ASSERT(extractor->missing == NULL); + + if (size == 0) + { + *contents = svn_stringbuf_dup(extractor->result, result_pool); + } + else + { + /* clip the selected range */ + if (start_offset > extractor->result->len) + start_offset = extractor->result->len; + + if (size > extractor->result->len - start_offset) + size = extractor->result->len - start_offset; + + *contents = svn_stringbuf_ncreate(extractor->result->data + start_offset, + size, result_pool); + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__write_reps_container(svn_stream_t *stream, + const svn_fs_x__reps_builder_t *builder, + apr_pool_t *pool) +{ + int i; + svn_packed__data_root_t *root = svn_packed__data_create_root(pool); + + /* one top-level stream for each array */ + svn_packed__int_stream_t *bases_stream + = svn_packed__create_int_stream(root, FALSE, FALSE); + svn_packed__int_stream_t *reps_stream + = svn_packed__create_int_stream(root, TRUE, FALSE); + svn_packed__int_stream_t *instructions_stream + = svn_packed__create_int_stream(root, FALSE, FALSE); + + /* for misc stuff */ + svn_packed__int_stream_t *misc_stream + = svn_packed__create_int_stream(root, FALSE, FALSE); + + /* TEXT will be just a single string */ + svn_packed__byte_stream_t *text_stream + = svn_packed__create_bytes_stream(root); + + /* structure the struct streams such we can extract much of the redundancy + */ + svn_packed__create_int_substream(bases_stream, TRUE, TRUE); + svn_packed__create_int_substream(bases_stream, TRUE, FALSE); + svn_packed__create_int_substream(bases_stream, TRUE, FALSE); + svn_packed__create_int_substream(bases_stream, TRUE, FALSE); + + svn_packed__create_int_substream(instructions_stream, TRUE, TRUE); + svn_packed__create_int_substream(instructions_stream, FALSE, FALSE); + + /* text */ + svn_packed__add_bytes(text_stream, builder->text->data, builder->text->len); + + /* serialize bases */ + for (i = 0; i < builder->bases->nelts; ++i) + { + const base_t *base = &APR_ARRAY_IDX(builder->bases, i, base_t); + svn_packed__add_int(bases_stream, base->revision); + svn_packed__add_uint(bases_stream, base->item_index); + svn_packed__add_uint(bases_stream, base->priority); + svn_packed__add_uint(bases_stream, base->rep); + } + + /* serialize reps */ + for (i = 0; i < builder->reps->nelts; ++i) + { + const rep_t *rep = &APR_ARRAY_IDX(builder->reps, i, rep_t); + svn_packed__add_uint(reps_stream, rep->first_instruction); + } + + svn_packed__add_uint(reps_stream, builder->instructions->nelts); + + /* serialize instructions */ + for (i = 0; i < builder->instructions->nelts; ++i) + { + const instruction_t *instruction + = &APR_ARRAY_IDX(builder->instructions, i, instruction_t); + svn_packed__add_int(instructions_stream, instruction->offset); + svn_packed__add_uint(instructions_stream, instruction->count); + } + + /* other elements */ + svn_packed__add_uint(misc_stream, 0); + + /* write to stream */ + SVN_ERR(svn_packed__data_write(stream, root, pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__read_reps_container(svn_fs_x__reps_t **container, + svn_stream_t *stream, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + apr_size_t i; + + base_t *bases; + apr_uint32_t *first_instructions; + instruction_t *instructions; + + svn_fs_x__reps_t *reps = apr_pcalloc(result_pool, sizeof(*reps)); + + svn_packed__data_root_t *root; + svn_packed__int_stream_t *bases_stream; + svn_packed__int_stream_t *reps_stream; + svn_packed__int_stream_t *instructions_stream; + svn_packed__int_stream_t *misc_stream; + svn_packed__byte_stream_t *text_stream; + + /* read from disk */ + SVN_ERR(svn_packed__data_read(&root, stream, result_pool, scratch_pool)); + + bases_stream = svn_packed__first_int_stream(root); + reps_stream = svn_packed__next_int_stream(bases_stream); + instructions_stream = svn_packed__next_int_stream(reps_stream); + misc_stream = svn_packed__next_int_stream(instructions_stream); + text_stream = svn_packed__first_byte_stream(root); + + /* text */ + reps->text = svn_packed__get_bytes(text_stream, &reps->text_len); + reps->text = apr_pmemdup(result_pool, reps->text, reps->text_len); + + /* de-serialize bases */ + reps->base_count + = svn_packed__int_count(svn_packed__first_int_substream(bases_stream)); + bases = apr_palloc(result_pool, reps->base_count * sizeof(*bases)); + reps->bases = bases; + + for (i = 0; i < reps->base_count; ++i) + { + base_t *base = bases + i; + base->revision = svn_packed__get_int(bases_stream); + base->item_index = svn_packed__get_uint(bases_stream); + base->priority = svn_packed__get_uint(bases_stream); + base->rep = svn_packed__get_uint(bases_stream); + } + + /* de-serialize instructions */ + reps->instruction_count + = svn_packed__int_count + (svn_packed__first_int_substream(instructions_stream)); + instructions + = apr_palloc(result_pool, + reps->instruction_count * sizeof(*instructions)); + reps->instructions = instructions; + + for (i = 0; i < reps->instruction_count; ++i) + { + instruction_t *instruction = instructions + i; + instruction->offset = svn_packed__get_int(instructions_stream); + instruction->count = svn_packed__get_uint(instructions_stream); + } + + /* de-serialize reps */ + reps->rep_count = svn_packed__int_count(reps_stream); + first_instructions + = apr_palloc(result_pool, + (reps->rep_count + 1) * sizeof(*first_instructions)); + reps->first_instructions = first_instructions; + + for (i = 0; i < reps->rep_count; ++i) + first_instructions[i] + = (apr_uint32_t)svn_packed__get_uint(reps_stream); + first_instructions[reps->rep_count] = reps->instruction_count; + + /* other elements */ + reps->base_text_len = svn_packed__get_uint(misc_stream); + + /* return result */ + *container = reps; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__serialize_reps_container(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + svn_fs_x__reps_t *reps = in; + svn_stringbuf_t *serialized; + + /* make a guesstimate on the size of the serialized data. Erring on the + * low side will cause the serializer to re-alloc its buffer. */ + apr_size_t size + = reps->text_len + + reps->base_count * sizeof(*reps->bases) + + reps->rep_count * sizeof(*reps->first_instructions) + + reps->instruction_count * sizeof(*reps->instructions) + + 100; + + /* serialize array header and all its elements */ + svn_temp_serializer__context_t *context + = svn_temp_serializer__init(reps, sizeof(*reps), size, pool); + + /* serialize sub-structures */ + svn_temp_serializer__add_leaf(context, (const void **)&reps->text, + reps->text_len); + svn_temp_serializer__add_leaf(context, (const void **)&reps->bases, + reps->base_count * sizeof(*reps->bases)); + svn_temp_serializer__add_leaf(context, + (const void **)&reps->first_instructions, + reps->rep_count * + sizeof(*reps->first_instructions)); + svn_temp_serializer__add_leaf(context, (const void **)&reps->instructions, + reps->instruction_count * + sizeof(*reps->instructions)); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_reps_container(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + svn_fs_x__reps_t *reps = (svn_fs_x__reps_t *)data; + + /* de-serialize sub-structures */ + svn_temp_deserializer__resolve(reps, (void **)&reps->text); + svn_temp_deserializer__resolve(reps, (void **)&reps->bases); + svn_temp_deserializer__resolve(reps, (void **)&reps->first_instructions); + svn_temp_deserializer__resolve(reps, (void **)&reps->instructions); + + /* done */ + *out = reps; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__reps_get_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool) +{ + svn_fs_x__reps_baton_t *reps_baton = baton; + + /* get a usable reps structure */ + const svn_fs_x__reps_t *cached = data; + svn_fs_x__reps_t *reps = apr_pmemdup(pool, cached, sizeof(*reps)); + + reps->text + = svn_temp_deserializer__ptr(cached, (const void **)&cached->text); + reps->bases + = svn_temp_deserializer__ptr(cached, (const void **)&cached->bases); + reps->first_instructions + = svn_temp_deserializer__ptr(cached, + (const void **)&cached->first_instructions); + reps->instructions + = svn_temp_deserializer__ptr(cached, + (const void **)&cached->instructions); + + /* return an extractor for the selected item */ + SVN_ERR(svn_fs_x__reps_get((svn_fs_x__rep_extractor_t **)out, + reps_baton->fs, reps, reps_baton->idx, pool)); + + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_fs_x/reps.h b/subversion/libsvn_fs_x/reps.h new file mode 100644 index 0000000..234783a --- /dev/null +++ b/subversion/libsvn_fs_x/reps.h
@@ -0,0 +1,189 @@ +/* reps.h --- FSX representation container + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS__REPS_H +#define SVN_LIBSVN_FS__REPS_H + +#include "svn_io.h" +#include "fs.h" + +/* This container type implements the start-delta (aka pick lists) data + * structure plus functions to create it and read data from it. The key + * point is to identify common sub-strings within a whole set of fulltexts + * instead of only two as in the classic txdelta code. + * + * Because it is relatively expensive to optimize the final in-memory + * layout, representation containers cannot be updated. A builder object + * will do most of the space saving when adding fulltexts but the final + * data will only be created immediately before serializing everything to + * disk. So, builders are write only and representation containers are + * read-only. + * + * Extracting data from a representation container is O(length) but it + * may require multiple iterations if base representations outside the + * container were used. Therefore, you will first create an extractor + * object (this may happen while holding a cache lock) and the you need + * to "drive" the extractor outside any cache context. + */ + +/* A write-only constructor object for representation containers. + */ +typedef struct svn_fs_x__reps_builder_t svn_fs_x__reps_builder_t; + +/* A read-only representation container - + * an opaque collection of fulltexts, i.e. byte strings. + */ +typedef struct svn_fs_x__reps_t svn_fs_x__reps_t; + +/* The fulltext extractor utility object. + */ +typedef struct svn_fs_x__rep_extractor_t svn_fs_x__rep_extractor_t; + +/* Baton type to be passed to svn_fs_x__reps_get_func. + */ +typedef struct svn_fs_x__reps_baton_t +{ + /* filesystem the resulting extractor shall operate on */ + svn_fs_t *fs; + + /* element index of the item to extract from the container */ + apr_size_t idx; +} svn_fs_x__reps_baton_t; + +/* Create and populate noderev containers. */ + +/* Create and return a new builder object, allocated in POOL. + */ +svn_fs_x__reps_builder_t * +svn_fs_x__reps_builder_create(svn_fs_t *fs, + apr_pool_t *pool); + +/* To BUILDER, add reference to the fulltext currently stored in + * representation REP. Substrings matching with any of the base reps + * in BUILDER can be removed from the text base and be replaced by + * references to those base representations. + * + * The PRIORITY is a mere hint on which base representations should + * preferred in case we could re-use the same contents from multiple bases. + * Higher numerical value means higher priority / likelihood of being + * selected over others. + * + * Use SCRATCH_POOL for temporary allocations. + */ +svn_error_t * +svn_fs_x__reps_add_base(svn_fs_x__reps_builder_t *builder, + representation_t *rep, + int priority, + apr_pool_t *scratch_pool); + +/* Add the byte string CONTENTS to BUILDER. Return the item index under + * which the fulltext can be retrieved from the final container. + */ +apr_size_t +svn_fs_x__reps_add(svn_fs_x__reps_builder_t *builder, + const svn_string_t *contents); + +/* Return a rough estimate in bytes for the serialized representation + * of BUILDER. + */ +apr_size_t +svn_fs_x__reps_estimate_size(const svn_fs_x__reps_builder_t *builder); + +/* Read from representation containers. */ + +/* For fulltext IDX in CONTAINER in filesystem FS, create an extract object + * allocated in POOL and return it in *EXTRACTOR. + */ +svn_error_t * +svn_fs_x__reps_get(svn_fs_x__rep_extractor_t **extractor, + svn_fs_t *fs, + const svn_fs_x__reps_t *container, + apr_size_t idx, + apr_pool_t *pool); + +/* Let the EXTRACTOR object fetch all parts of the desired fulltext and + * return the latter in *CONTENTS. If SIZE is not 0, return SIZE bytes + * starting at offset START_OFFSET of the full contents. If that range + * lies partly or completely outside the content, clip it accordingly. + * Allocate the result in RESULT_POOL and use SCRATCH_POOL for temporary + * allocations. + * + * Note, you may not run this inside a cache access function. + */ +svn_error_t * +svn_fs_x__extractor_drive(svn_stringbuf_t** contents, + svn_fs_x__rep_extractor_t* extractor, + apr_size_t start_offset, + apr_size_t size, + apr_pool_t* result_pool, + apr_pool_t* scratch_pool); + +/* I/O interface. */ + +/* Write a serialized representation of the final container described by + * BUILDER to STREAM. Use POOL for temporary allocations. + */ +svn_error_t * +svn_fs_x__write_reps_container(svn_stream_t *stream, + const svn_fs_x__reps_builder_t *builder, + apr_pool_t *pool); + +/* Read a representations container from its serialized representation in + * STREAM. Allocate the result in RESULT_POOL and return it in *CONTAINER. + * Use SCRATCH_POOL for temporary allocations. + */ +svn_error_t * +svn_fs_x__read_reps_container(svn_fs_x__reps_t **container, + svn_stream_t *stream, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + +/* Implements #svn_cache__serialize_func_t for svn_fs_x__reps_t objects. + */ +svn_error_t * +svn_fs_x__serialize_reps_container(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/* Implements #svn_cache__deserialize_func_t for svn_fs_x__reps_t objects. + */ +svn_error_t * +svn_fs_x__deserialize_reps_container(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/* Implements svn_cache__partial_getter_func_t for svn_fs_x__reps_t, + * setting *OUT to an svn_fs_x__rep_extractor_t object defined by the + * svn_fs_x__reps_baton_t passed in as *BATON. This function is similar + * to svn_fs_x__reps_get but operates on the cache serialized + * representation of the container. + */ +svn_error_t * +svn_fs_x__reps_get_func(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool); + +#endif \ No newline at end of file
diff --git a/subversion/libsvn_fs_x/revprops.c b/subversion/libsvn_fs_x/revprops.c new file mode 100644 index 0000000..a47c748 --- /dev/null +++ b/subversion/libsvn_fs_x/revprops.c
@@ -0,0 +1,1738 @@ +/* revprops.c --- everything needed to handle revprops in FSX + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include <assert.h> + +#include "svn_pools.h" +#include "svn_hash.h" +#include "svn_dirent_uri.h" + +#include "fs_x.h" +#include "revprops.h" +#include "util.h" +#include "transaction.h" + +#include "private/svn_subr_private.h" +#include "private/svn_string_private.h" +#include "../libsvn_fs/fs-loader.h" + +#include "svn_private_config.h" + +/* Give writing processes 10 seconds to replace an existing revprop + file with a new one. After that time, we assume that the writing + process got aborted and that we have re-read revprops. */ +#define REVPROP_CHANGE_TIMEOUT (10 * 1000000) + +/* The following are names of atomics that will be used to communicate + * revprop updates across all processes on this machine. */ +#define ATOMIC_REVPROP_GENERATION "rev-prop-generation" +#define ATOMIC_REVPROP_TIMEOUT "rev-prop-timeout" +#define ATOMIC_REVPROP_NAMESPACE "rev-prop-atomics" + +svn_error_t * +svn_fs_x__upgrade_pack_revprops(svn_fs_t *fs, + svn_fs_upgrade_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + const char *revprops_shard_path; + const char *revprops_pack_file_dir; + apr_int64_t shard; + apr_int64_t first_unpacked_shard + = ffd->min_unpacked_rev / ffd->max_files_per_dir; + + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, + scratch_pool); + int compression_level = ffd->compress_packed_revprops + ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT + : SVN_DELTA_COMPRESSION_LEVEL_NONE; + + /* first, pack all revprops shards to match the packed revision shards */ + for (shard = 0; shard < first_unpacked_shard; ++shard) + { + revprops_pack_file_dir = svn_dirent_join(revsprops_dir, + apr_psprintf(iterpool, + "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, + shard), + iterpool); + revprops_shard_path = svn_dirent_join(revsprops_dir, + apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), + iterpool); + + SVN_ERR(svn_fs_x__pack_revprops_shard(revprops_pack_file_dir, + revprops_shard_path, + shard, ffd->max_files_per_dir, + (int)(0.9 * ffd->revprop_pack_size), + compression_level, + cancel_func, cancel_baton, iterpool)); + if (notify_func) + SVN_ERR(notify_func(notify_baton, shard, + svn_fs_upgrade_pack_revprops, iterpool)); + + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__upgrade_cleanup_pack_revprops(svn_fs_t *fs, + svn_fs_upgrade_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + const char *revprops_shard_path; + apr_int64_t shard; + apr_int64_t first_unpacked_shard + = ffd->min_unpacked_rev / ffd->max_files_per_dir; + + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, + scratch_pool); + + /* delete the non-packed revprops shards afterwards */ + for (shard = 0; shard < first_unpacked_shard; ++shard) + { + revprops_shard_path = svn_dirent_join(revsprops_dir, + apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), + iterpool); + SVN_ERR(svn_fs_x__delete_revprops_shard(revprops_shard_path, + shard, ffd->max_files_per_dir, + cancel_func, cancel_baton, + iterpool)); + if (notify_func) + SVN_ERR(notify_func(notify_baton, shard, + svn_fs_upgrade_cleanup_revprops, iterpool)); + + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Revprop caching management. + * + * Mechanism: + * ---------- + * + * Revprop caching needs to be activated and will be deactivated for the + * respective FS instance if the necessary infrastructure could not be + * initialized. In deactivated mode, there is almost no runtime overhead + * associated with revprop caching. As long as no revprops are being read + * or changed, revprop caching imposes no overhead. + * + * When activated, we cache revprops using (revision, generation) pairs + * as keys with the generation being incremented upon every revprop change. + * Since the cache is process-local, the generation needs to be tracked + * for at least as long as the process lives but may be reset afterwards. + * + * To track the revprop generation, we use two-layer approach. On the lower + * level, we use named atomics to have a system-wide consistent value for + * the current revprop generation. However, those named atomics will only + * remain valid for as long as at least one process / thread in the system + * accesses revprops in the respective repository. The underlying shared + * memory gets cleaned up afterwards. + * + * On the second level, we will use a persistent file to track the latest + * revprop generation. It will be written upon each revprop change but + * only be read if we are the first process to initialize the named atomics + * with that value. + * + * The overhead for the second and following accesses to revprops is + * almost zero on most systems. + * + * + * Tech aspects: + * ------------- + * + * A problem is that we need to provide a globally available file name to + * back the SHM implementation on OSes that need it. We can only assume + * write access to some file within the respective repositories. Because + * a given server process may access thousands of repositories during its + * lifetime, keeping the SHM data alive for all of them is also not an + * option. + * + * So, we store the new revprop generation on disk as part of each + * setrevprop call, i.e. this write will be serialized and the write order + * be guaranteed by the repository write lock. + * + * The only racy situation occurs when the data is being read again by two + * processes concurrently but in that situation, the first process to + * finish that procedure is guaranteed to be the only one that initializes + * the SHM data. Since even writers will first go through that + * initialization phase, they will never operate on stale data. + */ + +/* Read revprop generation as stored on disk for repository FS. The result + * is returned in *CURRENT. Default to 2 if no such file is available. + */ +static svn_error_t * +read_revprop_generation_file(apr_int64_t *current, + svn_fs_t *fs, + apr_pool_t *pool) +{ + svn_error_t *err; + apr_file_t *file; + char buf[80]; + apr_size_t len; + const char *path = svn_fs_x__path_revprop_generation(fs, pool); + + err = svn_io_file_open(&file, path, + APR_READ | APR_BUFFERED, + APR_OS_DEFAULT, pool); + if (err && APR_STATUS_IS_ENOENT(err->apr_err)) + { + svn_error_clear(err); + *current = 2; + + return SVN_NO_ERROR; + } + SVN_ERR(err); + + len = sizeof(buf); + SVN_ERR(svn_io_read_length_line(file, buf, &len, pool)); + + /* Check that the first line contains only digits. */ + SVN_ERR(svn_fs_x__check_file_buffer_numeric(buf, 0, path, + "Revprop Generation", pool)); + SVN_ERR(svn_cstring_atoi64(current, buf)); + + return svn_io_file_close(file, pool); +} + +/* Write the CURRENT revprop generation to disk for repository FS. + */ +svn_error_t * +svn_fs_x__write_revprop_generation_file(svn_fs_t *fs, + apr_int64_t current, + apr_pool_t *pool) +{ + char buf[SVN_INT64_BUFFER_SIZE]; + apr_size_t len = svn__i64toa(buf, current); + buf[len] = '\n'; + + SVN_ERR(svn_io_write_atomic(svn_fs_x__path_revprop_generation(fs, pool), + buf, len + 1, + NULL /* copy_perms */, pool)); + + return SVN_NO_ERROR; +} + +/* Make sure the revprop_namespace member in FS is set. */ +static svn_error_t * +ensure_revprop_namespace(svn_fs_t *fs) +{ + fs_x_data_t *ffd = fs->fsap_data; + + return ffd->revprop_namespace == NULL + ? svn_atomic_namespace__create(&ffd->revprop_namespace, + svn_dirent_join(fs->path, + ATOMIC_REVPROP_NAMESPACE, + fs->pool), + fs->pool) + : SVN_NO_ERROR; +} + +/* Make sure the revprop_namespace member in FS is set. */ +svn_error_t * +svn_fs_x__cleanup_revprop_namespace(svn_fs_t *fs) +{ + const char *name = svn_dirent_join(fs->path, + ATOMIC_REVPROP_NAMESPACE, + fs->pool); + return svn_error_trace(svn_atomic_namespace__cleanup(name, fs->pool)); +} + +/* Make sure the revprop_generation member in FS is set and, if necessary, + * initialized with the latest value stored on disk. + */ +static svn_error_t * +ensure_revprop_generation(svn_fs_t *fs, apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + + SVN_ERR(ensure_revprop_namespace(fs)); + if (ffd->revprop_generation == NULL) + { + apr_int64_t current; + + SVN_ERR(svn_named_atomic__get(&ffd->revprop_generation, + ffd->revprop_namespace, + ATOMIC_REVPROP_GENERATION, + TRUE)); + + /* If the generation is at 0, we just created a new namespace + * (it would be at least 2 otherwise). Read the latest generation + * from disk and if we are the first one to initialize the atomic + * (i.e. is still 0), set it to the value just gotten. + */ + SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation)); + if (current == 0) + { + SVN_ERR(read_revprop_generation_file(¤t, fs, pool)); + SVN_ERR(svn_named_atomic__cmpxchg(NULL, current, 0, + ffd->revprop_generation)); + } + } + + return SVN_NO_ERROR; +} + +/* Make sure the revprop_timeout member in FS is set. */ +static svn_error_t * +ensure_revprop_timeout(svn_fs_t *fs) +{ + fs_x_data_t *ffd = fs->fsap_data; + + SVN_ERR(ensure_revprop_namespace(fs)); + return ffd->revprop_timeout == NULL + ? svn_named_atomic__get(&ffd->revprop_timeout, + ffd->revprop_namespace, + ATOMIC_REVPROP_TIMEOUT, + TRUE) + : SVN_NO_ERROR; +} + +/* Create an error object with the given MESSAGE and pass it to the + WARNING member of FS. Clears UNDERLYING_ERR. */ +static void +log_revprop_cache_init_warning(svn_fs_t *fs, + svn_error_t *underlying_err, + const char *message, + apr_pool_t *pool) +{ + svn_error_t *err = svn_error_createf( + SVN_ERR_FS_REVPROP_CACHE_INIT_FAILURE, + underlying_err, message, + svn_dirent_local_style(fs->path, pool)); + + if (fs->warning) + (fs->warning)(fs->warning_baton, err); + + svn_error_clear(err); +} + +/* Test whether revprop cache and necessary infrastructure are + available in FS. */ +static svn_boolean_t +has_revprop_cache(svn_fs_t *fs, apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_error_t *error; + + /* is the cache (still) enabled? */ + if (ffd->revprop_cache == NULL) + return FALSE; + + /* is it efficient? */ + if (!svn_named_atomic__is_efficient()) + { + /* access to it would be quite slow + * -> disable the revprop cache for good + */ + ffd->revprop_cache = NULL; + log_revprop_cache_init_warning(fs, NULL, + "Revprop caching for '%s' disabled" + " because it would be inefficient.", + pool); + + return FALSE; + } + + /* try to access our SHM-backed infrastructure */ + error = ensure_revprop_generation(fs, pool); + if (error) + { + /* failure -> disable revprop cache for good */ + + ffd->revprop_cache = NULL; + log_revprop_cache_init_warning(fs, error, + "Revprop caching for '%s' disabled " + "because SHM infrastructure for revprop " + "caching failed to initialize.", + pool); + + return FALSE; + } + + return TRUE; +} + +/* Baton structure for revprop_generation_fixup. */ +typedef struct revprop_generation_fixup_t +{ + /* revprop generation to read */ + apr_int64_t *generation; + + /* containing the revprop_generation member to query */ + fs_x_data_t *ffd; +} revprop_generation_upgrade_t; + +/* If the revprop generation has an odd value, it means the original writer + of the revprop got killed. We don't know whether that process as able + to change the revprop data but we assume that it was. Therefore, we + increase the generation in that case to basically invalidate everyone's + cache content. + Execute this only while holding the write lock to the repo in baton->FFD. + */ +static svn_error_t * +revprop_generation_fixup(void *void_baton, + apr_pool_t *pool) +{ + revprop_generation_upgrade_t *baton = void_baton; + assert(baton->ffd->has_write_lock); + + /* Maybe, either the original revprop writer or some other reader has + already corrected / bumped the revprop generation. Thus, we need + to read it again. */ + SVN_ERR(svn_named_atomic__read(baton->generation, + baton->ffd->revprop_generation)); + + /* Cause everyone to re-read revprops upon their next access, if the + last revprop write did not complete properly. */ + while (*baton->generation % 2) + SVN_ERR(svn_named_atomic__add(baton->generation, + 1, + baton->ffd->revprop_generation)); + + return SVN_NO_ERROR; +} + +/* Read the current revprop generation and return it in *GENERATION. + Also, detect aborted / crashed writers and recover from that. + Use the access object in FS to set the shared mem values. */ +static svn_error_t * +read_revprop_generation(apr_int64_t *generation, + svn_fs_t *fs, + apr_pool_t *pool) +{ + apr_int64_t current = 0; + fs_x_data_t *ffd = fs->fsap_data; + + /* read the current revprop generation number */ + SVN_ERR(ensure_revprop_generation(fs, pool)); + SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation)); + + /* is an unfinished revprop write under the way? */ + if (current % 2) + { + apr_int64_t timeout = 0; + + /* read timeout for the write operation */ + SVN_ERR(ensure_revprop_timeout(fs)); + SVN_ERR(svn_named_atomic__read(&timeout, ffd->revprop_timeout)); + + /* has the writer process been aborted, + * i.e. has the timeout been reached? + */ + if (apr_time_now() > timeout) + { + revprop_generation_upgrade_t baton; + baton.generation = ¤t; + baton.ffd = ffd; + + /* Ensure that the original writer process no longer exists by + * acquiring the write lock to this repository. Then, fix up + * the revprop generation. + */ + if (ffd->has_write_lock) + SVN_ERR(revprop_generation_fixup(&baton, pool)); + else + SVN_ERR(svn_fs_x__with_write_lock(fs, revprop_generation_fixup, + &baton, pool)); + } + } + + /* return the value we just got */ + *generation = current; + return SVN_NO_ERROR; +} + +/* Set the revprop generation to the next odd number to indicate that + there is a revprop write process under way. If that times out, + readers shall recover from that state & re-read revprops. + Use the access object in FS to set the shared mem value. */ +static svn_error_t * +begin_revprop_change(svn_fs_t *fs, apr_pool_t *pool) +{ + apr_int64_t current; + fs_x_data_t *ffd = fs->fsap_data; + + /* set the timeout for the write operation */ + SVN_ERR(ensure_revprop_timeout(fs)); + SVN_ERR(svn_named_atomic__write(NULL, + apr_time_now() + REVPROP_CHANGE_TIMEOUT, + ffd->revprop_timeout)); + + /* set the revprop generation to an odd value to indicate + * that a write is in progress + */ + SVN_ERR(ensure_revprop_generation(fs, pool)); + do + { + SVN_ERR(svn_named_atomic__add(¤t, + 1, + ffd->revprop_generation)); + } + while (current % 2 == 0); + + return SVN_NO_ERROR; +} + +/* Set the revprop generation to the next even number to indicate that + a) readers shall re-read revprops, and + b) the write process has been completed (no recovery required) + Use the access object in FS to set the shared mem value. */ +static svn_error_t * +end_revprop_change(svn_fs_t *fs, apr_pool_t *pool) +{ + apr_int64_t current = 1; + fs_x_data_t *ffd = fs->fsap_data; + + /* set the revprop generation to an even value to indicate + * that a write has been completed + */ + SVN_ERR(ensure_revprop_generation(fs, pool)); + do + { + SVN_ERR(svn_named_atomic__add(¤t, + 1, + ffd->revprop_generation)); + } + while (current % 2); + + /* Save the latest generation to disk. FS is currently in a "locked" + * state such that we can be sure the be the only ones to write that + * file. + */ + return svn_fs_x__write_revprop_generation_file(fs, current, pool); +} + +/* Container for all data required to access the packed revprop file + * for a given REVISION. This structure will be filled incrementally + * by read_pack_revprops() its sub-routines. + */ +typedef struct packed_revprops_t +{ + /* revision number to read (not necessarily the first in the pack) */ + svn_revnum_t revision; + + /* current revprop generation. Used when populating the revprop cache */ + apr_int64_t generation; + + /* the actual revision properties */ + apr_hash_t *properties; + + /* their size when serialized to a single string + * (as found in PACKED_REVPROPS) */ + apr_size_t serialized_size; + + + /* name of the pack file (without folder path) */ + const char *filename; + + /* packed shard folder path */ + const char *folder; + + /* sum of values in SIZES */ + apr_size_t total_size; + + /* first revision in the pack */ + svn_revnum_t start_revision; + + /* size of the revprops in PACKED_REVPROPS */ + apr_array_header_t *sizes; + + /* offset of the revprops in PACKED_REVPROPS */ + apr_array_header_t *offsets; + + + /* concatenation of the serialized representation of all revprops + * in the pack, i.e. the pack content without header and compression */ + svn_stringbuf_t *packed_revprops; + + /* content of the manifest. + * Maps long(rev - START_REVISION) to const char* pack file name */ + apr_array_header_t *manifest; +} packed_revprops_t; + +/* Parse the serialized revprops in CONTENT and return them in *PROPERTIES. + * Also, put them into the revprop cache, if activated, for future use. + * Three more parameters are being used to update the revprop cache: FS is + * our file system, the revprops belong to REVISION and the global revprop + * GENERATION is used as well. + * + * The returned hash will be allocated in POOL, SCRATCH_POOL is being used + * for temporary allocations. + */ +static svn_error_t * +parse_revprop(apr_hash_t **properties, + svn_fs_t *fs, + svn_revnum_t revision, + apr_int64_t generation, + svn_string_t *content, + apr_pool_t *pool, + apr_pool_t *scratch_pool) +{ + svn_stream_t *stream = svn_stream_from_string(content, scratch_pool); + *properties = apr_hash_make(pool); + + SVN_ERR(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool)); + if (has_revprop_cache(fs, pool)) + { + fs_x_data_t *ffd = fs->fsap_data; + pair_cache_key_t key = { 0 }; + + key.revision = revision; + key.second = generation; + SVN_ERR(svn_cache__set(ffd->revprop_cache, &key, *properties, + scratch_pool)); + } + + return SVN_NO_ERROR; +} + +/* Read the non-packed revprops for revision REV in FS, put them into the + * revprop cache if activated and return them in *PROPERTIES. GENERATION + * is the current revprop generation. + * + * If the data could not be read due to an otherwise recoverable error, + * leave *PROPERTIES unchanged. No error will be returned in that case. + * + * Allocations will be done in POOL. + */ +static svn_error_t * +read_non_packed_revprop(apr_hash_t **properties, + svn_fs_t *fs, + svn_revnum_t rev, + apr_int64_t generation, + apr_pool_t *pool) +{ + svn_stringbuf_t *content = NULL; + apr_pool_t *iterpool = svn_pool_create(pool); + svn_boolean_t missing = FALSE; + int i; + + for (i = 0; i < RECOVERABLE_RETRY_COUNT && !missing && !content; ++i) + { + svn_pool_clear(iterpool); + SVN_ERR(svn_fs_x__try_stringbuf_from_file(&content, + &missing, + svn_fs_x__path_revprops(fs, rev, iterpool), + i + 1 < RECOVERABLE_RETRY_COUNT, + iterpool)); + } + + if (content) + SVN_ERR(parse_revprop(properties, fs, rev, generation, + svn_stringbuf__morph_into_string(content), + pool, iterpool)); + + svn_pool_clear(iterpool); + + return SVN_NO_ERROR; +} + +/* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST + * members. Use POOL for allocating results and SCRATCH_POOL for temporaries. + */ +static svn_error_t * +get_revprop_packname(svn_fs_t *fs, + packed_revprops_t *revprops, + apr_pool_t *pool, + apr_pool_t *scratch_pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_stringbuf_t *content = NULL; + const char *manifest_file_path; + int idx; + + /* read content of the manifest file */ + revprops->folder + = svn_fs_x__path_revprops_pack_shard(fs, revprops->revision, pool); + manifest_file_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); + + SVN_ERR(svn_fs_x__read_content(&content, manifest_file_path, pool)); + + /* parse the manifest. Every line is a file name */ + revprops->manifest = apr_array_make(pool, ffd->max_files_per_dir, + sizeof(const char*)); + while (content->data) + { + APR_ARRAY_PUSH(revprops->manifest, const char*) = content->data; + content->data = strchr(content->data, '\n'); + if (content->data) + { + *content->data = 0; + content->data++; + } + } + + /* Index for our revision. Rev 0 is excluded from the first shard. */ + idx = (int)(revprops->revision % ffd->max_files_per_dir); + if (revprops->revision < ffd->max_files_per_dir) + --idx; + + if (revprops->manifest->nelts <= idx) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Packed revprop manifest for r%ld too " + "small"), revprops->revision); + + /* Now get the file name */ + revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*); + + return SVN_NO_ERROR; +} + +/* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS, + * fill the START_REVISION, SIZES, OFFSETS members. Also, make + * PACKED_REVPROPS point to the first serialized revprop. + * + * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as + * well as the SERIALIZED_SIZE member. If revprop caching has been + * enabled, parse all revprops in the pack and cache them. + */ +static svn_error_t * +parse_packed_revprops(svn_fs_t *fs, + packed_revprops_t *revprops, + apr_pool_t *pool, + apr_pool_t *scratch_pool) +{ + svn_stream_t *stream; + apr_int64_t first_rev, count, i; + apr_off_t offset; + const char *header_end; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + + /* decompress (even if the data is only "stored", there is still a + * length header to remove) */ + svn_stringbuf_t *compressed = revprops->packed_revprops; + svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(pool); + SVN_ERR(svn__decompress(compressed, uncompressed, APR_SIZE_MAX)); + + /* read first revision number and number of revisions in the pack */ + stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); + SVN_ERR(svn_fs_x__read_number_from_stream(&first_rev, NULL, stream, + iterpool)); + SVN_ERR(svn_fs_x__read_number_from_stream(&count, NULL, stream, iterpool)); + + /* make PACKED_REVPROPS point to the first char after the header. + * This is where the serialized revprops are. */ + header_end = strstr(uncompressed->data, "\n\n"); + if (header_end == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Header end not found")); + + offset = header_end - uncompressed->data + 2; + + revprops->packed_revprops = svn_stringbuf_create_empty(pool); + revprops->packed_revprops->data = uncompressed->data + offset; + revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset); + revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset); + + /* STREAM still points to the first entry in the sizes list. + * Init / construct REVPROPS members. */ + revprops->start_revision = (svn_revnum_t)first_rev; + revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset)); + revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset)); + + /* Now parse, revision by revision, the size and content of each + * revisions' revprops. */ + for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i) + { + apr_int64_t size; + svn_string_t serialized; + apr_hash_t *properties; + svn_revnum_t revision = (svn_revnum_t)(first_rev + i); + + /* read & check the serialized size */ + SVN_ERR(svn_fs_x__read_number_from_stream(&size, NULL, stream, + iterpool)); + if (size + offset > (apr_int64_t)revprops->packed_revprops->len) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Packed revprop size exceeds pack file size")); + + /* Parse this revprops list, if necessary */ + serialized.data = revprops->packed_revprops->data + offset; + serialized.len = (apr_size_t)size; + + if (revision == revprops->revision) + { + SVN_ERR(parse_revprop(&revprops->properties, fs, revision, + revprops->generation, &serialized, + pool, iterpool)); + revprops->serialized_size = serialized.len; + } + else + { + /* If revprop caching is enabled, parse any revprops. + * They will get cached as a side-effect of this. */ + if (has_revprop_cache(fs, pool)) + SVN_ERR(parse_revprop(&properties, fs, revision, + revprops->generation, &serialized, + iterpool, iterpool)); + } + + /* fill REVPROPS data structures */ + APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len; + APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset; + revprops->total_size += serialized.len; + + offset += serialized.len; + + svn_pool_clear(iterpool); + } + + return SVN_NO_ERROR; +} + +/* In filesystem FS, read the packed revprops for revision REV into + * *REVPROPS. Use GENERATION to populate the revprop cache, if enabled. + * Allocate data in POOL. + */ +static svn_error_t * +read_pack_revprop(packed_revprops_t **revprops, + svn_fs_t *fs, + svn_revnum_t rev, + apr_int64_t generation, + apr_pool_t *pool) +{ + apr_pool_t *iterpool = svn_pool_create(pool); + svn_boolean_t missing = FALSE; + svn_error_t *err; + packed_revprops_t *result; + int i; + + /* someone insisted that REV is packed. Double-check if necessary */ + if (!svn_fs_x__is_packed_revprop(fs, rev)) + SVN_ERR(svn_fs_x__update_min_unpacked_rev(fs, iterpool)); + + if (!svn_fs_x__is_packed_revprop(fs, rev)) + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("No such packed revision %ld"), rev); + + /* initialize the result data structure */ + result = apr_pcalloc(pool, sizeof(*result)); + result->revision = rev; + result->generation = generation; + + /* try to read the packed revprops. This may require retries if we have + * concurrent writers. */ + for (i = 0; i < RECOVERABLE_RETRY_COUNT && !result->packed_revprops; ++i) + { + const char *file_path; + + /* there might have been concurrent writes. + * Re-read the manifest and the pack file. + */ + SVN_ERR(get_revprop_packname(fs, result, pool, iterpool)); + file_path = svn_dirent_join(result->folder, + result->filename, + iterpool); + SVN_ERR(svn_fs_x__try_stringbuf_from_file(&result->packed_revprops, + &missing, + file_path, + i + 1 < RECOVERABLE_RETRY_COUNT, + pool)); + + /* If we could not find the file, there was a write. + * So, we should refresh our revprop generation info as well such + * that others may find data we will put into the cache. They would + * consider it outdated, otherwise. + */ + if (missing && has_revprop_cache(fs, pool)) + SVN_ERR(read_revprop_generation(&result->generation, fs, pool)); + + svn_pool_clear(iterpool); + } + + /* the file content should be available now */ + if (!result->packed_revprops) + return svn_error_createf(SVN_ERR_FS_PACKED_REVPROP_READ_FAILURE, NULL, + _("Failed to read revprop pack file for r%ld"), rev); + + /* parse it. RESULT will be complete afterwards. */ + err = parse_packed_revprops(fs, result, pool, iterpool); + svn_pool_destroy(iterpool); + if (err) + return svn_error_createf(SVN_ERR_FS_CORRUPT, err, + _("Revprop pack file for r%ld is corrupt"), rev); + + *revprops = result; + + return SVN_NO_ERROR; +} + +/* Read the revprops for revision REV in FS and return them in *PROPERTIES_P. + * + * Allocations will be done in POOL. + */ +svn_error_t * +svn_fs_x__get_revision_proplist(apr_hash_t **proplist_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + apr_int64_t generation = 0; + + /* not found, yet */ + *proplist_p = NULL; + + /* should they be available at all? */ + SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, pool)); + + /* Try cache lookup first. */ + if (has_revprop_cache(fs, pool)) + { + svn_boolean_t is_cached; + pair_cache_key_t key = { 0 }; + + SVN_ERR(read_revprop_generation(&generation, fs, pool)); + + key.revision = rev; + key.second = generation; + SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached, + ffd->revprop_cache, &key, pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + /* if REV had not been packed when we began, try reading it from the + * non-packed shard. If that fails, we will fall through to packed + * shard reads. */ + if (!svn_fs_x__is_packed_revprop(fs, rev)) + { + svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev, + generation, pool); + if (err) + { + if (!APR_STATUS_IS_ENOENT(err->apr_err)) + return svn_error_trace(err); + + svn_error_clear(err); + *proplist_p = NULL; /* in case read_non_packed_revprop changed it */ + } + } + + /* if revprop packing is available and we have not read the revprops, yet, + * try reading them from a packed shard. If that fails, REV is most + * likely invalid (or its revprops highly contested). */ + if (!*proplist_p) + { + packed_revprops_t *packed_revprops; + SVN_ERR(read_pack_revprop(&packed_revprops, fs, rev, generation, pool)); + *proplist_p = packed_revprops->properties; + } + + /* The revprops should have been there. Did we get them? */ + if (!*proplist_p) + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("Could not read revprops for revision %ld"), + rev); + + return SVN_NO_ERROR; +} + +/* Serialize the revision property list PROPLIST of revision REV in + * filesystem FS to a non-packed file. Return the name of that temporary + * file in *TMP_PATH and the file path that it must be moved to in + * *FINAL_PATH. + * + * Use POOL for allocations. + */ +static svn_error_t * +write_non_packed_revprop(const char **final_path, + const char **tmp_path, + svn_fs_t *fs, + svn_revnum_t rev, + apr_hash_t *proplist, + apr_pool_t *pool) +{ + svn_stream_t *stream; + *final_path = svn_fs_x__path_revprops(fs, rev, pool); + + /* ### do we have a directory sitting around already? we really shouldn't + ### have to get the dirname here. */ + SVN_ERR(svn_stream_open_unique(&stream, tmp_path, + svn_dirent_dirname(*final_path, pool), + svn_io_file_del_none, pool, pool)); + SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_stream_close(stream)); + + return SVN_NO_ERROR; +} + +/* After writing the new revprop file(s), call this function to move the + * file at TMP_PATH to FINAL_PATH and give it the permissions from + * PERMS_REFERENCE. + * + * If indicated in BUMP_GENERATION, increase FS' revprop generation. + * Finally, delete all the temporary files given in FILES_TO_DELETE. + * The latter may be NULL. + * + * Use POOL for temporary allocations. + */ +static svn_error_t * +switch_to_new_revprop(svn_fs_t *fs, + const char *final_path, + const char *tmp_path, + const char *perms_reference, + apr_array_header_t *files_to_delete, + svn_boolean_t bump_generation, + apr_pool_t *pool) +{ + /* Now, we may actually be replacing revprops. Make sure that all other + threads and processes will know about this. */ + if (bump_generation) + SVN_ERR(begin_revprop_change(fs, pool)); + + SVN_ERR(svn_fs_x__move_into_place(tmp_path, final_path, perms_reference, + pool)); + + /* Indicate that the update (if relevant) has been completed. */ + if (bump_generation) + SVN_ERR(end_revprop_change(fs, pool)); + + /* Clean up temporary files, if necessary. */ + if (files_to_delete) + { + apr_pool_t *iterpool = svn_pool_create(pool); + int i; + + for (i = 0; i < files_to_delete->nelts; ++i) + { + const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*); + SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + } + return SVN_NO_ERROR; +} + +/* Write a pack file header to STREAM that starts at revision START_REVISION + * and contains the indexes [START,END) of SIZES. + */ +static svn_error_t * +serialize_revprops_header(svn_stream_t *stream, + svn_revnum_t start_revision, + apr_array_header_t *sizes, + int start, + int end, + apr_pool_t *pool) +{ + apr_pool_t *iterpool = svn_pool_create(pool); + int i; + + SVN_ERR_ASSERT(start < end); + + /* start revision and entry count */ + SVN_ERR(svn_stream_printf(stream, pool, "%ld\n", start_revision)); + SVN_ERR(svn_stream_printf(stream, pool, "%d\n", end - start)); + + /* the sizes array */ + for (i = start; i < end; ++i) + { + apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t); + SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n", + size)); + } + + /* the double newline char indicates the end of the header */ + SVN_ERR(svn_stream_printf(stream, iterpool, "\n")); + + svn_pool_clear(iterpool); + return SVN_NO_ERROR; +} + +/* Writes the a pack file to FILE_STREAM. It copies the serialized data + * from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX. + * + * The data for the latter is taken from NEW_SERIALIZED. Note, that + * CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is + * taken in that case but only a subset of the old data will be copied. + * + * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size. + * POOL is used for temporary allocations. + */ +static svn_error_t * +repack_revprops(svn_fs_t *fs, + packed_revprops_t *revprops, + int start, + int end, + int changed_index, + svn_stringbuf_t *new_serialized, + apr_off_t new_total_size, + svn_stream_t *file_stream, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_stream_t *stream; + int i; + + /* create data empty buffers and the stream object */ + svn_stringbuf_t *uncompressed + = svn_stringbuf_create_ensure((apr_size_t)new_total_size, pool); + svn_stringbuf_t *compressed + = svn_stringbuf_create_empty(pool); + stream = svn_stream_from_stringbuf(uncompressed, pool); + + /* write the header*/ + SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start, + revprops->sizes, start, end, pool)); + + /* append the serialized revprops */ + for (i = start; i < end; ++i) + if (i == changed_index) + { + SVN_ERR(svn_stream_write(stream, + new_serialized->data, + &new_serialized->len)); + } + else + { + apr_size_t size + = (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t); + apr_size_t offset + = (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t); + + SVN_ERR(svn_stream_write(stream, + revprops->packed_revprops->data + offset, + &size)); + } + + /* flush the stream buffer (if any) to our underlying data buffer */ + SVN_ERR(svn_stream_close(stream)); + + /* compress / store the data */ + SVN_ERR(svn__compress(uncompressed, + compressed, + ffd->compress_packed_revprops + ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT + : SVN_DELTA_COMPRESSION_LEVEL_NONE)); + + /* finally, write the content to the target stream and close it */ + SVN_ERR(svn_stream_write(file_stream, compressed->data, &compressed->len)); + SVN_ERR(svn_stream_close(file_stream)); + + return SVN_NO_ERROR; +} + +/* Allocate a new pack file name for the revisions at index [START,END) + * of REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE, + * auto-create that array if necessary. Return an open file stream to + * the new file in *STREAM allocated in POOL. + */ +static svn_error_t * +repack_stream_open(svn_stream_t **stream, + svn_fs_t *fs, + packed_revprops_t *revprops, + int start, + int end, + apr_array_header_t **files_to_delete, + apr_pool_t *pool) +{ + apr_int64_t tag; + const char *tag_string; + svn_string_t *new_filename; + int i; + apr_file_t *file; + + /* get the old (= current) file name and enlist it for later deletion */ + const char *old_filename + = APR_ARRAY_IDX(revprops->manifest, start, const char*); + + if (*files_to_delete == NULL) + *files_to_delete = apr_array_make(pool, 3, sizeof(const char*)); + + APR_ARRAY_PUSH(*files_to_delete, const char*) + = svn_dirent_join(revprops->folder, old_filename, pool); + + /* increase the tag part, i.e. the counter after the dot */ + tag_string = strchr(old_filename, '.'); + if (tag_string == NULL) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Packed file '%s' misses a tag"), + old_filename); + + SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1)); + new_filename = svn_string_createf(pool, "%ld.%" APR_INT64_T_FMT, + revprops->start_revision + start, + ++tag); + + /* update the manifest to point to the new file */ + for (i = start; i < end; ++i) + APR_ARRAY_IDX(revprops->manifest, i, const char*) = new_filename->data; + + /* create a file stream for the new file */ + SVN_ERR(svn_io_file_open(&file, svn_dirent_join(revprops->folder, + new_filename->data, + pool), + APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool)); + *stream = svn_stream_from_aprfile2(file, FALSE, pool); + + return SVN_NO_ERROR; +} + +/* For revision REV in filesystem FS, set the revision properties to + * PROPLIST. Return a new file in *TMP_PATH that the caller shall move + * to *FINAL_PATH to make the change visible. Files to be deleted will + * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated. + * Use POOL for allocations. + */ +static svn_error_t * +write_packed_revprop(const char **final_path, + const char **tmp_path, + apr_array_header_t **files_to_delete, + svn_fs_t *fs, + svn_revnum_t rev, + apr_hash_t *proplist, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + packed_revprops_t *revprops; + apr_int64_t generation = 0; + svn_stream_t *stream; + svn_stringbuf_t *serialized; + apr_off_t new_total_size; + int changed_index; + + /* read the current revprop generation. This value will not change + * while we hold the global write lock to this FS. */ + if (has_revprop_cache(fs, pool)) + SVN_ERR(read_revprop_generation(&generation, fs, pool)); + + /* read contents of the current pack file */ + SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, pool)); + + /* serialize the new revprops */ + serialized = svn_stringbuf_create_empty(pool); + stream = svn_stream_from_stringbuf(serialized, pool); + SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_stream_close(stream)); + + /* calculate the size of the new data */ + changed_index = (int)(rev - revprops->start_revision); + new_total_size = revprops->total_size - revprops->serialized_size + + serialized->len + + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE; + + APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len; + + /* can we put the new data into the same pack as the before? */ + if ( new_total_size < ffd->revprop_pack_size + || revprops->sizes->nelts == 1) + { + /* simply replace the old pack file with new content as we do it + * in the non-packed case */ + + *final_path = svn_dirent_join(revprops->folder, revprops->filename, + pool); + SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder, + svn_io_file_del_none, pool, pool)); + SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts, + changed_index, serialized, new_total_size, + stream, pool)); + } + else + { + /* split the pack file into two of roughly equal size */ + int right_count, left_count, i; + + int left = 0; + int right = revprops->sizes->nelts - 1; + apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE; + apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE; + + /* let left and right side grow such that their size difference + * is minimal after each step. */ + while (left <= right) + if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) + < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)) + { + left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) + + SVN_INT64_BUFFER_SIZE; + ++left; + } + else + { + right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t) + + SVN_INT64_BUFFER_SIZE; + --right; + } + + /* since the items need much less than SVN_INT64_BUFFER_SIZE + * bytes to represent their length, the split may not be optimal */ + left_count = left; + right_count = revprops->sizes->nelts - left; + + /* if new_size is large, one side may exceed the pack size limit. + * In that case, split before and after the modified revprop.*/ + if ( left_size > ffd->revprop_pack_size + || right_size > ffd->revprop_pack_size) + { + left_count = changed_index; + right_count = revprops->sizes->nelts - left_count - 1; + } + + /* write the new, split files */ + if (left_count) + { + SVN_ERR(repack_stream_open(&stream, fs, revprops, 0, + left_count, files_to_delete, pool)); + SVN_ERR(repack_revprops(fs, revprops, 0, left_count, + changed_index, serialized, new_total_size, + stream, pool)); + } + + if (left_count + right_count < revprops->sizes->nelts) + { + SVN_ERR(repack_stream_open(&stream, fs, revprops, changed_index, + changed_index + 1, files_to_delete, + pool)); + SVN_ERR(repack_revprops(fs, revprops, changed_index, + changed_index + 1, + changed_index, serialized, new_total_size, + stream, pool)); + } + + if (right_count) + { + SVN_ERR(repack_stream_open(&stream, fs, revprops, + revprops->sizes->nelts - right_count, + revprops->sizes->nelts, + files_to_delete, pool)); + SVN_ERR(repack_revprops(fs, revprops, + revprops->sizes->nelts - right_count, + revprops->sizes->nelts, changed_index, + serialized, new_total_size, stream, + pool)); + } + + /* write the new manifest */ + *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); + SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder, + svn_io_file_del_none, pool, pool)); + + for (i = 0; i < revprops->manifest->nelts; ++i) + { + const char *filename = APR_ARRAY_IDX(revprops->manifest, i, + const char*); + SVN_ERR(svn_stream_printf(stream, pool, "%s\n", filename)); + } + + SVN_ERR(svn_stream_close(stream)); + } + + return SVN_NO_ERROR; +} + +/* Set the revision property list of revision REV in filesystem FS to + PROPLIST. Use POOL for temporary allocations. */ +svn_error_t * +svn_fs_x__set_revision_proplist(svn_fs_t *fs, + svn_revnum_t rev, + apr_hash_t *proplist, + apr_pool_t *pool) +{ + svn_boolean_t is_packed; + svn_boolean_t bump_generation = FALSE; + const char *final_path; + const char *tmp_path; + const char *perms_reference; + apr_array_header_t *files_to_delete = NULL; + + SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, pool)); + + /* this info will not change while we hold the global FS write lock */ + is_packed = svn_fs_x__is_packed_revprop(fs, rev); + + /* Test whether revprops already exist for this revision. + * Only then will we need to bump the revprop generation. */ + if (has_revprop_cache(fs, pool)) + { + if (is_packed) + { + bump_generation = TRUE; + } + else + { + svn_node_kind_t kind; + SVN_ERR(svn_io_check_path(svn_fs_x__path_revprops(fs, rev, pool), + &kind, pool)); + bump_generation = kind != svn_node_none; + } + } + + /* Serialize the new revprop data */ + if (is_packed) + SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete, + fs, rev, proplist, pool)); + else + SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path, + fs, rev, proplist, pool)); + + /* We use the rev file of this revision as the perms reference, + * because when setting revprops for the first time, the revprop + * file won't exist and therefore can't serve as its own reference. + * (Whereas the rev file should already exist at this point.) + */ + perms_reference = svn_fs_x__path_rev_absolute(fs, rev, pool); + + /* Now, switch to the new revprop data. */ + SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference, + files_to_delete, bump_generation, pool)); + + return SVN_NO_ERROR; +} + +/* Return TRUE, if for REVISION in FS, we can find the revprop pack file. + * Use POOL for temporary allocations. + * Set *MISSING, if the reason is a missing manifest or pack file. + */ +svn_boolean_t +svn_fs_x__packed_revprop_available(svn_boolean_t *missing, + svn_fs_t *fs, + svn_revnum_t revision, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_stringbuf_t *content = NULL; + + /* try to read the manifest file */ + const char *folder = svn_fs_x__path_revprops_pack_shard(fs, revision, pool); + const char *manifest_path = svn_dirent_join(folder, PATH_MANIFEST, pool); + + svn_error_t *err = svn_fs_x__try_stringbuf_from_file(&content, + missing, + manifest_path, + FALSE, + pool); + + /* if the manifest cannot be read, consider the pack files inaccessible + * even if the file itself exists. */ + if (err) + { + svn_error_clear(err); + return FALSE; + } + + if (*missing) + return FALSE; + + /* parse manifest content until we find the entry for REVISION. + * Revision 0 is never packed. */ + revision = revision < ffd->max_files_per_dir + ? revision - 1 + : revision % ffd->max_files_per_dir; + while (content->data) + { + char *next = strchr(content->data, '\n'); + if (next) + { + *next = 0; + ++next; + } + + if (revision-- == 0) + { + /* the respective pack file must exist (and be a file) */ + svn_node_kind_t kind; + err = svn_io_check_path(svn_dirent_join(folder, content->data, + pool), + &kind, pool); + if (err) + { + svn_error_clear(err); + return FALSE; + } + + *missing = kind == svn_node_none; + return kind == svn_node_file; + } + + content->data = next; + } + + return FALSE; +} + + +/****** Packing FSX shards *********/ + +/* Copy revprop files for revisions [START_REV, END_REV) from SHARD_PATH + * to the pack file at PACK_FILE_NAME in PACK_FILE_DIR. + * + * The file sizes have already been determined and written to SIZES. + * Please note that this function will be executed while the filesystem + * has been locked and that revprops files will therefore not be modified + * while the pack is in progress. + * + * COMPRESSION_LEVEL defines how well the resulting pack file shall be + * compressed or whether is shall be compressed at all. TOTAL_SIZE is + * a hint on which initial buffer size we should use to hold the pack file + * content. + * + * CANCEL_FUNC and CANCEL_BATON are used as usual. Temporary allocations + * are done in SCRATCH_POOL. + */ +svn_error_t * +svn_fs_x__copy_revprops(const char *pack_file_dir, + const char *pack_filename, + const char *shard_path, + svn_revnum_t start_rev, + svn_revnum_t end_rev, + apr_array_header_t *sizes, + apr_size_t total_size, + int compression_level, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + svn_stream_t *pack_stream; + apr_file_t *pack_file; + svn_revnum_t rev; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + svn_stream_t *stream; + + /* create empty data buffer and a write stream on top of it */ + svn_stringbuf_t *uncompressed + = svn_stringbuf_create_ensure(total_size, scratch_pool); + svn_stringbuf_t *compressed + = svn_stringbuf_create_empty(scratch_pool); + pack_stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); + + /* write the pack file header */ + SVN_ERR(serialize_revprops_header(pack_stream, start_rev, sizes, 0, + sizes->nelts, iterpool)); + + /* Some useful paths. */ + SVN_ERR(svn_io_file_open(&pack_file, svn_dirent_join(pack_file_dir, + pack_filename, + scratch_pool), + APR_WRITE | APR_CREATE, APR_OS_DEFAULT, + scratch_pool)); + + /* Iterate over the revisions in this shard, squashing them together. */ + for (rev = start_rev; rev <= end_rev; rev++) + { + const char *path; + + svn_pool_clear(iterpool); + + /* Construct the file name. */ + path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), + iterpool); + + /* Copy all the bits from the non-packed revprop file to the end of + * the pack file. */ + SVN_ERR(svn_stream_open_readonly(&stream, path, iterpool, iterpool)); + SVN_ERR(svn_stream_copy3(stream, pack_stream, + cancel_func, cancel_baton, iterpool)); + } + + /* flush stream buffers to content buffer */ + SVN_ERR(svn_stream_close(pack_stream)); + + /* compress the content (or just store it for COMPRESSION_LEVEL 0) */ + SVN_ERR(svn__compress(uncompressed, compressed, compression_level)); + + /* write the pack file content to disk */ + stream = svn_stream_from_aprfile2(pack_file, FALSE, scratch_pool); + SVN_ERR(svn_stream_write(stream, compressed->data, &compressed->len)); + SVN_ERR(svn_stream_close(stream)); + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* For the revprop SHARD at SHARD_PATH with exactly MAX_FILES_PER_DIR + * revprop files in it, create a packed shared at PACK_FILE_DIR. + * + * COMPRESSION_LEVEL defines how well the resulting pack file shall be + * compressed or whether is shall be compressed at all. Individual pack + * file containing more than one revision will be limited to a size of + * MAX_PACK_SIZE bytes before compression. + * + * CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary + * allocations are done in SCRATCH_POOL. + */ +svn_error_t * +svn_fs_x__pack_revprops_shard(const char *pack_file_dir, + const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + apr_off_t max_pack_size, + int compression_level, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + const char *manifest_file_path, *pack_filename = NULL; + svn_stream_t *manifest_stream; + svn_revnum_t start_rev, end_rev, rev; + apr_off_t total_size; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + apr_array_header_t *sizes; + + /* Some useful paths. */ + manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, + scratch_pool); + + /* Remove any existing pack file for this shard, since it is incomplete. */ + SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton, + scratch_pool)); + + /* Create the new directory and manifest file stream. */ + SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool)); + SVN_ERR(svn_stream_open_writable(&manifest_stream, manifest_file_path, + scratch_pool, scratch_pool)); + + /* revisions to handle. Special case: revision 0 */ + start_rev = (svn_revnum_t) (shard * max_files_per_dir); + end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1); + if (start_rev == 0) + ++start_rev; + /* Special special case: if max_files_per_dir is 1, then at this point + start_rev == 1 and end_rev == 0 (!). Fortunately, everything just + works. */ + + /* initialize the revprop size info */ + sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_off_t)); + total_size = 2 * SVN_INT64_BUFFER_SIZE; + + /* Iterate over the revisions in this shard, determine their size and + * squashing them together into pack files. */ + for (rev = start_rev; rev <= end_rev; rev++) + { + apr_finfo_t finfo; + const char *path; + + svn_pool_clear(iterpool); + + /* Get the size of the file. */ + path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), + iterpool); + SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool)); + + /* if we already have started a pack file and this revprop cannot be + * appended to it, write the previous pack file. */ + if (sizes->nelts != 0 && + total_size + SVN_INT64_BUFFER_SIZE + finfo.size > max_pack_size) + { + SVN_ERR(svn_fs_x__copy_revprops(pack_file_dir, pack_filename, + shard_path, start_rev, rev-1, + sizes, (apr_size_t)total_size, + compression_level, cancel_func, + cancel_baton, iterpool)); + + /* next pack file starts empty again */ + apr_array_clear(sizes); + total_size = 2 * SVN_INT64_BUFFER_SIZE; + start_rev = rev; + } + + /* Update the manifest. Allocate a file name for the current pack + * file if it is a new one */ + if (sizes->nelts == 0) + pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev); + + SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%s\n", + pack_filename)); + + /* add to list of files to put into the current pack file */ + APR_ARRAY_PUSH(sizes, apr_off_t) = finfo.size; + total_size += SVN_INT64_BUFFER_SIZE + finfo.size; + } + + /* write the last pack file */ + if (sizes->nelts != 0) + SVN_ERR(svn_fs_x__copy_revprops(pack_file_dir, pack_filename, shard_path, + start_rev, rev-1, sizes, + (apr_size_t)total_size, compression_level, + cancel_func, cancel_baton, iterpool)); + + /* flush the manifest file and update permissions */ + SVN_ERR(svn_stream_close(manifest_stream)); + SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Delete the non-packed revprop SHARD at SHARD_PATH with exactly + * MAX_FILES_PER_DIR revprop files in it. If this is shard 0, keep the + * revprop file for revision 0. + * + * CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary + * allocations are done in SCRATCH_POOL. + */ +svn_error_t * +svn_fs_x__delete_revprops_shard(const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + if (shard == 0) + { + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + int i; + + /* delete all files except the one for revision 0 */ + for (i = 1; i < max_files_per_dir; ++i) + { + const char *path = svn_dirent_join(shard_path, + apr_psprintf(iterpool, "%d", i), + iterpool); + if (cancel_func) + SVN_ERR((*cancel_func)(cancel_baton)); + + SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + } + else + SVN_ERR(svn_io_remove_dir2(shard_path, TRUE, + cancel_func, cancel_baton, scratch_pool)); + + return SVN_NO_ERROR; +} +
diff --git a/subversion/libsvn_fs_x/revprops.h b/subversion/libsvn_fs_x/revprops.h new file mode 100644 index 0000000..b5fcf2c --- /dev/null +++ b/subversion/libsvn_fs_x/revprops.h
@@ -0,0 +1,168 @@ +/* revprops.h --- everything needed to handle revprops in FSX + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "svn_fs.h" + +/* Write the CURRENT revprop generation to disk for repository FS. + */ +svn_error_t * +svn_fs_x__write_revprop_generation_file(svn_fs_t *fs, + apr_int64_t current, + apr_pool_t *pool); + +/* Make sure the revprop_namespace member in FS is set. */ +svn_error_t * +svn_fs_x__cleanup_revprop_namespace(svn_fs_t *fs); + +/* In the filesystem FS, pack all revprop shards up to min_unpacked_rev. + * + * NOTE: Keep the old non-packed shards around until after the format bump. + * Otherwise, re-running upgrade will drop the packed revprop shard but + * have no unpacked data anymore. Call upgrade_cleanup_pack_revprops after + * the bump. + * + * NOTIFY_FUNC and NOTIFY_BATON as well as CANCEL_FUNC and CANCEL_BATON are + * used in the usual way. Temporary allocations are done in SCRATCH_POOL. + */ +svn_error_t * +svn_fs_x__upgrade_pack_revprops(svn_fs_t *fs, + svn_fs_upgrade_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool); + +/* In the filesystem FS, remove all non-packed revprop shards up to + * min_unpacked_rev. Temporary allocations are done in SCRATCH_POOL. + * + * NOTIFY_FUNC and NOTIFY_BATON as well as CANCEL_FUNC and CANCEL_BATON are + * used in the usual way. Cancellation is supported in the sense that we + * will cleanly abort the operation. However, there will be remnant shards + * that must be removed manually. + * + * See upgrade_pack_revprops for more info. + */ +svn_error_t * +svn_fs_x__upgrade_cleanup_pack_revprops(svn_fs_t *fs, + svn_fs_upgrade_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool); + +/* Read the revprops for revision REV in FS and return them in *PROPERTIES_P. + * + * Allocations will be done in POOL. + */ +svn_error_t * +svn_fs_x__get_revision_proplist(apr_hash_t **proplist_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +/* Set the revision property list of revision REV in filesystem FS to + PROPLIST. Use POOL for temporary allocations. */ +svn_error_t * +svn_fs_x__set_revision_proplist(svn_fs_t *fs, + svn_revnum_t rev, + apr_hash_t *proplist, + apr_pool_t *pool); + + +/* Return TRUE, if for REVISION in FS, we can find the revprop pack file. + * Use POOL for temporary allocations. + * Set *MISSING, if the reason is a missing manifest or pack file. + */ +svn_boolean_t +svn_fs_x__packed_revprop_available(svn_boolean_t *missing, + svn_fs_t *fs, + svn_revnum_t revision, + apr_pool_t *pool); + + +/****** Packing FSX shards *********/ + +/* Copy revprop files for revisions [START_REV, END_REV) from SHARD_PATH + * to the pack file at PACK_FILE_NAME in PACK_FILE_DIR. + * + * The file sizes have already been determined and written to SIZES. + * Please note that this function will be executed while the filesystem + * has been locked and that revprops files will therefore not be modified + * while the pack is in progress. + * + * COMPRESSION_LEVEL defines how well the resulting pack file shall be + * compressed or whether is shall be compressed at all. TOTAL_SIZE is + * a hint on which initial buffer size we should use to hold the pack file + * content. + * + * CANCEL_FUNC and CANCEL_BATON are used as usual. Temporary allocations + * are done in SCRATCH_POOL. + */ +svn_error_t * +svn_fs_x__copy_revprops(const char *pack_file_dir, + const char *pack_filename, + const char *shard_path, + svn_revnum_t start_rev, + svn_revnum_t end_rev, + apr_array_header_t *sizes, + apr_size_t total_size, + int compression_level, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool); + +/* For the revprop SHARD at SHARD_PATH with exactly MAX_FILES_PER_DIR + * revprop files in it, create a packed shared at PACK_FILE_DIR. + * + * COMPRESSION_LEVEL defines how well the resulting pack file shall be + * compressed or whether is shall be compressed at all. Individual pack + * file containing more than one revision will be limited to a size of + * MAX_PACK_SIZE bytes before compression. + * + * CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary + * allocations are done in SCRATCH_POOL. + */ +svn_error_t * +svn_fs_x__pack_revprops_shard(const char *pack_file_dir, + const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + apr_off_t max_pack_size, + int compression_level, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool); + +/* Delete the non-packed revprop SHARD at SHARD_PATH with exactly + * MAX_FILES_PER_DIR revprop files in it. If this is shard 0, keep the + * revprop file for revision 0. + * + * CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary + * allocations are done in SCRATCH_POOL. + */ +svn_error_t * +svn_fs_x__delete_revprops_shard(const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool);
diff --git a/subversion/libsvn_fs_x/string_table.c b/subversion/libsvn_fs_x/string_table.c new file mode 100644 index 0000000..5e13635 --- /dev/null +++ b/subversion/libsvn_fs_x/string_table.c
@@ -0,0 +1,900 @@ +/* string_table.c : operations on string tables + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include <assert.h> +#include <string.h> +#include <apr_tables.h> + +#include "svn_string.h" +#include "svn_sorts.h" +#include "private/svn_string_private.h" +#include "private/svn_subr_private.h" +#include "private/svn_packed_data.h" +#include "string_table.h" + + + +#define MAX_DATA_SIZE 0xffff +#define MAX_SHORT_STRING_LEN (MAX_DATA_SIZE / 4) +#define TABLE_SHIFT 13 +#define MAX_STRINGS_PER_TABLE (1 << (TABLE_SHIFT - 1)) +#define LONG_STRING_MASK (1 << (TABLE_SHIFT - 1)) +#define STRING_INDEX_MASK ((1 << (TABLE_SHIFT - 1)) - 1) +#define PADDING (sizeof(apr_uint64_t)) + + +typedef struct builder_string_t +{ + svn_string_t string; + int position; + apr_size_t depth; + struct builder_string_t *previous; + struct builder_string_t *next; + apr_size_t previous_match_len; + apr_size_t next_match_len; + struct builder_string_t *left; + struct builder_string_t *right; +} builder_string_t; + +typedef struct builder_table_t +{ + apr_size_t max_data_size; + builder_string_t *top; + builder_string_t *first; + builder_string_t *last; + apr_array_header_t *short_strings; + apr_array_header_t *long_strings; + apr_hash_t *long_string_dict; + apr_size_t long_string_size; +} builder_table_t; + +struct string_table_builder_t +{ + apr_pool_t *pool; + apr_array_header_t *tables; +}; + +typedef struct string_header_t +{ + apr_uint16_t head_string; + apr_uint16_t head_length; + apr_uint16_t tail_start; + apr_uint16_t tail_length; +} string_header_t; + +typedef struct string_sub_table_t +{ + const char *data; + apr_size_t data_size; + + string_header_t *short_strings; + apr_size_t short_string_count; + + svn_string_t *long_strings; + apr_size_t long_string_count; +} string_sub_table_t; + +struct string_table_t +{ + apr_size_t size; + string_sub_table_t *sub_tables; +}; + + +/* Accessing ID Pieces. */ + +static builder_table_t * +add_table(string_table_builder_t *builder) +{ + builder_table_t *table = apr_pcalloc(builder->pool, sizeof(*table)); + table->max_data_size = MAX_DATA_SIZE - PADDING; /* ensure there remain a few + unused bytes at the end */ + table->short_strings = apr_array_make(builder->pool, 64, + sizeof(builder_string_t *)); + table->long_strings = apr_array_make(builder->pool, 0, + sizeof(svn_string_t)); + table->long_string_dict = svn_hash__make(builder->pool); + + APR_ARRAY_PUSH(builder->tables, builder_table_t *) = table; + + return table; +} + +string_table_builder_t * +svn_fs_x__string_table_builder_create(apr_pool_t *pool) +{ + string_table_builder_t *result = apr_palloc(pool, sizeof(*result)); + result->pool = pool; + result->tables = apr_array_make(pool, 1, sizeof(builder_table_t *)); + + add_table(result); + + return result; +} + +static void +balance(builder_table_t *table, + builder_string_t **parent, + builder_string_t *node) +{ + unsigned left_hight = node->left ? node->left->depth + 1 : 0; + unsigned right_hight = node->right ? node->right->depth + 1 : 0; + + if (left_hight > right_hight + 1) + { + builder_string_t *temp = node->left->right; + node->left->right = node; + *parent = node->left; + node->left = temp; + + --left_hight; + } + else if (left_hight + 1 < right_hight) + { + builder_string_t *temp = node->right->left; + *parent = node->right; + node->right->left = node; + node->right = temp; + + --right_hight; + } + + node->depth = MAX(left_hight, right_hight); +} + +static apr_uint16_t +match_length(const svn_string_t *lhs, + const svn_string_t *rhs) +{ + apr_size_t len = MIN(rhs->len, rhs->len); + return (apr_uint16_t)svn_cstring__match_length(lhs->data, rhs->data, len); +} + +static apr_uint16_t +insert_string(builder_table_t *table, + builder_string_t **parent, + builder_string_t *to_insert) +{ + apr_uint16_t result; + builder_string_t *current = *parent; + int diff = strcmp(current->string.data, to_insert->string.data); + if (diff == 0) + { + apr_array_pop(table->short_strings); + return current->position; + } + + if (diff < 0) + { + if (current->left == NULL) + { + current->left = to_insert; + + to_insert->previous = current->previous; + to_insert->next = current; + + if (to_insert->previous == NULL) + { + table->first = to_insert; + } + else + { + builder_string_t *previous = to_insert->previous; + to_insert->previous_match_len + = match_length(&previous->string, &to_insert->string); + + previous->next = to_insert; + previous->next_match_len = to_insert->previous_match_len; + } + + current->previous = to_insert; + to_insert->next_match_len + = match_length(¤t->string, &to_insert->string); + current->previous_match_len = to_insert->next_match_len; + + table->max_data_size -= to_insert->string.len; + if (to_insert->previous == NULL) + table->max_data_size += to_insert->next_match_len; + else + table->max_data_size += MIN(to_insert->previous_match_len, + to_insert->next_match_len); + + return to_insert->position; + } + else + result = insert_string(table, ¤t->left, to_insert); + } + else + { + if (current->right == NULL) + { + current->right = to_insert; + + to_insert->next = current->next; + to_insert->previous = current; + + if (to_insert->next == NULL) + { + table->last = to_insert; + } + else + { + builder_string_t *next = to_insert->next; + to_insert->next_match_len + = match_length(&next->string, &to_insert->string); + + next->previous = to_insert; + next->previous_match_len = to_insert->next_match_len; + } + + current->next = current->right; + to_insert->previous_match_len + = match_length(¤t->string, &to_insert->string); + current->next_match_len = to_insert->previous_match_len; + + table->max_data_size -= to_insert->string.len; + if (to_insert->next == NULL) + table->max_data_size += to_insert->previous_match_len; + else + table->max_data_size += MIN(to_insert->previous_match_len, + to_insert->next_match_len); + + return to_insert->position; + } + else + result = insert_string(table, ¤t->right, to_insert); + } + + balance(table, parent, current); + return result; +} + +apr_size_t +svn_fs_x__string_table_builder_add(string_table_builder_t *builder, + const char *string, + apr_size_t len) +{ + apr_size_t result = -1; + builder_table_t *table = APR_ARRAY_IDX(builder->tables, + builder->tables->nelts - 1, + builder_table_t *); + if (len == 0) + len = strlen(string); + + string = apr_pstrmemdup(builder->pool, string, len); + if (len > MAX_SHORT_STRING_LEN) + { + svn_string_t item; + item.data = string; + item.len = len; + + result + = (apr_uintptr_t)apr_hash_get(table->long_string_dict, string, len); + if (result) + return result - 1 + + LONG_STRING_MASK + + (((apr_size_t)builder->tables->nelts - 1) << TABLE_SHIFT); + + if (table->long_strings->nelts == MAX_STRINGS_PER_TABLE) + table = add_table(builder); + + result = table->long_strings->nelts + + LONG_STRING_MASK + + (((apr_size_t)builder->tables->nelts - 1) << TABLE_SHIFT); + APR_ARRAY_PUSH(table->long_strings, svn_string_t) = item; + apr_hash_set(table->long_string_dict, string, len, + (void*)(apr_uintptr_t)table->long_strings->nelts); + + table->long_string_size += len; + } + else + { + builder_string_t *item = apr_pcalloc(builder->pool, sizeof(*item)); + item->string.data = string; + item->string.len = len; + item->previous_match_len = 0; + item->next_match_len = 0; + + if ( table->short_strings->nelts == MAX_STRINGS_PER_TABLE + || table->max_data_size < len) + table = add_table(builder); + + item->position = (apr_size_t)table->short_strings->nelts; + APR_ARRAY_PUSH(table->short_strings, builder_string_t *) = item; + + if (table->top == NULL) + { + table->max_data_size -= len; + table->top = item; + table->first = item; + table->last = item; + + result = ((apr_size_t)builder->tables->nelts - 1) << TABLE_SHIFT; + } + else + { + result = insert_string(table, &table->top, item) + + (((apr_size_t)builder->tables->nelts - 1) << TABLE_SHIFT); + } + } + + return result; +} + +apr_size_t +svn_fs_x__string_table_builder_estimate_size(string_table_builder_t *builder) +{ + apr_size_t total = 0; + int i; + + for (i = 0; i < builder->tables->nelts; ++i) + { + builder_table_t *table + = APR_ARRAY_IDX(builder->tables, i, builder_table_t*); + + /* total number of chars to store, + * 8 bytes per short string table entry + * 4 bytes per long string table entry + * some static overhead */ + apr_size_t table_size + = MAX_DATA_SIZE - table->max_data_size + + table->long_string_size + + table->short_strings->nelts * 8 + + table->long_strings->nelts * 4 + + 10; + + total += table_size; + } + + /* ZIP compression should give us a 50% reduction. + * add some static overhead */ + return 200 + total / 2; + +} + +static void +create_table(string_sub_table_t *target, + builder_table_t *source, + apr_pool_t *pool, + apr_pool_t *scratch_pool) +{ + int i = 0; + apr_hash_t *tails = svn_hash__make(scratch_pool); + svn_stringbuf_t *data + = svn_stringbuf_create_ensure(MAX_DATA_SIZE - source->max_data_size, + scratch_pool); + + /* pack sub-strings */ + target->short_string_count = (apr_size_t)source->short_strings->nelts; + target->short_strings = apr_palloc(pool, sizeof(*target->short_strings) * + target->short_string_count); + for (i = 0; i < source->short_strings->nelts; ++i) + { + const builder_string_t *string + = APR_ARRAY_IDX(source->short_strings, i, const builder_string_t *); + + string_header_t *entry = &target->short_strings[i]; + const char *tail = string->string.data + string->previous_match_len; + string_header_t *tail_match; + apr_size_t head_length = string->previous_match_len; + + /* Minimize the number of strings to visit when reconstructing the + string head. So, skip all predecessors that don't contribute to + first HEAD_LENGTH chars of our string. */ + if (head_length) + { + const builder_string_t *furthest_prev = string->previous; + while (furthest_prev->previous_match_len >= head_length) + furthest_prev = furthest_prev->previous; + entry->head_string = furthest_prev->position; + } + else + entry->head_string = 0; + + /* head & tail length are known */ + entry->head_length = (apr_uint16_t)head_length; + entry->tail_length + = (apr_uint16_t)(string->string.len - entry->head_length); + + /* try to reuse an existing tail segment */ + tail_match = apr_hash_get(tails, tail, entry->tail_length); + if (tail_match) + { + entry->tail_start = tail_match->tail_start; + } + else + { + entry->tail_start = (apr_uint16_t)data->len; + svn_stringbuf_appendbytes(data, tail, entry->tail_length); + apr_hash_set(tails, tail, entry->tail_length, entry); + } + } + + /* pack long strings */ + target->long_string_count = (apr_size_t)source->long_strings->nelts; + target->long_strings = apr_palloc(pool, sizeof(*target->long_strings) * + target->long_string_count); + for (i = 0; i < source->long_strings->nelts; ++i) + { + svn_string_t *string = &target->long_strings[i]; + *string = APR_ARRAY_IDX(source->long_strings, i, svn_string_t); + string->data = apr_pstrmemdup(pool, string->data, string->len); + } + + data->len += PADDING; /* there a few extra bytes at then of the buffer + that we want to keep */ + assert(data->len < data->blocksize); + + target->data = apr_pmemdup(pool, data->data, data->len); + target->data_size = data->len; +} + +string_table_t * +svn_fs_x__string_table_create(const string_table_builder_t *builder, + apr_pool_t *pool) +{ + apr_size_t i; + + string_table_t *result = apr_pcalloc(pool, sizeof(*result)); + result->size = (apr_size_t)builder->tables->nelts; + result->sub_tables + = apr_pcalloc(pool, result->size * sizeof(*result->sub_tables)); + + for (i = 0; i < result->size; ++i) + create_table(&result->sub_tables[i], + APR_ARRAY_IDX(builder->tables, i, builder_table_t*), + pool, + builder->pool); + + return result; +} + +/* Masks used by table_copy_string. copy_mask[I] is used if the target + content to be preserved starts at byte I within the current chunk. + This is used to work around alignment issues. + */ +#if SVN_UNALIGNED_ACCESS_IS_OK +static const char *copy_masks[8] = { "\xff\xff\xff\xff\xff\xff\xff\xff", + "\x00\xff\xff\xff\xff\xff\xff\xff", + "\x00\x00\xff\xff\xff\xff\xff\xff", + "\x00\x00\x00\xff\xff\xff\xff\xff", + "\x00\x00\x00\x00\xff\xff\xff\xff", + "\x00\x00\x00\x00\x00\xff\xff\xff", + "\x00\x00\x00\x00\x00\x00\xff\xff", + "\x00\x00\x00\x00\x00\x00\x00\xff" }; +#endif + +static void +table_copy_string(char *buffer, + apr_size_t len, + const string_sub_table_t *table, + string_header_t *header) +{ + buffer[len] = '\0'; + do + { + assert(header->head_length <= len); + { +#if SVN_UNALIGNED_ACCESS_IS_OK + /* the sections that we copy tend to be short but we can copy + *all* of it chunky because we made sure that source and target + buffer have some extra padding to prevent segfaults. */ + apr_uint64_t mask; + apr_size_t to_copy = len - header->head_length; + apr_size_t copied = 0; + + const char *source = table->data + header->tail_start; + char *target = buffer + header->head_length; + len = header->head_length; + + /* copy whole chunks */ + while (to_copy >= copied + sizeof(apr_uint64_t)) + { + *(apr_uint64_t *)(target + copied) + = *(const apr_uint64_t *)(source + copied); + copied += sizeof(apr_uint64_t); + } + + /* copy the remainder assuming that we have up to 8 extra bytes + of addressable buffer on the source and target sides. + Now, we simply copy 8 bytes and use a mask to filter & merge + old with new data. */ + mask = *(const apr_uint64_t *)copy_masks[to_copy - copied]; + *(apr_uint64_t *)(target + copied) + = (*(apr_uint64_t *)(target + copied) & mask) + | (*(const apr_uint64_t *)(source + copied) & ~mask); +#else + memcpy(buffer + header->head_length, + table->data + header->tail_start, + len - header->head_length); + len = header->head_length; +#endif + } + + header = &table->short_strings[header->head_string]; + } + while (len); +} + +const char* +svn_fs_x__string_table_get(const string_table_t *table, + apr_size_t idx, + apr_size_t *length, + apr_pool_t *pool) +{ + apr_size_t table_number = idx >> TABLE_SHIFT; + apr_size_t sub_index = idx & STRING_INDEX_MASK; + + if (table_number < table->size) + { + string_sub_table_t *sub_table = &table->sub_tables[table_number]; + if (idx & LONG_STRING_MASK) + { + if (sub_index < sub_table->long_string_count) + { + if (length) + *length = sub_table->long_strings[sub_index].len; + + return apr_pstrmemdup(pool, + sub_table->long_strings[sub_index].data, + sub_table->long_strings[sub_index].len); + } + } + else + { + if (sub_index < sub_table->short_string_count) + { + string_header_t *header = sub_table->short_strings + sub_index; + apr_size_t len = header->head_length + header->tail_length; + char *result = apr_palloc(pool, len + PADDING); + + if (length) + *length = len; + table_copy_string(result, len, sub_table, header); + + return result; + } + } + } + + return apr_pstrmemdup(pool, "", 0); +} + +svn_error_t * +svn_fs_x__write_string_table(svn_stream_t *stream, + const string_table_t *table, + apr_pool_t *pool) +{ + apr_size_t i, k; + + svn_packed__data_root_t *root = svn_packed__data_create_root(pool); + + svn_packed__int_stream_t *table_sizes + = svn_packed__create_int_stream(root, FALSE, FALSE); + svn_packed__int_stream_t *small_strings_headers + = svn_packed__create_int_stream(root, FALSE, FALSE); + svn_packed__byte_stream_t *large_strings + = svn_packed__create_bytes_stream(root); + svn_packed__byte_stream_t *small_strings_data + = svn_packed__create_bytes_stream(root); + + svn_packed__create_int_substream(small_strings_headers, TRUE, FALSE); + svn_packed__create_int_substream(small_strings_headers, FALSE, FALSE); + svn_packed__create_int_substream(small_strings_headers, TRUE, FALSE); + svn_packed__create_int_substream(small_strings_headers, FALSE, FALSE); + + /* number of sub-tables */ + + svn_packed__add_uint(table_sizes, table->size); + + /* all short-string char data sizes */ + + for (i = 0; i < table->size; ++i) + svn_packed__add_uint(table_sizes, + table->sub_tables[i].short_string_count); + + for (i = 0; i < table->size; ++i) + svn_packed__add_uint(table_sizes, + table->sub_tables[i].long_string_count); + + /* all strings */ + + for (i = 0; i < table->size; ++i) + { + string_sub_table_t *sub_table = &table->sub_tables[i]; + svn_packed__add_bytes(small_strings_data, + sub_table->data, + sub_table->data_size); + + for (k = 0; k < sub_table->short_string_count; ++k) + { + string_header_t *string = &sub_table->short_strings[k]; + + svn_packed__add_uint(small_strings_headers, string->head_string); + svn_packed__add_uint(small_strings_headers, string->head_length); + svn_packed__add_uint(small_strings_headers, string->tail_start); + svn_packed__add_uint(small_strings_headers, string->tail_length); + } + + for (k = 0; k < sub_table->long_string_count; ++k) + svn_packed__add_bytes(large_strings, + sub_table->long_strings[k].data, + sub_table->long_strings[k].len + 1); + } + + /* write to target stream */ + + SVN_ERR(svn_packed__data_write(stream, root, pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__read_string_table(string_table_t **table_p, + svn_stream_t *stream, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + apr_size_t i, k; + + string_table_t *table = apr_palloc(result_pool, sizeof(*table)); + + svn_packed__data_root_t *root; + svn_packed__int_stream_t *table_sizes; + svn_packed__byte_stream_t *large_strings; + svn_packed__byte_stream_t *small_strings_data; + svn_packed__int_stream_t *headers; + + SVN_ERR(svn_packed__data_read(&root, stream, result_pool, scratch_pool)); + table_sizes = svn_packed__first_int_stream(root); + headers = svn_packed__next_int_stream(table_sizes); + large_strings = svn_packed__first_byte_stream(root); + small_strings_data = svn_packed__next_byte_stream(large_strings); + + /* create sub-tables */ + + table->size = svn_packed__get_uint(table_sizes); + table->sub_tables = apr_pcalloc(result_pool, + table->size * sizeof(*table->sub_tables)); + + /* read short strings */ + + for (i = 0; i < table->size; ++i) + { + string_sub_table_t *sub_table = &table->sub_tables[i]; + + sub_table->short_string_count = svn_packed__get_uint(table_sizes); + if (sub_table->short_string_count) + { + sub_table->short_strings + = apr_pcalloc(result_pool, sub_table->short_string_count + * sizeof(*sub_table->short_strings)); + + /* read short string headers */ + + for (k = 0; k < sub_table->short_string_count; ++k) + { + string_header_t *string = &sub_table->short_strings[k]; + + string->head_string = svn_packed__get_uint(headers); + string->head_length = svn_packed__get_uint(headers); + string->tail_start = svn_packed__get_uint(headers); + string->tail_length = svn_packed__get_uint(headers); + } + } + + sub_table->data = svn_packed__get_bytes(small_strings_data, + &sub_table->data_size); + } + + /* read long strings */ + + for (i = 0; i < table->size; ++i) + { + /* initialize long string table */ + string_sub_table_t *sub_table = &table->sub_tables[i]; + + sub_table->long_string_count = svn_packed__get_uint(table_sizes); + if (sub_table->long_string_count) + { + sub_table->long_strings + = apr_pcalloc(result_pool, sub_table->long_string_count + * sizeof(*sub_table->long_strings)); + + /* read long strings */ + + for (k = 0; k < sub_table->long_string_count; ++k) + { + svn_string_t *string = &sub_table->long_strings[k]; + string->data = svn_packed__get_bytes(large_strings, + &string->len); + string->len--; + } + } + } + + /* done */ + + *table_p = table; + + return SVN_NO_ERROR; +} + +void +svn_fs_x__serialize_string_table(svn_temp_serializer__context_t *context, + string_table_t **st) +{ + apr_size_t i, k; + string_table_t *string_table = *st; + if (string_table == NULL) + return; + + /* string table struct */ + svn_temp_serializer__push(context, + (const void * const *)st, + sizeof(*string_table)); + + /* sub-table array (all structs in a single memory block) */ + svn_temp_serializer__push(context, + (const void * const *)&string_table->sub_tables, + sizeof(*string_table->sub_tables) * + string_table->size); + + /* sub-elements of all sub-tables */ + for (i = 0; i < string_table->size; ++i) + { + string_sub_table_t *sub_table = &string_table->sub_tables[i]; + svn_temp_serializer__add_leaf(context, + (const void * const *)&sub_table->data, + sub_table->data_size); + svn_temp_serializer__add_leaf(context, + (const void * const *)&sub_table->short_strings, + sub_table->short_string_count * sizeof(string_header_t)); + + /* all "long string" instances form a single memory block */ + svn_temp_serializer__push(context, + (const void * const *)&sub_table->long_strings, + sub_table->long_string_count * sizeof(svn_string_t)); + + /* serialize actual long string contents */ + for (k = 0; k < sub_table->long_string_count; ++k) + { + svn_string_t *string = &sub_table->long_strings[k]; + svn_temp_serializer__add_leaf(context, + (const void * const *)&string->data, + string->len + 1); + } + + svn_temp_serializer__pop(context); + } + + /* back to the caller's nesting level */ + svn_temp_serializer__pop(context); + svn_temp_serializer__pop(context); +} + +void +svn_fs_x__deserialize_string_table(void *buffer, + string_table_t **table) +{ + apr_size_t i, k; + string_sub_table_t *sub_tables; + + svn_temp_deserializer__resolve(buffer, (void **)table); + if (*table == NULL) + return; + + svn_temp_deserializer__resolve(*table, (void **)&(*table)->sub_tables); + sub_tables = (*table)->sub_tables; + for (i = 0; i < (*table)->size; ++i) + { + string_sub_table_t *sub_table = sub_tables + i; + + svn_temp_deserializer__resolve(sub_tables, + (void **)&sub_table->data); + svn_temp_deserializer__resolve(sub_tables, + (void **)&sub_table->short_strings); + svn_temp_deserializer__resolve(sub_tables, + (void **)&sub_table->long_strings); + + for (k = 0; k < sub_table->long_string_count; ++k) + svn_temp_deserializer__resolve(sub_table->long_strings, + (void **)&sub_table->long_strings[k].data); + } +} + +const char* +svn_fs_x__string_table_get_func(const string_table_t *table, + apr_size_t idx, + apr_size_t *length, + apr_pool_t *pool) +{ + apr_size_t table_number = idx >> TABLE_SHIFT; + apr_size_t sub_index = idx & STRING_INDEX_MASK; + + if (table_number < table->size) + { + /* resolve TABLE->SUB_TABLES pointer and select sub-table */ + string_sub_table_t *sub_tables + = (string_sub_table_t *)svn_temp_deserializer__ptr(table, + (const void *const *)&table->sub_tables); + string_sub_table_t *sub_table = sub_tables + table_number; + + /* pick the right kind of string */ + if (idx & LONG_STRING_MASK) + { + if (sub_index < sub_table->long_string_count) + { + /* resolve SUB_TABLE->LONG_STRINGS, select the string we want + and resolve the pointer to its char data */ + svn_string_t *long_strings + = (svn_string_t *)svn_temp_deserializer__ptr(sub_table, + (const void *const *)&sub_table->long_strings); + const char *str_data + = (const char*)svn_temp_deserializer__ptr(long_strings, + (const void *const *)&long_strings[sub_index].data); + + /* return a copy of the char data */ + if (length) + *length = long_strings[sub_index].len; + + return apr_pstrmemdup(pool, + str_data, + long_strings[sub_index].len); + } + } + else + { + if (sub_index < sub_table->short_string_count) + { + string_header_t *header; + apr_size_t len; + char *result; + + /* construct a copy of our sub-table struct with SHORT_STRINGS + and DATA pointers resolved. Leave all other pointers as + they are. This allows us to use the same code for string + reconstruction here as in the non-serialized case. */ + string_sub_table_t table_copy = *sub_table; + table_copy.data + = (const char *)svn_temp_deserializer__ptr(sub_tables, + (const void *const *)&sub_table->data); + table_copy.short_strings + = (string_header_t *)svn_temp_deserializer__ptr(sub_tables, + (const void *const *)&sub_table->short_strings); + + /* reconstruct the char data and return it */ + header = table_copy.short_strings + sub_index; + len = header->head_length + header->tail_length; + result = apr_palloc(pool, len + PADDING); + if (length) + *length = len; + + table_copy_string(result, len, &table_copy, header); + + return result; + } + } + } + + return ""; +}
diff --git a/subversion/libsvn_fs_x/string_table.h b/subversion/libsvn_fs_x/string_table.h new file mode 100644 index 0000000..ae22cf2 --- /dev/null +++ b/subversion/libsvn_fs_x/string_table.h
@@ -0,0 +1,133 @@ +/* string_table.h : interface to string tables, private to libsvn_fs_x + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS_X_STRING_TABLE_H +#define SVN_LIBSVN_FS_X_STRING_TABLE_H + +#include "svn_io.h" +#include "private/svn_temp_serializer.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/* A string table is a very space efficient, read-only representation for + * a set of strings with high degreed of prefix and postfix overhead. + * + * Creating a string table is a two-stage process: Use a builder class, + * stuff all the strings in there and let it then do the heavy lifting of + * classification and compression to create the actual string table object. + * + * We will use this for the various path values in FSX change lists and + * node revision items. + */ + +/* the string table builder */ +typedef struct string_table_builder_t string_table_builder_t; + +/* the string table */ +typedef struct string_table_t string_table_t; + +/* Returns a new string table builder object, allocated in POOL. + */ +string_table_builder_t * +svn_fs_x__string_table_builder_create(apr_pool_t *pool); + +/* Add an arbitrary NUL-terminated C-string STRING of the given length LEN + * to BUILDER. Return the index of that string in the future string table. + * If LEN is 0, determine the length of the C-string internally. + */ +apr_size_t +svn_fs_x__string_table_builder_add(string_table_builder_t *builder, + const char *string, + apr_size_t len); + +/* Return an estimate for the on-disk size of the resulting string table. + * The estimate may err in both directions but tends to overestimate the + * space requirements for larger tables. + */ +apr_size_t +svn_fs_x__string_table_builder_estimate_size(string_table_builder_t *builder); + +/* From the given BUILDER object, create a string table object allocated + * in POOL that contains all strings previously added to BUILDER. + */ +string_table_t * +svn_fs_x__string_table_create(const string_table_builder_t *builder, + apr_pool_t *pool); + +/* Extract string number INDEX from TABLE and return a copy of it allocated + * in POOL. If LENGTH is not NULL, set *LENGTH to strlen() of the result + * string. Returns an empty string for invalid indexes. + */ +const char* +svn_fs_x__string_table_get(const string_table_t *table, + apr_size_t index, + apr_size_t *length, + apr_pool_t *pool); + +/* Write a serialized representation of the string table TABLE to STREAM. + * Use POOL for temporary allocations. + */ +svn_error_t * +svn_fs_x__write_string_table(svn_stream_t *stream, + const string_table_t *table, + apr_pool_t *pool); + +/* Read the serialized string table representation from STREAM and return + * the resulting runtime representation in *TABLE_P. Allocate it in + * RESULT_POOL and use SCRATCH_POOL for temporary allocations. + */ +svn_error_t * +svn_fs_x__read_string_table(string_table_t **table_p, + svn_stream_t *stream, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + +/* Serialize string table *ST within the serialization CONTEXT. + */ +void +svn_fs_x__serialize_string_table(svn_temp_serializer__context_t *context, + string_table_t **st); + +/* Deserialize string table *TABLE within the BUFFER. + */ +void +svn_fs_x__deserialize_string_table(void *buffer, + string_table_t **table); + +/* Extract string number INDEX from the cache serialized representation at + * TABLE and return a copy of it allocated in POOL. If LENGTH is not NULL, + * set *LENGTH to strlen() of the result string. Returns an empty string + * for invalid indexes. + */ +const char* +svn_fs_x__string_table_get_func(const string_table_t *table, + apr_size_t idx, + apr_size_t *length, + apr_pool_t *pool); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* SVN_LIBSVN_FS_X_STRING_TABLE_H */
diff --git a/subversion/libsvn_fs_x/structure b/subversion/libsvn_fs_x/structure new file mode 100644 index 0000000..41caf1d --- /dev/null +++ b/subversion/libsvn_fs_x/structure
@@ -0,0 +1,621 @@ +This file describes the design, layouts, and file formats of a +libsvn_fs_fs repository. + +Design +------ + +In FSFS, each committed revision is represented as an immutable file +containing the new node-revisions, contents, and changed-path +information for the revision, plus a second, changeable file +containing the revision properties. + +In contrast to the BDB back end, the contents of recent revision of +files are stored as deltas against earlier revisions, instead of the +other way around. This is less efficient for common-case checkouts, +but brings greater simplicity and robustness, as well as the +flexibility to make commits work without write access to existing +revisions. Skip-deltas and delta combination mitigate the checkout +cost. + +In-progress transactions are represented with a prototype rev file +containing only the new text representations of files (appended to as +changed file contents come in), along with a separate file for each +node-revision, directory representation, or property representation +which has been changed or added in the transaction. During the final +stage of the commit, these separate files are marshalled onto the end +of the prototype rev file to form the immutable revision file. + +Layout of the FS directory +-------------------------- + +The layout of the FS directory (the "db" subdirectory of the +repository) is: + + revs/ Subdirectory containing revs + <shard>/ Shard directory, if sharding is in use (see below) + <revnum> File containing rev <revnum> + <shard>.pack/ Pack directory, if the repo has been packed (see below) + pack Pack file, if the repository has been packed (see below) + manifest Pack manifest file, if a pack file exists (see below) + revprops/ Subdirectory containing rev-props + <shard>/ Shard directory, if sharding is in use (see below) + <revnum> File containing rev-props for <revnum> + <shard>.pack/ Pack directory, if the repo has been packed (see below) + <rev>.<count> Pack file, if the repository has been packed (see below) + manifest Pack manifest file, if a pack file exists (see below) + revprops.db SQLite database of the packed revision properties + transactions/ Subdirectory containing transactions + <txnid>.txn/ Directory containing transaction <txnid> + txn-protorevs/ Subdirectory containing transaction proto-revision files + <txnid>.rev Proto-revision file for transaction <txnid> + <txnid>.rev-lock Write lock for proto-rev file + txn-current File containing the next transaction key + locks/ Subdirectory containing locks + <partial-digest>/ Subdirectory named for first 3 letters of an MD5 digest + <digest> File containing locks/children for path with <digest> + node-origins/ Lazy cache of origin noderevs for nodes + <partial-nodeid> File containing noderev ID of origins of nodes + current File specifying current revision and next node/copy id + fs-type File identifying this filesystem as an FSFS filesystem + write-lock Empty file, locked to serialise writers + txn-current-lock Empty file, locked to serialise 'txn-current' + uuid File containing the UUID of the repository + format File containing the format number of this filesystem + fsfs.conf Configuration file + min-unpacked-rev File containing the oldest revision not in a pack file + min-unpacked-revprop File containing the oldest revision of unpacked revprop + rep-cache.db SQLite database mapping rep checksums to locations + +Files in the revprops directory are in the hash dump format used by +svn_hash_write. + +The format of the "current" file is: + + * Format 3 and above: a single line of the form + "<youngest-revision>\n" giving the youngest revision for the + repository. + + * Format 2 and below: a single line of the form "<youngest-revision> + <next-node-id> <next-copy-id>\n" giving the youngest revision, the + next unique node-ID, and the next unique copy-ID for the + repository. + +The "write-lock" file is an empty file which is locked before the +final stage of a commit and unlocked after the new "current" file has +been moved into place to indicate that a new revision is present. It +is also locked during a revprop propchange while the revprop file is +read in, mutated, and written out again. Note that readers are never +blocked by any operation - writers must ensure that the filesystem is +always in a consistent state. + +The "txn-current" file is a file with a single line of text that +contains only a base-36 number. The current value will be used in the +next transaction name, along with the revision number the transaction +is based on. This sequence number ensures that transaction names are +not reused, even if the transaction is aborted and a new transaction +based on the same revision is begun. The only operation that FSFS +performs on this file is "get and increment"; the "txn-current-lock" +file is locked during this operation. + +"fsfs.conf" is a configuration file in the standard Subversion/Python +config format. It is automatically generated when you create a new +repository; read the generated file for details on what it controls. + +When representation sharing is enabled, the filesystem tracks +representation checksum and location mappings using a SQLite database in +"rep-cache.db". The database has a single table, which stores the sha1 +hash text as the primary key, mapped to the representation revision, offset, +size and expanded size. This file is only consulted during writes and never +during reads. Consequently, it is not required, and may be removed at an +abritrary time, with the subsequent loss of rep-sharing capabilities for +revisions written thereafter. + +Filesystem formats +------------------ + +The "format" file defines what features are permitted within the +filesystem, and indicates changes that are not backward-compatible. +It serves the same purpose as the repository file of the same name. + +The filesystem format file was introduced in Subversion 1.2, and so +will not be present if the repository was created with an older +version of Subversion. An absent format file should be interpreted as +indicating a format 1 filesystem. + +The format file is a single line of the form "<format number>\n", +followed by any number of lines specifying 'format options' - +additional information about the filesystem's format. Each format +option line is of the form "<option>\n" or "<option> <parameters>\n". + +Clients should raise an error if they encounter an option not +permitted by the format number in use. + +The formats are: + + Format 1, understood by Subversion 1.1+ + Format 2, understood by Subversion 1.4+ + Format 3, understood by Subversion 1.5+ + Format 4, understood by Subversion 1.6+ + Format 5, understood by Subversion 1.7-dev, never released + Format 6, understood by Subversion 1.8 + +The differences between the formats are: + +Delta representation in revision files + Format 1: svndiff0 only + Formats 2+: svndiff0 or svndiff1 + +Format options + Formats 1-2: none permitted + Format 3+: "layout" option + +Transaction name reuse + Formats 1-2: transaction names may be reused + Format 3+: transaction names generated using txn-current file + +Location of proto-rev file and its lock + Formats 1-2: transactions/<txnid>/rev and + transactions/<txnid>/rev-lock. + Format 3+: txn-protorevs/<txnid>.rev and + txn-protorevs/<txnid>.rev-lock. + +Node-ID and copy-ID generation + Formats 1-2: Node-IDs and copy-IDs are guaranteed to form a + monotonically increasing base36 sequence using the "current" + file. + Format 3+: Node-IDs and copy-IDs use the new revision number to + ensure uniqueness and the "current" file just contains the + youngest revision. + +Mergeinfo metadata: + Format 1-2: minfo-here and minfo-count node-revision fields are not + stored. svn_fs_get_mergeinfo returns an error. + Format 3+: minfo-here and minfo-count node-revision fields are + maintained. svn_fs_get_mergeinfo works. + +Revision changed paths list: + Format 1-3: Does not contain the node's kind. + Format 4+: Contains the node's kind. + +Shard packing: + Format 4: Applied to revision data only. + Format 5: Revprops would be packed independently of revision data. + Format 6+: Applied equally to revision data and revprop data + (i.e. same min packed revision) + +# Incomplete list. See SVN_FS_FS__MIN_*_FORMAT + + +Filesystem format options +------------------------- + +Currently, the only recognised format option is "layout", which +specifies the paths that will be used to store the revision files and +revision property files. + +The "layout" option is followed by the name of the filesystem layout +and any required parameters. The default layout, if no "layout" +keyword is specified, is the 'linear' layout. + +The known layouts, and the parameters they require, are as follows: + +"linear" + Revision files and rev-prop files are named after the revision they + represent, and are placed directly in the revs/ and revprops/ + directories. r1234 will be represented by the revision file + revs/1234 and the rev-prop file revprops/1234. + +"sharded <max-files-per-directory>" + Revision files and rev-prop files are named after the revision they + represent, and are placed in a subdirectory of the revs/ and + revprops/ directories named according to the 'shard' they belong to. + + Shards are numbered from zero and contain between one and the + maximum number of files per directory specified in the layout's + parameters. + + For the "sharded 1000" layout, r1234 will be represented by the + revision file revs/1/1234 and rev-prop file revprops/1/1234. The + revs/0/ directory will contain revisions 0-999, revs/1/ will contain + 1000-1999, and so on. + +Packing revisions +----------------- + +A filesystem can optionally be "packed" to conserve space on disk. The +packing process concatenates all the revision files in each full shard to +create pack files. A manifest file is also created for each shard which +records the indexes of the corresponding revision files in the pack file. +In addition, the original shard is removed, and reads are redirected to the +pack file. + +The manifest file consists of a list of offsets, one for each revision in the +pack file. The offsets are stored as ASCII decimal, and separated by a newline +character. + +Packing revision properties (format 5: SQLite) +--------------------------- + +This was supported by 1.7-dev builds but never included in a blessed release. + +See r1143829 of this file: +http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/structure?view=markup&pathrev=1143829 + + +Packing revision properties (format 6+) +--------------------------- + +Similarly to the revision data, packing will concatenate multiple +revprops into a single file. Since they are mutable data, we put an +upper limit to the size of these files: We will concatenate the data +up to the limit and then use a new file for the following revisions. + +The limit can be set and changed at will in the configuration file. +It is 64kB by default. Because a pack file must contain at least one +complete property list, files containing just one revision may exceed +that limit. + +Furthermore, pack files can be compressed which saves about 75% of +disk space. A configuration file flag enables the compression; it is +off by default and may be switched on and off at will. The pack size +limit is always applied to the uncompressed data. For this reason, +the default is 256kB while compression has been enabled. + +Files are named after their start revision as "<rev>.<counter>" where +counter will be increased whenever we rewrite a pack file due to a +revprop change. The manifest file contains the list of pack file +names, one line for each revision. + +Many tools track repository global data in revision properties at +revision 0. To minimize I/O overhead for those applications, we +will never pack that revision, i.e. its data is always being kept +in revprops/0/0. + +Pack file format + + Top level: <packed container> + + We always apply data compression to the pack file - using the + SVN_DELTA_COMPRESSION_LEVEL_NONE level if compression is disabled. + (Note that compression at SVN_DELTA_COMPRESSION_LEVEL_NONE is not + a no-op stream transformation although most of the data will remain + human readable.) + + container := header '\n' (revprops)+ + header := start_rev '\n' rev_count '\n' (size '\n')+ + + All numbers in the header are given as ASCII decimals. rev_count + is the number of revisions packed into this container. There must + be exactly as many "size" and serialized "revprops". The "size" + values in the list are the length in bytes of the serialized + revprops of the respective revision. + +Writing to packed revprops + + The old pack file is being read and the new revprops serialized. + If they fit into the same pack file, a temp file with the new + content gets written and moved into place just like an non-packed + revprop file would. No name change or manifest update required. + + If they don't fit into the same pack file, i.e. exceed the pack + size limit, the pack will be split into 2 or 3 new packs just + before and / or after the modified revision. + + In the current implementation, they will never be merged again. + To minimize fragmentation, the initial packing process will only + use about 90% of the limit, i.e. leave some room for growth. + + When a pack file gets split, its counter is being increased + creating a new file and leaving the old content in place and + available for concurrent readers. Only after the new manifest + file got moved into place, will the old pack files be deleted. + + Write access to revprops is being serialized by the global + filesystem write lock. We only need to build a few retries into + the reader code to gracefully handle manifest changes and pack + file deletions. + + +Node-revision IDs +----------------- + +A node-rev ID consists of the following three fields: + + node_revision_id ::= node_id '.' copy_id '.' txn_id + +At this level, the form of the ID is the same as for BDB - see the +section called "ID's" in <../libsvn_fs_base/notes/structure>. + +In order to support efficient lookup of node-revisions by their IDs +and to simplify the allocation of fresh node-IDs during a transaction, +we treat the fields of a node-rev ID in new and interesting ways. + +Within a new transaction: + + New node-revision IDs assigned within a transaction have a txn-id + field of the form "t<txnid>". + + When a new node-id or copy-id is assigned in a transaction, the ID + used is a "_" followed by a base36 number unique to the transaction. + +Within a revision: + + Within a revision file, node-revs have a txn-id field of the form + "r<rev>/<offset>", to support easy lookup. The <offset> is the (ASCII + decimal) number of bytes from the start of the revision file to the + start of the node-rev. + + During the final phase of a commit, node-revision IDs are rewritten + to have repository-wide unique node-ID and copy-ID fields, and to have + "r<rev>/<offset>" txn-id fields. + + In Format 3 and above, this uniqueness is done by changing a temporary + id of "_<base36>" to "<base36>-<rev>". Note that this means that the + originating revision of a line of history or a copy can be determined + by looking at the node ID. + + In Format 2 and below, the "current" file contains global base36 + node-ID and copy-ID counters; during the commit, the counter value is + added to the transaction-specific base36 ID, and the value in + "current" is adjusted. + + (It is legal for Format 3 repositories to contain Format 2-style IDs; + this just prevents I/O-less node-origin-rev lookup for those nodes.) + +The temporary assignment of node-ID and copy-ID fields has +implications for svn_fs_compare_ids and svn_fs_check_related. The ID +_1.0.t1 is not related to the ID _1.0.t2 even though they have the +same node-ID, because temporary node-IDs are restricted in scope to +the transactions they belong to. + +There is a lazily created cache mapping from node-IDs to the full +node-revision ID where they are created. This is in the node-origins +directory; the file name is the node-ID without its last character (or +"0" for single-character node IDs) and the contents is a serialized +hash mapping from node-ID to node-revision ID. This cache is only +used for node-IDs of the pre-Format 3 style. + +Copy-IDs and copy roots +----------------------- + +Copy-IDs are assigned in the same manner as they are in the BDB +implementation: + + * A node-rev resulting from a creation operation (with no copy + history) receives the copy-ID of its parent directory. + + * A node-rev resulting from a copy operation receives a fresh + copy-ID, as one would expect. + + * A node-rev resulting from a modification operation receives a + copy-ID depending on whether its predecessor derives from a + copy operation or whether it derives from a creation operation + with no intervening copies: + + - If the predecessor does not derive from a copy, the new + node-rev receives the copy-ID of its parent directory. If the + node-rev is being modified through its created-path, this will + be the same copy-ID as the predecessor node-rev has; however, + if the node-rev is being modified through a copied ancestor + directory (i.e. we are performing a "lazy copy"), this will be + a different copy-ID. + + - If the predecessor derives from a copy and the node-rev is + being modified through its created-path, the new node-rev + receives the copy-ID of the predecessor. + + - If the predecessor derives from a copy and the node-rev is not + being modified through its created path, the new node-rev + receives a fresh copy-ID. This is called a "soft copy" + operation, as distinct from a "true copy" operation which was + actually requested through the svn_fs interface. Soft copies + exist to ensure that the same <node-ID,copy-ID> pair is not + used twice within a transaction. + +Unlike the BDB implementation, we do not have a "copies" table. +Instead, each node-revision record contains a "copyroot" field +identifying the node-rev resulting from the true copy operation most +proximal to the node-rev. If the node-rev does not itself derive from +a copy operation, then the copyroot field identifies the copy of an +ancestor directory; if no ancestor directories derive from a copy +operation, then the copyroot field identifies the root directory of +rev 0. + +Revision file format +-------------------- + +A revision file contains a concatenation of various kinds of data: + + * Text and property representations + * Node-revisions + * The changed-path data + * Two offsets at the very end + +A representation begins with a line containing either "PLAIN\n" or +"DELTA\n" or "DELTA <rev> <offset> <length>\n", where <rev>, <offset>, +and <length> give the location of the delta base of the representation +and the amount of data it contains (not counting the header or +trailer). If no base location is given for a delta, the base is the +empty stream. After the initial line comes raw svndiff data, followed +by a cosmetic trailer "ENDREP\n". + +If the representation is for the text contents of a directory node, +the expanded contents are in hash dump format mapping entry names to +"<type> <id>" pairs, where <type> is "file" or "dir" and <id> gives +the ID of the child node-rev. + +If a representation is for a property list, the expanded contents are +in the form of a dumped hash map mapping property names to property +values. + +The marshalling syntax for node-revs is a series of fields terminated +by a blank line. Fields have the syntax "<name>: <value>\n", where +<name> is a symbolic field name (each symbolic name is used only once +in a given node-rev) and <value> is the value data. Unrecognized +fields are ignored, for extensibility. The following fields are +defined: + + id The ID of the node-rev + type "file" or "dir" + pred The ID of the predecessor node-rev + count Count of node-revs since the base of the node + text "<rev> <offset> <length> <size> <digest>" for text rep + props "<rev> <offset> <length> <size> <digest>" for props rep + <rev> and <offset> give location of rep + <length> gives length of rep, sans header and trailer + <size> gives size of expanded rep; may be 0 if equal + to the length + <digest> gives hex MD5 digest of expanded rep + ### in formats >=4, also present: + <sha1-digest> gives hex SHA1 digest of expanded rep + <uniquifier> see representation_t->uniquifier in fs.h + cpath FS pathname node was created at + copyfrom "<rev> <path>" of copyfrom data + copyroot "<rev> <created-path>" of the root of this copy + minfo-cnt The number of nodes under (and including) this node + which have svn:mergeinfo. + minfo-here Exists if this node itself has svn:mergeinfo. + +The predecessor of a node-rev crosses both soft and true copies; +together with the count field, it allows efficient determination of +the base for skip-deltas. The first node-rev of a node contains no +"pred" field. A node-revision with no properties may omit the "props" +field. A node-revision with no contents (a zero-length file or an +empty directory) may omit the "text" field. In a node-revision +resulting from a true copy operation, the "copyfrom" field gives the +copyfrom data. The "copyroot" field identifies the root node-revision +of the copy; it may be omitted if the node-rev is its own copy root +(as is the case for node-revs with copy history, and for the root node +of revision 0). Copy roots are identified by revision and +created-path, not by node-rev ID, because a copy root may be a +node-rev which exists later on within the same revision file, meaning +its offset is not yet known. + +The changed-path data is represented as a series of changed-path +items, each consisting of two lines. The first line has the format +"<id> <action> <text-mod> <prop-mod> <path>\n", where <id> is the +node-rev ID of the new node-rev, <action> is "add", "delete", +"replace", or "modify", <text-mod> and <prop-mod> are "true" or +"false" indicating whether the text and/or properties changed, and +<path> is the changed pathname. For deletes, <id> is the node-rev ID +of the deleted node-rev, and <text-mod> and <prop-mod> are always +"false". The second line has the format "<rev> <path>\n" containing +the node-rev's copyfrom information if it has any; if it does not, the +second line is blank. + +Starting with FS format 4, <action> may contain the kind ("file" or +"dir") of the node, after a hyphen; for example, an added directory +may be represented as "add-dir". + +At the very end of a rev file is a pair of lines containing +"\n<root-offset> <cp-offset>\n", where <root-offset> is the offset of +the root directory node revision and <cp-offset> is the offset of the +changed-path data. + +All numbers in the rev file format are unsigned and are represented as +ASCII decimal. + +Transaction layout +------------------ + +A transaction directory has the following layout: + + props Transaction props + next-ids Next temporary node-ID and copy-ID + changes Changed-path information so far + node.<nid>.<cid> New node-rev data for node + node.<nid>.<cid>.props Props for new node-rev, if changed + node.<nid>.<cid>.children Directory contents for node-rev + <sha1> Text representation of that sha1 + +In FS formats 1 and 2, it also contains: + + rev Prototype rev file with new text reps + rev-lock Lockfile for writing to the above + +In newer formats, these files are in the txn-protorevs/ directory. + +The prototype rev file is used to store the text representations as +they are received from the client. To ensure that only one client is +writing to the file at a given time, the "rev-lock" file is locked for +the duration of each write. + +The two kinds of props files are all in hash dump format. The "props" +file will always be present. The "node.<nid>.<cid>.props" file will +only be present if the node-rev properties have been changed. + +The <sha1> files have been introduced in FS format 6. Their content +is that of text rep references: "<rev> <offset> <length> <size> <digest>" +They will be written for text reps in the current transaction and be +used to eliminate duplicate reps within that transaction. + +The "next-ids" file contains a single line "<next-temp-node-id> +<next-temp-copy-id>\n" giving the next temporary node-ID and copy-ID +assignments (without the leading underscores). The next node-ID is +also used as a uniquifier for representations which may share the same +underlying rep. + +The "children" file for a node-revision begins with a copy of the hash +dump representation of the directory entries from the old node-rev (or +a dump of the empty hash for new directories), and then an incremental +hash dump entry for each change made to the directory. + +The "changes" file contains changed-path entries in the same form as +the changed-path entries in a rev file, except that <id> and <action> +may both be "reset" (in which case <text-mod> and <prop-mod> are both +always "false") to indicate that all changes to a path should be +considered undone. Reset entries are only used during the final merge +phase of a transaction. Actions in the "changes" file always contain +a node kind, even if the FS format is older than format 4. + +The node-rev files have the same format as node-revs in a revision +file, except that the "text" and "props" fields are augmented as +follows: + + * The "props" field may have the value "-1" if properties have + been changed and are contained in a "props" file within the + node-rev subdirectory. + + * For directory node-revs, the "text" field may have the value + "-1" if entries have been changed and are contained in a + "contents" file in the node-rev subdirectory. + + * For the directory node-rev representing the root of the + transaction, the "is-fresh-txn-root" field indicates that it has + not been made mutable yet (see Issue #2608). + + * For file node-revs, the "text" field may have the value "-1 + <offset> <length> <size> <digest>" if the text representation is + within the prototype rev file. + + * The "copyroot" field may have the value "-1 <created-path>" if the + copy root of the node-rev is part of the transaction in process. + +Locks layout +------------ + +Locks in FSFS are stored in serialized hash format in files whose +names are MD5 digests of the FS path which the lock is associated +with. For the purposes of keeping directory inode usage down, these +digest files live in subdirectories of the main lock directory whose +names are the first 3 characters of the digest filename. + +Also stored in the digest file for a given FS path are pointers to +other digest files which contain information associated with other FS +paths that are beneath our path (an immediate child thereof, or a +grandchild, or a great-grandchild, ...). + +To answer the question, "Does path FOO have a lock associated with +it?", one need only generate the MD5 digest of FOO's +absolute-in-the-FS path (say, 3b1b011fed614a263986b5c4869604e8), look +for a file located like so: + + /path/to/repos/locks/3b1/3b1b011fed614a263986b5c4869604e8 + +And then see if that file contains lock information. + +To inquire about locks on children of the path FOO, you would +reference the same path as above, but look for a list of children in +that file (instead of lock information). Children are listed as MD5 +digests, too, so you would simply iterate over those digests and +consult the files they reference for lock information.
diff --git a/subversion/libsvn_fs_x/temp_serializer.c b/subversion/libsvn_fs_x/temp_serializer.c new file mode 100644 index 0000000..efa72d6 --- /dev/null +++ b/subversion/libsvn_fs_x/temp_serializer.c
@@ -0,0 +1,1334 @@ +/* temp_serializer.c: serialization functions for caching of FSX structures + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include <apr_pools.h> + +#include "svn_pools.h" +#include "svn_hash.h" + +#include "id.h" +#include "svn_fs.h" + +#include "private/svn_fs_util.h" +#include "private/svn_temp_serializer.h" +#include "private/svn_subr_private.h" + +#include "temp_serializer.h" +#include "low_level.h" + +/* Utility to encode a signed NUMBER into a variable-length sequence of + * 8-bit chars in KEY_BUFFER and return the last writen position. + * + * Numbers will be stored in 7 bits / byte and using byte values above + * 32 (' ') to make them combinable with other string by simply separating + * individual parts with spaces. + */ +static char* +encode_number(apr_int64_t number, char *key_buffer) +{ + /* encode the sign in the first byte */ + if (number < 0) + { + number = -number; + *key_buffer = (char)((number & 63) + ' ' + 65); + } + else + *key_buffer = (char)((number & 63) + ' ' + 1); + number /= 64; + + /* write 7 bits / byte until no significant bits are left */ + while (number) + { + *++key_buffer = (char)((number & 127) + ' ' + 1); + number /= 128; + } + + /* return the last written position */ + return key_buffer; +} + +const char* +svn_fs_x__combine_number_and_string(apr_int64_t number, + const char *string, + apr_pool_t *pool) +{ + apr_size_t len = strlen(string); + + /* number part requires max. 10x7 bits + 1 space. + * Add another 1 for the terminal 0 */ + char *key_buffer = apr_palloc(pool, len + 12); + const char *key = key_buffer; + + /* Prepend the number to the string and separate them by space. No other + * number can result in the same prefix, no other string in the same + * postfix nor can the boundary between them be ambiguous. */ + key_buffer = encode_number(number, key_buffer); + *++key_buffer = ' '; + memcpy(++key_buffer, string, len+1); + + /* return the start of the key */ + return key; +} + +/* Utility function to serialize string S in the given serialization CONTEXT. + */ +static void +serialize_svn_string(svn_temp_serializer__context_t *context, + const svn_string_t * const *s) +{ + const svn_string_t *string = *s; + + /* Nothing to do for NULL string references. */ + if (string == NULL) + return; + + svn_temp_serializer__push(context, + (const void * const *)s, + sizeof(*string)); + + /* the "string" content may actually be arbitrary binary data. + * Thus, we cannot use svn_temp_serializer__add_string. */ + svn_temp_serializer__add_leaf(context, + (const void * const *)&string->data, + string->len + 1); + + /* back to the caller's nesting level */ + svn_temp_serializer__pop(context); +} + +/* Utility function to deserialize the STRING inside the BUFFER. + */ +static void +deserialize_svn_string(void *buffer, svn_string_t **string) +{ + svn_temp_deserializer__resolve(buffer, (void **)string); + if (*string == NULL) + return; + + svn_temp_deserializer__resolve(*string, (void **)&(*string)->data); +} + +/* Utility function to serialize the REPRESENTATION within the given + * serialization CONTEXT. + */ +static void +serialize_representation(svn_temp_serializer__context_t *context, + representation_t * const *representation) +{ + const representation_t * rep = *representation; + if (rep == NULL) + return; + + /* serialize the representation struct itself */ + svn_temp_serializer__add_leaf(context, + (const void * const *)representation, + sizeof(*rep)); +} + +void +svn_fs_x__serialize_apr_array(svn_temp_serializer__context_t *context, + apr_array_header_t **a) +{ + const apr_array_header_t *array = *a; + + /* Nothing to do for NULL string references. */ + if (array == NULL) + return; + + /* array header struct */ + svn_temp_serializer__push(context, + (const void * const *)a, + sizeof(*array)); + + /* contents */ + svn_temp_serializer__add_leaf(context, + (const void * const *)&array->elts, + (apr_size_t)array->nelts * array->elt_size); + + /* back to the caller's nesting level */ + svn_temp_serializer__pop(context); +} + +void +svn_fs_x__deserialize_apr_array(void *buffer, + apr_array_header_t **array, + apr_pool_t *pool) +{ + svn_temp_deserializer__resolve(buffer, (void **)array); + if (*array == NULL) + return; + + svn_temp_deserializer__resolve(*array, (void **)&(*array)->elts); + (*array)->pool = pool; +} + +/* auxilliary structure representing the content of a directory hash */ +typedef struct hash_data_t +{ + /* number of entries in the directory */ + apr_size_t count; + + /* number of unused dir entry buckets in the index */ + apr_size_t over_provision; + + /* internal modifying operations counter + * (used to repack data once in a while) */ + apr_size_t operations; + + /* size of the serialization buffer actually used. + * (we will allocate more than we actually need such that we may + * append more data in situ later) */ + apr_size_t len; + + /* reference to the entries */ + svn_fs_dirent_t **entries; + + /* size of the serialized entries and don't be too wasteful + * (needed since the entries are no longer in sequence) */ + apr_uint32_t *lengths; +} hash_data_t; + +static int +compare_dirent_id_names(const void *lhs, const void *rhs) +{ + return strcmp((*(const svn_fs_dirent_t *const *)lhs)->name, + (*(const svn_fs_dirent_t *const *)rhs)->name); +} + +/* Utility function to serialize the *ENTRY_P into a the given + * serialization CONTEXT. Return the serialized size of the + * dir entry in *LENGTH. + */ +static void +serialize_dir_entry(svn_temp_serializer__context_t *context, + svn_fs_dirent_t **entry_p, + apr_uint32_t *length) +{ + svn_fs_dirent_t *entry = *entry_p; + apr_size_t initial_length = svn_temp_serializer__get_length(context); + + svn_temp_serializer__push(context, + (const void * const *)entry_p, + sizeof(svn_fs_dirent_t)); + + svn_fs_x__id_serialize(context, &entry->id); + svn_temp_serializer__add_string(context, &entry->name); + + *length = (apr_uint32_t)( svn_temp_serializer__get_length(context) + - APR_ALIGN_DEFAULT(initial_length)); + + svn_temp_serializer__pop(context); +} + +/* Utility function to serialize the ENTRIES into a new serialization + * context to be returned. Allocation will be made form POOL. + */ +static svn_temp_serializer__context_t * +serialize_dir(apr_hash_t *entries, apr_pool_t *pool) +{ + hash_data_t hash_data; + apr_hash_index_t *hi; + apr_size_t i = 0; + svn_temp_serializer__context_t *context; + + /* calculate sizes */ + apr_size_t count = apr_hash_count(entries); + apr_size_t over_provision = 2 + count / 4; + apr_size_t entries_len = (count + over_provision) * sizeof(svn_fs_dirent_t*); + apr_size_t lengths_len = (count + over_provision) * sizeof(apr_uint32_t); + + /* copy the hash entries to an auxilliary struct of known layout */ + hash_data.count = count; + hash_data.over_provision = over_provision; + hash_data.operations = 0; + hash_data.entries = apr_palloc(pool, entries_len); + hash_data.lengths = apr_palloc(pool, lengths_len); + + for (hi = apr_hash_first(pool, entries); hi; hi = apr_hash_next(hi), ++i) + hash_data.entries[i] = svn__apr_hash_index_val(hi); + + /* sort entry index by ID name */ + qsort(hash_data.entries, + count, + sizeof(*hash_data.entries), + compare_dirent_id_names); + + /* Serialize that aux. structure into a new one. Also, provide a good + * estimate for the size of the buffer that we will need. */ + context = svn_temp_serializer__init(&hash_data, + sizeof(hash_data), + 50 + count * 200 + entries_len, + pool); + + /* serialize entries references */ + svn_temp_serializer__push(context, + (const void * const *)&hash_data.entries, + entries_len); + + /* serialize the individual entries and their sub-structures */ + for (i = 0; i < count; ++i) + serialize_dir_entry(context, + &hash_data.entries[i], + &hash_data.lengths[i]); + + svn_temp_serializer__pop(context); + + /* serialize entries references */ + svn_temp_serializer__push(context, + (const void * const *)&hash_data.lengths, + lengths_len); + + return context; +} + +/* Utility function to reconstruct a dir entries hash from serialized data + * in BUFFER and HASH_DATA. Allocation will be made form POOL. + */ +static apr_hash_t * +deserialize_dir(void *buffer, hash_data_t *hash_data, apr_pool_t *pool) +{ + apr_hash_t *result = svn_hash__make(pool); + apr_size_t i; + apr_size_t count; + svn_fs_dirent_t *entry; + svn_fs_dirent_t **entries; + + /* resolve the reference to the entries array */ + svn_temp_deserializer__resolve(buffer, (void **)&hash_data->entries); + entries = hash_data->entries; + + /* fixup the references within each entry and add it to the hash */ + for (i = 0, count = hash_data->count; i < count; ++i) + { + svn_temp_deserializer__resolve(entries, (void **)&entries[i]); + entry = hash_data->entries[i]; + + /* pointer fixup */ + svn_temp_deserializer__resolve(entry, (void **)&entry->name); + svn_fs_x__id_deserialize(entry, (svn_fs_id_t **)&entry->id); + + /* add the entry to the hash */ + svn_hash_sets(result, entry->name, entry); + } + + /* return the now complete hash */ + return result; +} + +void +svn_fs_x__noderev_serialize(svn_temp_serializer__context_t *context, + node_revision_t * const *noderev_p) +{ + const node_revision_t *noderev = *noderev_p; + if (noderev == NULL) + return; + + /* serialize the representation struct itself */ + svn_temp_serializer__push(context, + (const void * const *)noderev_p, + sizeof(*noderev)); + + /* serialize sub-structures */ + svn_fs_x__id_serialize(context, &noderev->id); + svn_fs_x__id_serialize(context, &noderev->predecessor_id); + serialize_representation(context, &noderev->prop_rep); + serialize_representation(context, &noderev->data_rep); + + svn_temp_serializer__add_string(context, &noderev->copyfrom_path); + svn_temp_serializer__add_string(context, &noderev->copyroot_path); + svn_temp_serializer__add_string(context, &noderev->created_path); + + /* return to the caller's nesting level */ + svn_temp_serializer__pop(context); +} + + +void +svn_fs_x__noderev_deserialize(void *buffer, + node_revision_t **noderev_p) +{ + node_revision_t *noderev; + + /* fixup the reference to the representation itself, + * if this is part of a parent structure. */ + if (buffer != *noderev_p) + svn_temp_deserializer__resolve(buffer, (void **)noderev_p); + + noderev = *noderev_p; + if (noderev == NULL) + return; + + /* fixup of sub-structures */ + svn_fs_x__id_deserialize(noderev, (svn_fs_id_t **)&noderev->id); + svn_fs_x__id_deserialize(noderev, (svn_fs_id_t **)&noderev->predecessor_id); + svn_temp_deserializer__resolve(noderev, (void **)&noderev->prop_rep); + svn_temp_deserializer__resolve(noderev, (void **)&noderev->data_rep); + + svn_temp_deserializer__resolve(noderev, (void **)&noderev->copyfrom_path); + svn_temp_deserializer__resolve(noderev, (void **)&noderev->copyroot_path); + svn_temp_deserializer__resolve(noderev, (void **)&noderev->created_path); +} + + +/* Utility function to serialize COUNT svn_txdelta_op_t objects + * at OPS in the given serialization CONTEXT. + */ +static void +serialize_txdelta_ops(svn_temp_serializer__context_t *context, + const svn_txdelta_op_t * const * ops, + apr_size_t count) +{ + if (*ops == NULL) + return; + + /* the ops form a contiguous chunk of memory with no further references */ + svn_temp_serializer__add_leaf(context, + (const void * const *)ops, + count * sizeof(svn_txdelta_op_t)); +} + +/* Utility function to serialize W in the given serialization CONTEXT. + */ +static void +serialize_txdeltawindow(svn_temp_serializer__context_t *context, + svn_txdelta_window_t * const * w) +{ + svn_txdelta_window_t *window = *w; + + /* serialize the window struct itself */ + svn_temp_serializer__push(context, + (const void * const *)w, + sizeof(svn_txdelta_window_t)); + + /* serialize its sub-structures */ + serialize_txdelta_ops(context, &window->ops, window->num_ops); + serialize_svn_string(context, &window->new_data); + + svn_temp_serializer__pop(context); +} + +svn_error_t * +svn_fs_x__serialize_txdelta_window(void **buffer, + apr_size_t *buffer_size, + void *item, + apr_pool_t *pool) +{ + svn_fs_x__txdelta_cached_window_t *window_info = item; + svn_stringbuf_t *serialized; + + /* initialize the serialization process and allocate a buffer large + * enough to do without the need of re-allocations in most cases. */ + apr_size_t text_len = window_info->window->new_data + ? window_info->window->new_data->len + : 0; + svn_temp_serializer__context_t *context = + svn_temp_serializer__init(window_info, + sizeof(*window_info), + 500 + text_len, + pool); + + /* serialize the sub-structure(s) */ + serialize_txdeltawindow(context, &window_info->window); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *buffer = serialized->data; + *buffer_size = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_txdelta_window(void **item, + void *buffer, + apr_size_t buffer_size, + apr_pool_t *pool) +{ + svn_txdelta_window_t *window; + + /* Copy the _full_ buffer as it also contains the sub-structures. */ + svn_fs_x__txdelta_cached_window_t *window_info = + (svn_fs_x__txdelta_cached_window_t *)buffer; + + /* pointer reference fixup */ + svn_temp_deserializer__resolve(window_info, + (void **)&window_info->window); + window = window_info->window; + + svn_temp_deserializer__resolve(window, (void **)&window->ops); + + deserialize_svn_string(window, (svn_string_t**)&window->new_data); + + /* done */ + *item = window_info; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__serialize_manifest(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + apr_array_header_t *manifest = in; + + *data_len = sizeof(apr_off_t) *manifest->nelts; + *data = apr_palloc(pool, *data_len); + memcpy(*data, manifest->elts, *data_len); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_manifest(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + apr_array_header_t *manifest = apr_array_make(pool, 1, sizeof(apr_off_t)); + + manifest->nelts = (int) (data_len / sizeof(apr_off_t)); + manifest->nalloc = (int) (data_len / sizeof(apr_off_t)); + manifest->elts = (char*)data; + + *out = manifest; + + return SVN_NO_ERROR; +} + +/* Auxilliary structure representing the content of a properties hash. + This structure is much easier to (de-)serialize than an apr_hash. + */ +typedef struct properties_data_t +{ + /* number of entries in the hash */ + apr_size_t count; + + /* reference to the keys */ + const char **keys; + + /* reference to the values */ + const svn_string_t **values; +} properties_data_t; + +/* Serialize COUNT C-style strings from *STRINGS into CONTEXT. */ +static void +serialize_cstring_array(svn_temp_serializer__context_t *context, + const char ***strings, + apr_size_t count) +{ + apr_size_t i; + const char **entries = *strings; + + /* serialize COUNT entries pointers (the array) */ + svn_temp_serializer__push(context, + (const void * const *)strings, + count * sizeof(const char*)); + + /* serialize array elements */ + for (i = 0; i < count; ++i) + svn_temp_serializer__add_string(context, &entries[i]); + + svn_temp_serializer__pop(context); +} + +/* Serialize COUNT svn_string_t* items from *STRINGS into CONTEXT. */ +static void +serialize_svn_string_array(svn_temp_serializer__context_t *context, + const svn_string_t ***strings, + apr_size_t count) +{ + apr_size_t i; + const svn_string_t **entries = *strings; + + /* serialize COUNT entries pointers (the array) */ + svn_temp_serializer__push(context, + (const void * const *)strings, + count * sizeof(const char*)); + + /* serialize array elements */ + for (i = 0; i < count; ++i) + serialize_svn_string(context, &entries[i]); + + svn_temp_serializer__pop(context); +} + +svn_error_t * +svn_fs_x__serialize_properties(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + apr_hash_t *hash = in; + properties_data_t properties; + svn_temp_serializer__context_t *context; + apr_hash_index_t *hi; + svn_stringbuf_t *serialized; + apr_size_t i; + + /* create our auxilliary data structure */ + properties.count = apr_hash_count(hash); + properties.keys = apr_palloc(pool, sizeof(const char*) * (properties.count + 1)); + properties.values = apr_palloc(pool, sizeof(const char*) * properties.count); + + /* populate it with the hash entries */ + for (hi = apr_hash_first(pool, hash), i=0; hi; hi = apr_hash_next(hi), ++i) + { + properties.keys[i] = svn__apr_hash_index_key(hi); + properties.values[i] = svn__apr_hash_index_val(hi); + } + + /* serialize it */ + context = svn_temp_serializer__init(&properties, + sizeof(properties), + properties.count * 100, + pool); + + properties.keys[i] = ""; + serialize_cstring_array(context, &properties.keys, properties.count + 1); + serialize_svn_string_array(context, &properties.values, properties.count); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_properties(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + apr_hash_t *hash = svn_hash__make(pool); + properties_data_t *properties = (properties_data_t *)data; + size_t i; + + /* de-serialize our auxilliary data structure */ + svn_temp_deserializer__resolve(properties, (void**)&properties->keys); + svn_temp_deserializer__resolve(properties, (void**)&properties->values); + + /* de-serialize each entry and put it into the hash */ + for (i = 0; i < properties->count; ++i) + { + apr_size_t len = properties->keys[i+1] - properties->keys[i] - 1; + svn_temp_deserializer__resolve((void*)properties->keys, + (void**)&properties->keys[i]); + + deserialize_svn_string((void*)properties->values, + (svn_string_t **)&properties->values[i]); + + apr_hash_set(hash, + properties->keys[i], len, + properties->values[i]); + } + + /* done */ + *out = hash; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__serialize_id(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + const svn_fs_id_t *id = in; + svn_stringbuf_t *serialized; + + /* create an (empty) serialization context with plenty of buffer space */ + svn_temp_serializer__context_t *context = + svn_temp_serializer__init(NULL, 0, 250, pool); + + /* serialize the id */ + svn_fs_x__id_serialize(context, &id); + + /* return serialized data */ + serialized = svn_temp_serializer__get(context); + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_id(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + /* Copy the _full_ buffer as it also contains the sub-structures. */ + svn_fs_id_t *id = (svn_fs_id_t *)data; + + /* fixup of all pointers etc. */ + svn_fs_x__id_deserialize(id, &id); + + /* done */ + *out = id; + return SVN_NO_ERROR; +} + +/** Caching node_revision_t objects. **/ + +svn_error_t * +svn_fs_x__serialize_node_revision(void **buffer, + apr_size_t *buffer_size, + void *item, + apr_pool_t *pool) +{ + svn_stringbuf_t *serialized; + node_revision_t *noderev = item; + + /* create an (empty) serialization context with plenty of (initial) + * buffer space. */ + svn_temp_serializer__context_t *context = + svn_temp_serializer__init(NULL, 0, + 1024 - SVN_TEMP_SERIALIZER__OVERHEAD, + pool); + + /* serialize the noderev */ + svn_fs_x__noderev_serialize(context, &noderev); + + /* return serialized data */ + serialized = svn_temp_serializer__get(context); + *buffer = serialized->data; + *buffer_size = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_node_revision(void **item, + void *buffer, + apr_size_t buffer_size, + apr_pool_t *pool) +{ + /* Copy the _full_ buffer as it also contains the sub-structures. */ + node_revision_t *noderev = (node_revision_t *)buffer; + + /* fixup of all pointers etc. */ + svn_fs_x__noderev_deserialize(noderev, &noderev); + + /* done */ + *item = noderev; + return SVN_NO_ERROR; +} + +/* Utility function that returns the directory serialized inside CONTEXT + * to DATA and DATA_LEN. */ +static svn_error_t * +return_serialized_dir_context(svn_temp_serializer__context_t *context, + void **data, + apr_size_t *data_len) +{ + svn_stringbuf_t *serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->blocksize; + ((hash_data_t *)serialized->data)->len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__serialize_dir_entries(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + apr_hash_t *dir = in; + + /* serialize the dir content into a new serialization context + * and return the serialized data */ + return return_serialized_dir_context(serialize_dir(dir, pool), + data, + data_len); +} + +svn_error_t * +svn_fs_x__deserialize_dir_entries(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + /* Copy the _full_ buffer as it also contains the sub-structures. */ + hash_data_t *hash_data = (hash_data_t *)data; + + /* reconstruct the hash from the serialized data */ + *out = deserialize_dir(hash_data, hash_data, pool); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__get_sharded_offset(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool) +{ + const apr_off_t *manifest = data; + apr_int64_t shard_pos = *(apr_int64_t *)baton; + + *(apr_off_t *)out = manifest[shard_pos]; + + return SVN_NO_ERROR; +} + +/* Utility function that returns the lowest index of the first entry in + * *ENTRIES that points to a dir entry with a name equal or larger than NAME. + * If an exact match has been found, *FOUND will be set to TRUE. COUNT is + * the number of valid entries in ENTRIES. + */ +static apr_size_t +find_entry(svn_fs_dirent_t **entries, + const char *name, + apr_size_t count, + svn_boolean_t *found) +{ + /* binary search for the desired entry by name */ + apr_size_t lower = 0; + apr_size_t upper = count; + apr_size_t middle; + + for (middle = upper / 2; lower < upper; middle = (upper + lower) / 2) + { + const svn_fs_dirent_t *entry = + svn_temp_deserializer__ptr(entries, (const void *const *)&entries[middle]); + const char* entry_name = + svn_temp_deserializer__ptr(entry, (const void *const *)&entry->name); + + int diff = strcmp(entry_name, name); + if (diff < 0) + lower = middle + 1; + else + upper = middle; + } + + /* check whether we actually found a match */ + *found = FALSE; + if (lower < count) + { + const svn_fs_dirent_t *entry = + svn_temp_deserializer__ptr(entries, (const void *const *)&entries[lower]); + const char* entry_name = + svn_temp_deserializer__ptr(entry, (const void *const *)&entry->name); + + if (strcmp(entry_name, name) == 0) + *found = TRUE; + } + + return lower; +} + +svn_error_t * +svn_fs_x__extract_dir_entry(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool) +{ + const hash_data_t *hash_data = data; + const char* name = baton; + svn_boolean_t found; + + /* resolve the reference to the entries array */ + const svn_fs_dirent_t * const *entries = + svn_temp_deserializer__ptr(data, (const void *const *)&hash_data->entries); + + /* resolve the reference to the lengths array */ + const apr_uint32_t *lengths = + svn_temp_deserializer__ptr(data, (const void *const *)&hash_data->lengths); + + /* binary search for the desired entry by name */ + apr_size_t pos = find_entry((svn_fs_dirent_t **)entries, + name, + hash_data->count, + &found); + + /* de-serialize that entry or return NULL, if no match has been found */ + *out = NULL; + if (found) + { + const svn_fs_dirent_t *source = + svn_temp_deserializer__ptr(entries, (const void *const *)&entries[pos]); + + /* Entries have been serialized one-by-one, each time including all + * nested structures and strings. Therefore, they occupy a single + * block of memory whose end-offset is either the beginning of the + * next entry or the end of the buffer + */ + apr_size_t size = lengths[pos]; + + /* copy & deserialize the entry */ + svn_fs_dirent_t *new_entry = apr_palloc(pool, size); + memcpy(new_entry, source, size); + + svn_temp_deserializer__resolve(new_entry, (void **)&new_entry->name); + svn_fs_x__id_deserialize(new_entry, (svn_fs_id_t **)&new_entry->id); + *(svn_fs_dirent_t **)out = new_entry; + } + + return SVN_NO_ERROR; +} + +/* Utility function for svn_fs_x__replace_dir_entry that implements the + * modification as a simply deserialize / modify / serialize sequence. + */ +static svn_error_t * +slowly_replace_dir_entry(void **data, + apr_size_t *data_len, + void *baton, + apr_pool_t *pool) +{ + replace_baton_t *replace_baton = (replace_baton_t *)baton; + hash_data_t *hash_data = (hash_data_t *)*data; + apr_hash_t *dir; + + SVN_ERR(svn_fs_x__deserialize_dir_entries((void **)&dir, + *data, + hash_data->len, + pool)); + svn_hash_sets(dir, replace_baton->name, replace_baton->new_entry); + + return svn_fs_x__serialize_dir_entries(data, data_len, dir, pool); +} + +svn_error_t * +svn_fs_x__replace_dir_entry(void **data, + apr_size_t *data_len, + void *baton, + apr_pool_t *pool) +{ + replace_baton_t *replace_baton = (replace_baton_t *)baton; + hash_data_t *hash_data = (hash_data_t *)*data; + svn_boolean_t found; + svn_fs_dirent_t **entries; + apr_uint32_t *lengths; + apr_uint32_t length; + apr_size_t pos; + + svn_temp_serializer__context_t *context; + + /* after quite a number of operations, let's re-pack everything. + * This is to limit the number of vasted space as we cannot overwrite + * existing data but must always append. */ + if (hash_data->operations > 2 + hash_data->count / 4) + return slowly_replace_dir_entry(data, data_len, baton, pool); + + /* resolve the reference to the entries array */ + entries = (svn_fs_dirent_t **) + svn_temp_deserializer__ptr((const char *)hash_data, + (const void *const *)&hash_data->entries); + + /* resolve the reference to the lengths array */ + lengths = (apr_uint32_t *) + svn_temp_deserializer__ptr((const char *)hash_data, + (const void *const *)&hash_data->lengths); + + /* binary search for the desired entry by name */ + pos = find_entry(entries, replace_baton->name, hash_data->count, &found); + + /* handle entry removal (if found at all) */ + if (replace_baton->new_entry == NULL) + { + if (found) + { + /* remove reference to the entry from the index */ + memmove(&entries[pos], + &entries[pos + 1], + sizeof(entries[pos]) * (hash_data->count - pos)); + memmove(&lengths[pos], + &lengths[pos + 1], + sizeof(lengths[pos]) * (hash_data->count - pos)); + + hash_data->count--; + hash_data->over_provision++; + hash_data->operations++; + } + + return SVN_NO_ERROR; + } + + /* if not found, prepare to insert the new entry */ + if (!found) + { + /* fallback to slow operation if there is no place left to insert an + * new entry to index. That will automatically give add some spare + * entries ("overprovision"). */ + if (hash_data->over_provision == 0) + return slowly_replace_dir_entry(data, data_len, baton, pool); + + /* make entries[index] available for pointing to the new entry */ + memmove(&entries[pos + 1], + &entries[pos], + sizeof(entries[pos]) * (hash_data->count - pos)); + memmove(&lengths[pos + 1], + &lengths[pos], + sizeof(lengths[pos]) * (hash_data->count - pos)); + + hash_data->count++; + hash_data->over_provision--; + hash_data->operations++; + } + + /* de-serialize the new entry */ + entries[pos] = replace_baton->new_entry; + context = svn_temp_serializer__init_append(hash_data, + entries, + hash_data->len, + *data_len, + pool); + serialize_dir_entry(context, &entries[pos], &length); + + /* return the updated serialized data */ + SVN_ERR (return_serialized_dir_context(context, + data, + data_len)); + + /* since the previous call may have re-allocated the buffer, the lengths + * pointer may no longer point to the entry in that buffer. Therefore, + * re-map it again and store the length value after that. */ + + hash_data = (hash_data_t *)*data; + lengths = (apr_uint32_t *) + svn_temp_deserializer__ptr((const char *)hash_data, + (const void *const *)&hash_data->lengths); + lengths[pos] = length; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__serialize_rep_header(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + svn_fs_x__rep_header_t *copy = apr_palloc(pool, sizeof(*copy)); + *copy = *(svn_fs_x__rep_header_t *)in; + + *data_len = sizeof(svn_fs_x__rep_header_t); + *data = copy; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_rep_header(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + svn_fs_x__rep_header_t *copy = apr_palloc(pool, sizeof(*copy)); + SVN_ERR_ASSERT(data_len == sizeof(*copy)); + + *copy = *(svn_fs_x__rep_header_t *)data; + *out = data; + + return SVN_NO_ERROR; +} + +/* Utility function to serialize change CHANGE_P in the given serialization + * CONTEXT. + */ +static void +serialize_change(svn_temp_serializer__context_t *context, + change_t * const *change_p) +{ + const change_t * change = *change_p; + if (change == NULL) + return; + + /* serialize the change struct itself */ + svn_temp_serializer__push(context, + (const void * const *)change_p, + sizeof(*change)); + + /* serialize sub-structures */ + svn_fs_x__id_serialize(context, &change->info.node_rev_id); + + svn_temp_serializer__add_string(context, &change->path.data); + svn_temp_serializer__add_string(context, &change->info.copyfrom_path); + + /* return to the caller's nesting level */ + svn_temp_serializer__pop(context); +} + +/* Utility function to serialize the CHANGE_P within the given + * serialization CONTEXT. + */ +static void +deserialize_change(void *buffer, change_t **change_p) +{ + change_t * change; + + /* fix-up of the pointer to the struct in question */ + svn_temp_deserializer__resolve(buffer, (void **)change_p); + + change = *change_p; + if (change == NULL) + return; + + /* fix-up of sub-structures */ + svn_fs_x__id_deserialize(change, (svn_fs_id_t **)&change->info.node_rev_id); + + svn_temp_deserializer__resolve(change, (void **)&change->path.data); + svn_temp_deserializer__resolve(change, (void **)&change->info.copyfrom_path); +} + +/* Auxiliary structure representing the content of a change_t array. + This structure is much easier to (de-)serialize than an APR array. + */ +typedef struct changes_data_t +{ + /* number of entries in the array */ + int count; + + /* reference to the changes */ + change_t **changes; +} changes_data_t; + +svn_error_t * +svn_fs_x__serialize_changes(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + apr_array_header_t *array = in; + changes_data_t changes; + svn_temp_serializer__context_t *context; + svn_stringbuf_t *serialized; + int i; + + /* initialize our auxiliary data structure */ + changes.count = array->nelts; + changes.changes = apr_palloc(pool, sizeof(change_t*) * changes.count); + + /* populate it with the array elements */ + for (i = 0; i < changes.count; ++i) + changes.changes[i] = APR_ARRAY_IDX(array, i, change_t*); + + /* serialize it and all its elements */ + context = svn_temp_serializer__init(&changes, + sizeof(changes), + changes.count * 100, + pool); + + svn_temp_serializer__push(context, + (const void * const *)&changes.changes, + changes.count * sizeof(change_t*)); + + for (i = 0; i < changes.count; ++i) + serialize_change(context, &changes.changes[i]); + + svn_temp_serializer__pop(context); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_changes(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + int i; + changes_data_t *changes = (changes_data_t *)data; + apr_array_header_t *array = apr_array_make(pool, changes->count, + sizeof(change_t *)); + + /* de-serialize our auxiliary data structure */ + svn_temp_deserializer__resolve(changes, (void**)&changes->changes); + + /* de-serialize each entry and add it to the array */ + for (i = 0; i < changes->count; ++i) + { + deserialize_change((void*)changes->changes, + (change_t **)&changes->changes[i]); + APR_ARRAY_PUSH(array, change_t *) = changes->changes[i]; + } + + /* done */ + *out = array; + + return SVN_NO_ERROR; +} + +/* Auxiliary structure representing the content of a svn_mergeinfo_t hash. + This structure is much easier to (de-)serialize than an APR array. + */ +typedef struct mergeinfo_data_t +{ + /* number of paths in the hash */ + unsigned count; + + /* COUNT keys (paths) */ + const char **keys; + + /* COUNT keys lengths (strlen of path) */ + apr_ssize_t *key_lengths; + + /* COUNT entries, each giving the number of ranges for the key */ + int *range_counts; + + /* all ranges in a single, concatenated buffer */ + svn_merge_range_t *ranges; +} mergeinfo_data_t; + +svn_error_t * +svn_fs_x__serialize_mergeinfo(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + svn_mergeinfo_t mergeinfo = in; + mergeinfo_data_t merges; + svn_temp_serializer__context_t *context; + svn_stringbuf_t *serialized; + apr_hash_index_t *hi; + unsigned i; + int k; + apr_size_t range_count; + + /* initialize our auxiliary data structure */ + merges.count = apr_hash_count(mergeinfo); + merges.keys = apr_palloc(pool, sizeof(*merges.keys) * merges.count); + merges.key_lengths = apr_palloc(pool, sizeof(*merges.key_lengths) * + merges.count); + merges.range_counts = apr_palloc(pool, sizeof(*merges.range_counts) * + merges.count); + + i = 0; + range_count = 0; + for (hi = apr_hash_first(pool, mergeinfo); hi; hi = apr_hash_next(hi), ++i) + { + svn_rangelist_t *ranges; + apr_hash_this(hi, (const void**)&merges.keys[i], + &merges.key_lengths[i], + (void **)&ranges); + merges.range_counts[i] = ranges->nelts; + range_count += ranges->nelts; + } + + merges.ranges = apr_palloc(pool, sizeof(*merges.ranges) * range_count); + + i = 0; + for (hi = apr_hash_first(pool, mergeinfo); hi; hi = apr_hash_next(hi)) + { + svn_rangelist_t *ranges = svn__apr_hash_index_val(hi); + for (k = 0; k < ranges->nelts; ++k, ++i) + merges.ranges[i] = *APR_ARRAY_IDX(ranges, k, svn_merge_range_t*); + } + + /* serialize it and all its elements */ + context = svn_temp_serializer__init(&merges, + sizeof(merges), + range_count * 30, + pool); + + /* keys array */ + svn_temp_serializer__push(context, + (const void * const *)&merges.keys, + merges.count * sizeof(*merges.keys)); + + for (i = 0; i < merges.count; ++i) + svn_temp_serializer__add_string(context, &merges.keys[i]); + + svn_temp_serializer__pop(context); + + /* key lengths array */ + svn_temp_serializer__add_leaf(context, + (const void * const *)&merges.key_lengths, + merges.count * sizeof(*merges.key_lengths)); + + /* range counts array */ + svn_temp_serializer__add_leaf(context, + (const void * const *)&merges.range_counts, + merges.count * sizeof(*merges.range_counts)); + + /* ranges */ + svn_temp_serializer__add_leaf(context, + (const void * const *)&merges.ranges, + range_count * sizeof(*merges.ranges)); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deserialize_mergeinfo(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + unsigned i; + int k, n; + mergeinfo_data_t *merges = (mergeinfo_data_t *)data; + svn_mergeinfo_t mergeinfo; + + /* de-serialize our auxiliary data structure */ + svn_temp_deserializer__resolve(merges, (void**)&merges->keys); + svn_temp_deserializer__resolve(merges, (void**)&merges->key_lengths); + svn_temp_deserializer__resolve(merges, (void**)&merges->range_counts); + svn_temp_deserializer__resolve(merges, (void**)&merges->ranges); + + /* de-serialize keys and add entries to the result */ + n = 0; + mergeinfo = svn_hash__make(pool); + for (i = 0; i < merges->count; ++i) + { + svn_rangelist_t *ranges = apr_array_make(pool, + merges->range_counts[i], + sizeof(svn_merge_range_t*)); + for (k = 0; k < merges->range_counts[i]; ++k, ++n) + APR_ARRAY_PUSH(ranges, svn_merge_range_t*) = &merges->ranges[n]; + + svn_temp_deserializer__resolve((void*)merges->keys, + (void**)&merges->keys[i]); + apr_hash_set(mergeinfo, merges->keys[i], merges->key_lengths[i], ranges); + } + + /* done */ + *out = mergeinfo; + + return SVN_NO_ERROR; +} +
diff --git a/subversion/libsvn_fs_x/temp_serializer.h b/subversion/libsvn_fs_x/temp_serializer.h new file mode 100644 index 0000000..fda2015 --- /dev/null +++ b/subversion/libsvn_fs_x/temp_serializer.h
@@ -0,0 +1,305 @@ +/* temp_serializer.h : serialization functions for caching of FSX structures + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS__TEMP_SERIALIZER_H +#define SVN_LIBSVN_FS__TEMP_SERIALIZER_H + +#include "fs.h" + +/** + * Prepend the @a number to the @a string in a space efficient way such that + * no other (number,string) combination can produce the same result. + * Allocate temporaries as well as the result from @a pool. + */ +const char* +svn_fs_x__combine_number_and_string(apr_int64_t number, + const char *string, + apr_pool_t *pool); + +/** + * Serialize a @a noderev_p within the serialization @a context. + */ +void +svn_fs_x__noderev_serialize(struct svn_temp_serializer__context_t *context, + node_revision_t * const *noderev_p); + +/** + * Deserialize a @a noderev_p within the @a buffer. + */ +void +svn_fs_x__noderev_deserialize(void *buffer, + node_revision_t **noderev_p); + +/** + * Serialize APR array @a *a within the serialization @a context. + * The elements within the array must not contain pointers. + */ +void +svn_fs_x__serialize_apr_array(struct svn_temp_serializer__context_t *context, + apr_array_header_t **a); + +/** + * Deserialize APR @a *array within the @a buffer. Set its pool member to + * @a pool. The elements within the array must not contain pointers. + */ +void +svn_fs_x__deserialize_apr_array(void *buffer, + apr_array_header_t **array, + apr_pool_t *pool); + + +/** + * #svn_txdelta_window_t is not sufficient for caching the data it + * represents because data read process needs auxilliary information. + */ +typedef struct +{ + /* the txdelta window information cached / to be cached */ + svn_txdelta_window_t *window; + + /* the revision file read pointer position before reading the window */ + apr_off_t start_offset; + + /* the revision file read pointer position right after reading the window */ + apr_off_t end_offset; +} svn_fs_x__txdelta_cached_window_t; + +/** + * Implements #svn_cache__serialize_func_t for + * #svn_fs_x__txdelta_cached_window_t. + */ +svn_error_t * +svn_fs_x__serialize_txdelta_window(void **buffer, + apr_size_t *buffer_size, + void *item, + apr_pool_t *pool); + +/** + * Implements #svn_cache__deserialize_func_t for + * #svn_fs_x__txdelta_cached_window_t. + */ +svn_error_t * +svn_fs_x__deserialize_txdelta_window(void **item, + void *buffer, + apr_size_t buffer_size, + apr_pool_t *pool); + +/** + * Implements #svn_cache__serialize_func_t for a manifest + * (@a in is an #apr_array_header_t of apr_off_t elements). + */ +svn_error_t * +svn_fs_x__serialize_manifest(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/** + * Implements #svn_cache__deserialize_func_t for a manifest + * (@a *out is an #apr_array_header_t of apr_off_t elements). + */ +svn_error_t * +svn_fs_x__deserialize_manifest(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/** + * Implements #svn_cache__serialize_func_t for a properties hash + * (@a in is an #apr_hash_t of svn_string_t elements, keyed by const char*). + */ +svn_error_t * +svn_fs_x__serialize_properties(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/** + * Implements #svn_cache__deserialize_func_t for a properties hash + * (@a *out is an #apr_hash_t of svn_string_t elements, keyed by const char*). + */ +svn_error_t * +svn_fs_x__deserialize_properties(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/** + * Implements #svn_cache__serialize_func_t for #svn_fs_id_t + */ +svn_error_t * +svn_fs_x__serialize_id(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/** + * Implements #svn_cache__deserialize_func_t for #svn_fs_id_t + */ +svn_error_t * +svn_fs_x__deserialize_id(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/** + * Implements #svn_cache__serialize_func_t for #node_revision_t + */ +svn_error_t * +svn_fs_x__serialize_node_revision(void **buffer, + apr_size_t *buffer_size, + void *item, + apr_pool_t *pool); + +/** + * Implements #svn_cache__deserialize_func_t for #node_revision_t + */ +svn_error_t * +svn_fs_x__deserialize_node_revision(void **item, + void *buffer, + apr_size_t buffer_size, + apr_pool_t *pool); + +/** + * Implements #svn_cache__serialize_func_t for a directory contents hash + */ +svn_error_t * +svn_fs_x__serialize_dir_entries(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/** + * Implements #svn_cache__deserialize_func_t for a directory contents hash + */ +svn_error_t * +svn_fs_x__deserialize_dir_entries(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/** + * Implements #svn_cache__partial_getter_func_t. Set (apr_off_t) @a *out + * to the element indexed by (apr_int64_t) @a *baton within the + * serialized manifest array @a data and @a data_len. */ +svn_error_t * +svn_fs_x__get_sharded_offset(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool); + +/** + * Implements #svn_cache__partial_getter_func_t for a single + * #svn_fs_dirent_t within a serialized directory contents hash, + * identified by its name (const char @a *baton). + */ +svn_error_t * +svn_fs_x__extract_dir_entry(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool); + +/** + * Describes the change to be done to a directory: Set the entry + * identify by @a name to the value @a new_entry. If the latter is + * @c NULL, the entry shall be removed if it exists. Otherwise it + * will be replaced or automatically added, respectively. + */ +typedef struct replace_baton_t +{ + /** name of the directory entry to modify */ + const char *name; + + /** directory entry to insert instead */ + svn_fs_dirent_t *new_entry; +} replace_baton_t; + +/** + * Implements #svn_cache__partial_setter_func_t for a single + * #svn_fs_dirent_t within a serialized directory contents hash, + * identified by its name in the #replace_baton_t in @a baton. + */ +svn_error_t * +svn_fs_x__replace_dir_entry(void **data, + apr_size_t *data_len, + void *baton, + apr_pool_t *pool); + +/** + * Implements #svn_cache__serialize_func_t for a #svn_fs_x__rep_header_t. + */ +svn_error_t * +svn_fs_x__serialize_rep_header(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/** + * Implements #svn_cache__deserialize_func_t for a #svn_fs_x__rep_header_t. + */ +svn_error_t * +svn_fs_x__deserialize_rep_header(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/** + * Implements #svn_cache__serialize_func_t for an #apr_array_header_t of + * #change_t *. + */ +svn_error_t * +svn_fs_x__serialize_changes(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/** + * Implements #svn_cache__deserialize_func_t for an #apr_array_header_t of + * #change_t *. + */ +svn_error_t * +svn_fs_x__deserialize_changes(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/** + * Implements #svn_cache__serialize_func_t for #svn_mergeinfo_t objects. + */ +svn_error_t * +svn_fs_x__serialize_mergeinfo(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/** + * Implements #svn_cache__deserialize_func_t for #svn_mergeinfo_t objects. + */ +svn_error_t * +svn_fs_x__deserialize_mergeinfo(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +#endif
diff --git a/subversion/libsvn_fs_x/transaction.c b/subversion/libsvn_fs_x/transaction.c new file mode 100644 index 0000000..f1d77bd --- /dev/null +++ b/subversion/libsvn_fs_x/transaction.c
@@ -0,0 +1,3301 @@ +/* transaction.c --- transaction-related functions of FSX + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "transaction.h" + +#include <assert.h> +#include <apr_sha1.h> + +#include "svn_hash.h" +#include "svn_props.h" +#include "svn_sorts.h" +#include "svn_time.h" +#include "svn_dirent_uri.h" + +#include "fs_x.h" +#include "tree.h" +#include "util.h" +#include "id.h" +#include "low_level.h" +#include "temp_serializer.h" +#include "cached_data.h" +#include "lock.h" +#include "rep-cache.h" +#include "index.h" + +#include "private/svn_fs_util.h" +#include "private/svn_subr_private.h" +#include "private/svn_string_private.h" +#include "../libsvn_fs/fs-loader.h" + +#include "svn_private_config.h" + +/* The vtable associated with an open transaction object. */ +static txn_vtable_t txn_vtable = { + svn_fs_x__commit_txn, + svn_fs_x__abort_txn, + svn_fs_x__txn_prop, + svn_fs_x__txn_proplist, + svn_fs_x__change_txn_prop, + svn_fs_x__txn_root, + svn_fs_x__change_txn_props +}; + +/* FSX-specific data being attached to svn_fs_txn_t. + */ +typedef struct fs_txn_data_t +{ + /* Strongly typed representation of the TXN's ID member. */ + svn_fs_x__id_part_t txn_id; +} fs_txn_data_t; + +const svn_fs_x__id_part_t * +svn_fs_x__txn_get_id(svn_fs_txn_t *txn) +{ + fs_txn_data_t *ftd = txn->fsap_data; + return &ftd->txn_id; +} + +/* Functions for working with shared transaction data. */ + +/* Return the transaction object for transaction TXN_ID from the + transaction list of filesystem FS (which must already be locked via the + txn_list_lock mutex). If the transaction does not exist in the list, + then create a new transaction object and return it (if CREATE_NEW is + true) or return NULL (otherwise). */ +static fs_x_shared_txn_data_t * +get_shared_txn(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + svn_boolean_t create_new) +{ + fs_x_data_t *ffd = fs->fsap_data; + fs_x_shared_data_t *ffsd = ffd->shared; + fs_x_shared_txn_data_t *txn; + + for (txn = ffsd->txns; txn; txn = txn->next) + if (svn_fs_x__id_part_eq(&txn->txn_id, txn_id)) + break; + + if (txn || !create_new) + return txn; + + /* Use the transaction object from the (single-object) freelist, + if one is available, or otherwise create a new object. */ + if (ffsd->free_txn) + { + txn = ffsd->free_txn; + ffsd->free_txn = NULL; + } + else + { + apr_pool_t *subpool = svn_pool_create(ffsd->common_pool); + txn = apr_palloc(subpool, sizeof(*txn)); + txn->pool = subpool; + } + + txn->txn_id = *txn_id; + txn->being_written = FALSE; + + /* Link this transaction into the head of the list. We will typically + be dealing with only one active transaction at a time, so it makes + sense for searches through the transaction list to look at the + newest transactions first. */ + txn->next = ffsd->txns; + ffsd->txns = txn; + + return txn; +} + +/* Free the transaction object for transaction TXN_ID, and remove it + from the transaction list of filesystem FS (which must already be + locked via the txn_list_lock mutex). Do nothing if the transaction + does not exist. */ +static void +free_shared_txn(svn_fs_t *fs, const svn_fs_x__id_part_t *txn_id) +{ + fs_x_data_t *ffd = fs->fsap_data; + fs_x_shared_data_t *ffsd = ffd->shared; + fs_x_shared_txn_data_t *txn, *prev = NULL; + + for (txn = ffsd->txns; txn; prev = txn, txn = txn->next) + if (svn_fs_x__id_part_eq(&txn->txn_id, txn_id)) + break; + + if (!txn) + return; + + if (prev) + prev->next = txn->next; + else + ffsd->txns = txn->next; + + /* As we typically will be dealing with one transaction after another, + we will maintain a single-object free list so that we can hopefully + keep reusing the same transaction object. */ + if (!ffsd->free_txn) + ffsd->free_txn = txn; + else + svn_pool_destroy(txn->pool); +} + + +/* Obtain a lock on the transaction list of filesystem FS, call BODY + with FS, BATON, and POOL, and then unlock the transaction list. + Return what BODY returned. */ +static svn_error_t * +with_txnlist_lock(svn_fs_t *fs, + svn_error_t *(*body)(svn_fs_t *fs, + const void *baton, + apr_pool_t *pool), + const void *baton, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + fs_x_shared_data_t *ffsd = ffd->shared; + + SVN_MUTEX__WITH_LOCK(ffsd->txn_list_lock, + body(fs, baton, pool)); + + return SVN_NO_ERROR; +} + + +/* Get a lock on empty file LOCK_FILENAME, creating it in POOL. */ +static svn_error_t * +get_lock_on_filesystem(const char *lock_filename, + apr_pool_t *pool) +{ + svn_error_t *err = svn_io_file_lock2(lock_filename, TRUE, FALSE, pool); + + if (err && APR_STATUS_IS_ENOENT(err->apr_err)) + { + /* No lock file? No big deal; these are just empty files + anyway. Create it and try again. */ + svn_error_clear(err); + err = NULL; + + SVN_ERR(svn_io_file_create_empty(lock_filename, pool)); + SVN_ERR(svn_io_file_lock2(lock_filename, TRUE, FALSE, pool)); + } + + return svn_error_trace(err); +} + +/* Reset the HAS_WRITE_LOCK member in the FFD given as BATON_VOID. + When registered with the pool holding the lock on the lock file, + this makes sure the flag gets reset just before we release the lock. */ +static apr_status_t +reset_lock_flag(void *baton_void) +{ + fs_x_data_t *ffd = baton_void; + ffd->has_write_lock = FALSE; + return APR_SUCCESS; +} + +/* Obtain a write lock on the file LOCK_FILENAME (protecting with + LOCK_MUTEX if APR is threaded) in a subpool of POOL, call BODY with + BATON and that subpool, destroy the subpool (releasing the write + lock) and return what BODY returned. If IS_GLOBAL_LOCK is set, + set the HAS_WRITE_LOCK flag while we keep the write lock. */ +static svn_error_t * +with_some_lock_file(svn_fs_t *fs, + svn_error_t *(*body)(void *baton, + apr_pool_t *pool), + void *baton, + const char *lock_filename, + svn_boolean_t is_global_lock, + apr_pool_t *pool) +{ + apr_pool_t *subpool = svn_pool_create(pool); + svn_error_t *err = get_lock_on_filesystem(lock_filename, subpool); + + if (!err) + { + fs_x_data_t *ffd = fs->fsap_data; + + if (is_global_lock) + { + /* set the "got the lock" flag and register reset function */ + apr_pool_cleanup_register(subpool, + ffd, + reset_lock_flag, + apr_pool_cleanup_null); + ffd->has_write_lock = TRUE; + } + + /* nobody else will modify the repo state + => read HEAD & pack info once */ + SVN_ERR(svn_fs_x__update_min_unpacked_rev(fs, pool)); + SVN_ERR(svn_fs_x__youngest_rev(&ffd->youngest_rev_cache, fs, pool)); + err = body(baton, subpool); + } + + svn_pool_destroy(subpool); + + return svn_error_trace(err); +} + +svn_error_t * +svn_fs_x__with_write_lock(svn_fs_t *fs, + svn_error_t *(*body)(void *baton, + apr_pool_t *pool), + void *baton, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + fs_x_shared_data_t *ffsd = ffd->shared; + + SVN_MUTEX__WITH_LOCK(ffsd->fs_write_lock, + with_some_lock_file(fs, body, baton, + svn_fs_x__path_lock(fs, pool), + TRUE, + pool)); + + return SVN_NO_ERROR; +} + +/* Run BODY (with BATON and POOL) while the txn-current file + of FS is locked. */ +static svn_error_t * +with_txn_current_lock(svn_fs_t *fs, + svn_error_t *(*body)(void *baton, + apr_pool_t *pool), + void *baton, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + fs_x_shared_data_t *ffsd = ffd->shared; + + SVN_MUTEX__WITH_LOCK(ffsd->txn_current_lock, + with_some_lock_file(fs, body, baton, + svn_fs_x__path_txn_current_lock(fs, pool), + FALSE, + pool)); + + return SVN_NO_ERROR; +} + +/* A structure used by unlock_proto_rev() and unlock_proto_rev_body(), + which see. */ +struct unlock_proto_rev_baton +{ + svn_fs_x__id_part_t txn_id; + void *lockcookie; +}; + +/* Callback used in the implementation of unlock_proto_rev(). */ +static svn_error_t * +unlock_proto_rev_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool) +{ + const struct unlock_proto_rev_baton *b = baton; + apr_file_t *lockfile = b->lockcookie; + fs_x_shared_txn_data_t *txn = get_shared_txn(fs, &b->txn_id, FALSE); + apr_status_t apr_err; + + if (!txn) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Can't unlock unknown transaction '%s'"), + svn_fs_x__id_txn_unparse(&b->txn_id, pool)); + if (!txn->being_written) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Can't unlock nonlocked transaction '%s'"), + svn_fs_x__id_txn_unparse(&b->txn_id, pool)); + + apr_err = apr_file_unlock(lockfile); + if (apr_err) + return svn_error_wrap_apr + (apr_err, + _("Can't unlock prototype revision lockfile for transaction '%s'"), + svn_fs_x__id_txn_unparse(&b->txn_id, pool)); + apr_err = apr_file_close(lockfile); + if (apr_err) + return svn_error_wrap_apr + (apr_err, + _("Can't close prototype revision lockfile for transaction '%s'"), + svn_fs_x__id_txn_unparse(&b->txn_id, pool)); + + txn->being_written = FALSE; + + return SVN_NO_ERROR; +} + +/* Unlock the prototype revision file for transaction TXN_ID in filesystem + FS using cookie LOCKCOOKIE. The original prototype revision file must + have been closed _before_ calling this function. + + Perform temporary allocations in POOL. */ +static svn_error_t * +unlock_proto_rev(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + void *lockcookie, + apr_pool_t *pool) +{ + struct unlock_proto_rev_baton b; + + b.txn_id = *txn_id; + b.lockcookie = lockcookie; + return with_txnlist_lock(fs, unlock_proto_rev_body, &b, pool); +} + +/* Same as unlock_proto_rev(), but requires that the transaction list + lock is already held. */ +static svn_error_t * +unlock_proto_rev_list_locked(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + void *lockcookie, + apr_pool_t *pool) +{ + struct unlock_proto_rev_baton b; + + b.txn_id = *txn_id; + b.lockcookie = lockcookie; + return unlock_proto_rev_body(fs, &b, pool); +} + +/* A structure used by get_writable_proto_rev() and + get_writable_proto_rev_body(), which see. */ +struct get_writable_proto_rev_baton +{ + apr_file_t **file; + void **lockcookie; + svn_fs_x__id_part_t txn_id; +}; + +/* Callback used in the implementation of get_writable_proto_rev(). */ +static svn_error_t * +get_writable_proto_rev_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool) +{ + const struct get_writable_proto_rev_baton *b = baton; + apr_file_t **file = b->file; + void **lockcookie = b->lockcookie; + svn_error_t *err; + fs_x_shared_txn_data_t *txn = get_shared_txn(fs, &b->txn_id, TRUE); + + /* First, ensure that no thread in this process (including this one) + is currently writing to this transaction's proto-rev file. */ + if (txn->being_written) + return svn_error_createf(SVN_ERR_FS_REP_BEING_WRITTEN, NULL, + _("Cannot write to the prototype revision file " + "of transaction '%s' because a previous " + "representation is currently being written by " + "this process"), + svn_fs_x__id_txn_unparse(&b->txn_id, pool)); + + + /* We know that no thread in this process is writing to the proto-rev + file, and by extension, that no thread in this process is holding a + lock on the prototype revision lock file. It is therefore safe + for us to attempt to lock this file, to see if any other process + is holding a lock. */ + + { + apr_file_t *lockfile; + apr_status_t apr_err; + const char *lockfile_path + = svn_fs_x__path_txn_proto_rev_lock(fs, &b->txn_id, pool); + + /* Open the proto-rev lockfile, creating it if necessary, as it may + not exist if the transaction dates from before the lockfiles were + introduced. + + ### We'd also like to use something like svn_io_file_lock2(), but + that forces us to create a subpool just to be able to unlock + the file, which seems a waste. */ + SVN_ERR(svn_io_file_open(&lockfile, lockfile_path, + APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool)); + + apr_err = apr_file_lock(lockfile, + APR_FLOCK_EXCLUSIVE | APR_FLOCK_NONBLOCK); + if (apr_err) + { + svn_error_clear(svn_io_file_close(lockfile, pool)); + + if (APR_STATUS_IS_EAGAIN(apr_err)) + return svn_error_createf(SVN_ERR_FS_REP_BEING_WRITTEN, NULL, + _("Cannot write to the prototype revision " + "file of transaction '%s' because a " + "previous representation is currently " + "being written by another process"), + svn_fs_x__id_txn_unparse(&b->txn_id, + pool)); + + return svn_error_wrap_apr(apr_err, + _("Can't get exclusive lock on file '%s'"), + svn_dirent_local_style(lockfile_path, pool)); + } + + *lockcookie = lockfile; + } + + /* We've successfully locked the transaction; mark it as such. */ + txn->being_written = TRUE; + + + /* Now open the prototype revision file and seek to the end. */ + err = svn_io_file_open(file, + svn_fs_x__path_txn_proto_rev(fs, &b->txn_id, pool), + APR_WRITE | APR_BUFFERED, APR_OS_DEFAULT, pool); + + /* You might expect that we could dispense with the following seek + and achieve the same thing by opening the file using APR_APPEND. + Unfortunately, APR's buffered file implementation unconditionally + places its initial file pointer at the start of the file (even for + files opened with APR_APPEND), so we need this seek to reconcile + the APR file pointer to the OS file pointer (since we need to be + able to read the current file position later). */ + if (!err) + { + apr_off_t offset = 0; + err = svn_io_file_seek(*file, APR_END, &offset, pool); + } + + if (err) + { + err = svn_error_compose_create( + err, + unlock_proto_rev_list_locked(fs, &b->txn_id, *lockcookie, pool)); + + *lockcookie = NULL; + } + + return svn_error_trace(err); +} + +/* Get a handle to the prototype revision file for transaction TXN_ID in + filesystem FS, and lock it for writing. Return FILE, a file handle + positioned at the end of the file, and LOCKCOOKIE, a cookie that + should be passed to unlock_proto_rev() to unlock the file once FILE + has been closed. + + If the prototype revision file is already locked, return error + SVN_ERR_FS_REP_BEING_WRITTEN. + + Perform all allocations in POOL. */ +static svn_error_t * +get_writable_proto_rev(apr_file_t **file, + void **lockcookie, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + struct get_writable_proto_rev_baton b; + + b.file = file; + b.lockcookie = lockcookie; + b.txn_id = *txn_id; + + return with_txnlist_lock(fs, get_writable_proto_rev_body, &b, pool); +} + +/* Callback used in the implementation of purge_shared_txn(). */ +static svn_error_t * +purge_shared_txn_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool) +{ + const svn_fs_x__id_part_t *txn_id = baton; + + free_shared_txn(fs, txn_id); + svn_fs_x__reset_txn_caches(fs); + + return SVN_NO_ERROR; +} + +/* Purge the shared data for transaction TXN_ID in filesystem FS. + Perform all allocations in POOL. */ +static svn_error_t * +purge_shared_txn(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + return with_txnlist_lock(fs, purge_shared_txn_body, txn_id, pool); +} + + +svn_error_t * +svn_fs_x__put_node_revision(svn_fs_t *fs, + const svn_fs_id_t *id, + node_revision_t *noderev, + svn_boolean_t fresh_txn_root, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + apr_file_t *noderev_file; + + noderev->is_fresh_txn_root = fresh_txn_root; + + if (! svn_fs_x__id_is_txn(id)) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Attempted to write to non-transaction '%s'"), + svn_fs_x__id_unparse(id, pool)->data); + + SVN_ERR(svn_io_file_open(&noderev_file, + svn_fs_x__path_txn_node_rev(fs, id, pool), + APR_WRITE | APR_CREATE | APR_TRUNCATE + | APR_BUFFERED, APR_OS_DEFAULT, pool)); + + SVN_ERR(svn_fs_x__write_noderev(svn_stream_from_aprfile2(noderev_file, TRUE, + pool), + noderev, ffd->format, pool)); + + SVN_ERR(svn_io_file_close(noderev_file, pool)); + + return SVN_NO_ERROR; +} + +/* For the in-transaction NODEREV within FS, write the sha1->rep mapping + * file in the respective transaction, if rep sharing has been enabled etc. + * Use POOL for temporary allocations. + */ +static svn_error_t * +store_sha1_rep_mapping(svn_fs_t *fs, + node_revision_t *noderev, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + + /* if rep sharing has been enabled and the noderev has a data rep and + * its SHA-1 is known, store the rep struct under its SHA1. */ + if ( ffd->rep_sharing_allowed + && noderev->data_rep + && noderev->data_rep->has_sha1) + { + apr_file_t *rep_file; + const char *file_name + = svn_fs_x__path_txn_sha1(fs, &noderev->data_rep->txn_id, + noderev->data_rep->sha1_digest, pool); + svn_stringbuf_t *rep_string + = svn_fs_x__unparse_representation(noderev->data_rep, + ffd->format, + (noderev->kind == svn_node_dir), + pool); + SVN_ERR(svn_io_file_open(&rep_file, file_name, + APR_WRITE | APR_CREATE | APR_TRUNCATE + | APR_BUFFERED, APR_OS_DEFAULT, pool)); + + SVN_ERR(svn_io_file_write_full(rep_file, rep_string->data, + rep_string->len, NULL, pool)); + + SVN_ERR(svn_io_file_close(rep_file, pool)); + } + + return SVN_NO_ERROR; +} + + +static const char * +unparse_dir_entry(svn_node_kind_t kind, const svn_fs_id_t *id, + apr_pool_t *pool) +{ + return apr_psprintf(pool, "%s %s", + (kind == svn_node_file) ? SVN_FS_X__KIND_FILE + : SVN_FS_X__KIND_DIR, + svn_fs_x__id_unparse(id, pool)->data); +} + +/* Given a hash ENTRIES of dirent structions, return a hash in + *STR_ENTRIES_P, that has svn_string_t as the values in the format + specified by the fs_x directory contents file. Perform + allocations in POOL. */ +static svn_error_t * +unparse_dir_entries(apr_hash_t **str_entries_p, + apr_hash_t *entries, + apr_pool_t *pool) +{ + apr_hash_index_t *hi; + + /* For now, we use a our own hash function to ensure that we get a + * (largely) stable order when serializing the data. It also gives + * us some performance improvement. + * + * ### TODO ### + * Use some sorted or other fixed order data container. + */ + *str_entries_p = svn_hash__make(pool); + + for (hi = apr_hash_first(pool, entries); hi; hi = apr_hash_next(hi)) + { + const void *key; + apr_ssize_t klen; + svn_fs_dirent_t *dirent = svn__apr_hash_index_val(hi); + const char *new_val; + + apr_hash_this(hi, &key, &klen, NULL); + new_val = unparse_dir_entry(dirent->kind, dirent->id, pool); + apr_hash_set(*str_entries_p, key, klen, + svn_string_create(new_val, pool)); + } + + return SVN_NO_ERROR; +} + +/* Merge the internal-use-only CHANGE into a hash of public-FS + svn_fs_path_change2_t CHANGES, collapsing multiple changes into a + single summarical (is that real word?) change per path. Also keep + the COPYFROM_CACHE up to date with new adds and replaces. */ +static svn_error_t * +fold_change(apr_hash_t *changes, + const change_t *change) +{ + apr_pool_t *pool = apr_hash_pool_get(changes); + svn_fs_path_change2_t *old_change, *new_change; + const svn_string_t *path = &change->path; + const svn_fs_path_change2_t *info = &change->info; + + if ((old_change = apr_hash_get(changes, path->data, path->len))) + { + /* This path already exists in the hash, so we have to merge + this change into the already existing one. */ + + /* Sanity check: only allow NULL node revision ID in the + `reset' case. */ + if ((! info->node_rev_id) + && (info->change_kind != svn_fs_path_change_reset)) + return svn_error_create + (SVN_ERR_FS_CORRUPT, NULL, + _("Missing required node revision ID")); + + /* Sanity check: we should be talking about the same node + revision ID as our last change except where the last change + was a deletion. */ + if (info->node_rev_id + && (! svn_fs_x__id_eq(old_change->node_rev_id, info->node_rev_id)) + && (old_change->change_kind != svn_fs_path_change_delete)) + return svn_error_create + (SVN_ERR_FS_CORRUPT, NULL, + _("Invalid change ordering: new node revision ID " + "without delete")); + + /* Sanity check: an add, replacement, or reset must be the first + thing to follow a deletion. */ + if ((old_change->change_kind == svn_fs_path_change_delete) + && (! ((info->change_kind == svn_fs_path_change_replace) + || (info->change_kind == svn_fs_path_change_reset) + || (info->change_kind == svn_fs_path_change_add)))) + return svn_error_create + (SVN_ERR_FS_CORRUPT, NULL, + _("Invalid change ordering: non-add change on deleted path")); + + /* Sanity check: an add can't follow anything except + a delete or reset. */ + if ((info->change_kind == svn_fs_path_change_add) + && (old_change->change_kind != svn_fs_path_change_delete) + && (old_change->change_kind != svn_fs_path_change_reset)) + return svn_error_create + (SVN_ERR_FS_CORRUPT, NULL, + _("Invalid change ordering: add change on preexisting path")); + + /* Now, merge that change in. */ + switch (info->change_kind) + { + case svn_fs_path_change_reset: + /* A reset here will simply remove the path change from the + hash. */ + old_change = NULL; + break; + + case svn_fs_path_change_delete: + if (old_change->change_kind == svn_fs_path_change_add) + { + /* If the path was introduced in this transaction via an + add, and we are deleting it, just remove the path + altogether. */ + old_change = NULL; + } + else + { + /* A deletion overrules all previous changes. */ + old_change->change_kind = svn_fs_path_change_delete; + old_change->text_mod = info->text_mod; + old_change->prop_mod = info->prop_mod; + old_change->copyfrom_rev = SVN_INVALID_REVNUM; + old_change->copyfrom_path = NULL; + } + break; + + case svn_fs_path_change_add: + case svn_fs_path_change_replace: + /* An add at this point must be following a previous delete, + so treat it just like a replace. */ + old_change->change_kind = svn_fs_path_change_replace; + old_change->node_rev_id = svn_fs_x__id_copy(info->node_rev_id, + pool); + old_change->text_mod = info->text_mod; + old_change->prop_mod = info->prop_mod; + if (info->copyfrom_rev == SVN_INVALID_REVNUM) + { + old_change->copyfrom_rev = SVN_INVALID_REVNUM; + old_change->copyfrom_path = NULL; + } + else + { + old_change->copyfrom_rev = info->copyfrom_rev; + old_change->copyfrom_path = apr_pstrdup(pool, + info->copyfrom_path); + } + break; + + case svn_fs_path_change_modify: + default: + if (info->text_mod) + old_change->text_mod = TRUE; + if (info->prop_mod) + old_change->prop_mod = TRUE; + break; + } + + /* remove old_change from the cache if it is no longer needed. */ + if (old_change == NULL) + apr_hash_set(changes, path->data, path->len, NULL); + } + else + { + /* This change is new to the hash, so make a new public change + structure from the internal one (in the hash's pool), and dup + the path into the hash's pool, too. */ + new_change = apr_pmemdup(pool, info, sizeof(*new_change)); + new_change->node_rev_id = svn_fs_x__id_copy(info->node_rev_id, pool); + if (info->copyfrom_path) + new_change->copyfrom_path = apr_pstrdup(pool, info->copyfrom_path); + + /* Add this path. The API makes no guarantees that this (new) key + will not be retained. Thus, we copy the key into the target pool + to ensure a proper lifetime. */ + apr_hash_set(changes, + apr_pstrmemdup(pool, path->data, path->len), path->len, + new_change); + } + + return SVN_NO_ERROR; +} + + +/* Examine all the changed path entries in CHANGES and store them in + *CHANGED_PATHS. Folding is done to remove redundant or unnecessary + *data. Do all allocations in POOL. */ +static svn_error_t * +process_changes(apr_hash_t *changed_paths, + apr_array_header_t *changes, + apr_pool_t *pool) +{ + apr_pool_t *iterpool = svn_pool_create(pool); + int i; + + /* Read in the changes one by one, folding them into our local hash + as necessary. */ + + for (i = 0; i < changes->nelts; ++i) + { + change_t *change = APR_ARRAY_IDX(changes, i, change_t *); + + SVN_ERR(fold_change(changed_paths, change)); + + /* Now, if our change was a deletion or replacement, we have to + blow away any changes thus far on paths that are (or, were) + children of this path. + ### i won't bother with another iteration pool here -- at + most we talking about a few extra dups of paths into what + is already a temporary subpool. + */ + + if ((change->info.change_kind == svn_fs_path_change_delete) + || (change->info.change_kind == svn_fs_path_change_replace)) + { + apr_hash_index_t *hi; + + /* a potential child path must contain at least 2 more chars + (the path separator plus at least one char for the name). + Also, we should not assume that all paths have been normalized + i.e. some might have trailing path separators. + */ + apr_ssize_t path_len = change->path.len; + apr_ssize_t min_child_len = path_len == 0 + ? 1 + : change->path.data[path_len-1] == '/' + ? path_len + 1 + : path_len + 2; + + /* CAUTION: This is the inner loop of an O(n^2) algorithm. + The number of changes to process may be >> 1000. + Therefore, keep the inner loop as tight as possible. + */ + for (hi = apr_hash_first(iterpool, changed_paths); + hi; + hi = apr_hash_next(hi)) + { + /* KEY is the path. */ + const void *path; + apr_ssize_t klen; + apr_hash_this(hi, &path, &klen, NULL); + + /* If we come across a child of our path, remove it. + Call svn_dirent_is_child only if there is a chance that + this is actually a sub-path. + */ + if ( klen >= min_child_len + && svn_dirent_is_child(change->path.data, path, iterpool)) + apr_hash_set(changed_paths, path, klen, NULL); + } + + /* Clear the per-iteration subpool. */ + svn_pool_clear(iterpool); + } + } + + /* Destroy the per-iteration subpool. */ + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__txn_changes_fetch(apr_hash_t **changed_paths_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + apr_file_t *file; + apr_hash_t *changed_paths = apr_hash_make(pool); + apr_array_header_t *changes; + apr_pool_t *scratch_pool = svn_pool_create(pool); + + SVN_ERR(svn_io_file_open(&file, + svn_fs_x__path_txn_changes(fs, txn_id, scratch_pool), + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, + scratch_pool)); + + SVN_ERR(svn_fs_x__read_changes(&changes, + svn_stream_from_aprfile2(file, TRUE, + scratch_pool), + scratch_pool)); + SVN_ERR(process_changes(changed_paths, changes, pool)); + svn_pool_destroy(scratch_pool); + + *changed_paths_p = changed_paths; + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__paths_changed(apr_hash_t **changed_paths_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + apr_hash_t *changed_paths; + apr_array_header_t *changes; + int i; + + SVN_ERR(svn_fs_x__get_changes(&changes, fs, rev, pool)); + + changed_paths = svn_hash__make(pool); + for (i = 0; i < changes->nelts; ++i) + { + change_t *change = APR_ARRAY_IDX(changes, i, change_t *); + apr_hash_set(changed_paths, change->path.data, change->path.len, + &change->info); + } + + *changed_paths_p = changed_paths; + + return SVN_NO_ERROR; +} + +/* Copy a revision node-rev SRC into the current transaction TXN_ID in + the filesystem FS. This is only used to create the root of a transaction. + Allocations are from POOL. */ +static svn_error_t * +create_new_txn_noderev_from_rev(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + svn_fs_id_t *src, + apr_pool_t *pool) +{ + node_revision_t *noderev; + const svn_fs_x__id_part_t *node_id, *copy_id; + + SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, src, pool)); + + if (svn_fs_x__id_is_txn(noderev->id)) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Copying from transactions not allowed")); + + noderev->predecessor_id = noderev->id; + noderev->predecessor_count++; + noderev->copyfrom_path = NULL; + noderev->copyfrom_rev = SVN_INVALID_REVNUM; + + /* For the transaction root, the copyroot never changes. */ + + node_id = svn_fs_x__id_node_id(noderev->id); + copy_id = svn_fs_x__id_copy_id(noderev->id); + noderev->id = svn_fs_x__id_txn_create(node_id, copy_id, txn_id, pool); + + return svn_fs_x__put_node_revision(fs, noderev->id, noderev, TRUE, pool); +} + +/* A structure used by get_and_increment_txn_key_body(). */ +struct get_and_increment_txn_key_baton { + svn_fs_t *fs; + apr_uint64_t txn_number; + apr_pool_t *pool; +}; + +/* Callback used in the implementation of create_txn_dir(). This gets + the current base 36 value in PATH_TXN_CURRENT and increments it. + It returns the original value by the baton. */ +static svn_error_t * +get_and_increment_txn_key_body(void *baton, apr_pool_t *pool) +{ + struct get_and_increment_txn_key_baton *cb = baton; + const char *txn_current_filename = svn_fs_x__path_txn_current(cb->fs, pool); + const char *tmp_filename; + char new_id_str[SVN_INT64_BUFFER_SIZE]; + + svn_stringbuf_t *buf; + SVN_ERR(svn_fs_x__read_content(&buf, txn_current_filename, cb->pool)); + + /* remove trailing newlines */ + cb->txn_number = svn__base36toui64(NULL, buf->data); + + /* Increment the key and add a trailing \n to the string so the + txn-current file has a newline in it. */ + SVN_ERR(svn_io_write_unique(&tmp_filename, + svn_dirent_dirname(txn_current_filename, pool), + new_id_str, + svn__ui64tobase36(new_id_str, cb->txn_number+1), + svn_io_file_del_none, pool)); + SVN_ERR(svn_fs_x__move_into_place(tmp_filename, txn_current_filename, + txn_current_filename, pool)); + + return SVN_NO_ERROR; +} + +/* Create a unique directory for a transaction in FS based on revision REV. + Return the ID for this transaction in *ID_P and *TXN_ID. Use a sequence + value in the transaction ID to prevent reuse of transaction IDs. */ +static svn_error_t * +create_txn_dir(const char **id_p, + svn_fs_x__id_part_t *txn_id, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + struct get_and_increment_txn_key_baton cb; + const char *txn_dir; + + /* Get the current transaction sequence value, which is a base-36 + number, from the txn-current file, and write an + incremented value back out to the file. Place the revision + number the transaction is based off into the transaction id. */ + cb.pool = pool; + cb.fs = fs; + SVN_ERR(with_txn_current_lock(fs, + get_and_increment_txn_key_body, + &cb, + pool)); + txn_id->revision = rev; + txn_id->number = cb.txn_number; + + *id_p = svn_fs_x__id_txn_unparse(txn_id, pool); + txn_dir = svn_dirent_join_many(pool, + fs->path, + PATH_TXNS_DIR, + apr_pstrcat(pool, *id_p, PATH_EXT_TXN, + (char *)NULL), + NULL); + + return svn_io_dir_make(txn_dir, APR_OS_DEFAULT, pool); +} + +svn_error_t * +svn_fs_x__create_txn(svn_fs_txn_t **txn_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + svn_fs_txn_t *txn; + fs_txn_data_t *ftd; + svn_fs_id_t *root_id; + + txn = apr_pcalloc(pool, sizeof(*txn)); + ftd = apr_pcalloc(pool, sizeof(*ftd)); + + /* Get the txn_id. */ + SVN_ERR(create_txn_dir(&txn->id, &ftd->txn_id, fs, rev, pool)); + + txn->fs = fs; + txn->base_rev = rev; + + txn->vtable = &txn_vtable; + txn->fsap_data = ftd; + *txn_p = txn; + + /* Create a new root node for this transaction. */ + SVN_ERR(svn_fs_x__rev_get_root(&root_id, fs, rev, pool)); + SVN_ERR(create_new_txn_noderev_from_rev(fs, &ftd->txn_id, root_id, pool)); + + /* Create an empty rev file. */ + SVN_ERR(svn_io_file_create_empty( + svn_fs_x__path_txn_proto_rev(fs, &ftd->txn_id, pool), + pool)); + + /* Create an empty rev-lock file. */ + SVN_ERR(svn_io_file_create_empty( + svn_fs_x__path_txn_proto_rev_lock(fs, &ftd->txn_id, pool), + pool)); + + /* Create an empty changes file. */ + SVN_ERR(svn_io_file_create_empty( + svn_fs_x__path_txn_changes(fs, &ftd->txn_id, pool), + pool)); + + /* Create the next-ids file. */ + return svn_io_file_create( + svn_fs_x__path_txn_next_ids(fs, &ftd->txn_id, pool), + "0 0\n", pool); +} + +/* Store the property list for transaction TXN_ID in PROPLIST. + Perform temporary allocations in POOL. */ +static svn_error_t * +get_txn_proplist(apr_hash_t *proplist, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + svn_stream_t *stream; + + /* Check for issue #3696. (When we find and fix the cause, we can change + * this to an assertion.) */ + if (!txn_id || !svn_fs_x__id_txn_used(txn_id)) + return svn_error_create(SVN_ERR_INCORRECT_PARAMS, NULL, + _("Internal error: a null transaction id was " + "passed to get_txn_proplist()")); + + /* Open the transaction properties file. */ + SVN_ERR(svn_stream_open_readonly(&stream, + svn_fs_x__path_txn_props(fs, txn_id, pool), + pool, pool)); + + /* Read in the property list. */ + SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool)); + + return svn_stream_close(stream); +} + +svn_error_t * +svn_fs_x__change_txn_prop(svn_fs_txn_t *txn, + const char *name, + const svn_string_t *value, + apr_pool_t *pool) +{ + apr_array_header_t *props = apr_array_make(pool, 1, sizeof(svn_prop_t)); + svn_prop_t prop; + + prop.name = name; + prop.value = value; + APR_ARRAY_PUSH(props, svn_prop_t) = prop; + + return svn_fs_x__change_txn_props(txn, props, pool); +} + +svn_error_t * +svn_fs_x__change_txn_props(svn_fs_txn_t *txn, + const apr_array_header_t *props, + apr_pool_t *pool) +{ + fs_txn_data_t *ftd = txn->fsap_data; + const char *txn_prop_filename; + svn_stringbuf_t *buf; + svn_stream_t *stream; + apr_hash_t *txn_prop = apr_hash_make(pool); + int i; + svn_error_t *err; + + err = get_txn_proplist(txn_prop, txn->fs, &ftd->txn_id, pool); + /* Here - and here only - we need to deal with the possibility that the + transaction property file doesn't yet exist. The rest of the + implementation assumes that the file exists, but we're called to set the + initial transaction properties as the transaction is being created. */ + if (err && (APR_STATUS_IS_ENOENT(err->apr_err))) + svn_error_clear(err); + else if (err) + return svn_error_trace(err); + + for (i = 0; i < props->nelts; i++) + { + svn_prop_t *prop = &APR_ARRAY_IDX(props, i, svn_prop_t); + + svn_hash_sets(txn_prop, prop->name, prop->value); + } + + /* Create a new version of the file and write out the new props. */ + /* Open the transaction properties file. */ + buf = svn_stringbuf_create_ensure(1024, pool); + stream = svn_stream_from_stringbuf(buf, pool); + SVN_ERR(svn_hash_write2(txn_prop, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_stream_close(stream)); + SVN_ERR(svn_io_write_unique(&txn_prop_filename, + svn_fs_x__path_txn_dir(txn->fs, &ftd->txn_id, pool), + buf->data, + buf->len, + svn_io_file_del_none, + pool)); + return svn_io_file_rename(txn_prop_filename, + svn_fs_x__path_txn_props(txn->fs, &ftd->txn_id, pool), + pool); +} + +svn_error_t * +svn_fs_x__get_txn(transaction_t **txn_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + transaction_t *txn; + node_revision_t *noderev; + svn_fs_id_t *root_id; + + txn = apr_pcalloc(pool, sizeof(*txn)); + txn->proplist = apr_hash_make(pool); + + SVN_ERR(get_txn_proplist(txn->proplist, fs, txn_id, pool)); + root_id = svn_fs_x__id_txn_create_root(txn_id, pool); + + SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, root_id, pool)); + + txn->root_id = svn_fs_x__id_copy(noderev->id, pool); + txn->base_id = svn_fs_x__id_copy(noderev->predecessor_id, pool); + txn->copies = NULL; + + *txn_p = txn; + + return SVN_NO_ERROR; +} + +/* Write out the currently available next node_id NODE_ID and copy_id + COPY_ID for transaction TXN_ID in filesystem FS. The next node-id is + used both for creating new unique nodes for the given transaction, as + well as uniquifying representations. Perform temporary allocations in + POOL. */ +static svn_error_t * +write_next_ids(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_uint64_t node_id, + apr_uint64_t copy_id, + apr_pool_t *pool) +{ + apr_file_t *file; + char buffer[2 * SVN_INT64_BUFFER_SIZE + 2]; + char *p = buffer; + + p += svn__ui64tobase36(p, node_id); + *(p++) = ' '; + p += svn__ui64tobase36(p, copy_id); + *(p++) = '\n'; + *(p++) = '\0'; + + SVN_ERR(svn_io_file_open(&file, + svn_fs_x__path_txn_next_ids(fs, txn_id, pool), + APR_WRITE | APR_TRUNCATE, + APR_OS_DEFAULT, pool)); + SVN_ERR(svn_io_file_write_full(file, buffer, p - buffer, NULL, pool)); + return svn_io_file_close(file, pool); +} + +/* Find out what the next unique node-id and copy-id are for + transaction TXN_ID in filesystem FS. Store the results in *NODE_ID + and *COPY_ID. The next node-id is used both for creating new unique + nodes for the given transaction, as well as uniquifying representations. + Perform all allocations in POOL. */ +static svn_error_t * +read_next_ids(apr_uint64_t *node_id, + apr_uint64_t *copy_id, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + svn_stringbuf_t *buf; + const char *str; + SVN_ERR(svn_fs_x__read_content(&buf, + svn_fs_x__path_txn_next_ids(fs, txn_id, pool), + pool)); + + /* Parse this into two separate strings. */ + + str = buf->data; + *node_id = svn__base36toui64(&str, str); + if (*str != ' ') + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("next-id file corrupt")); + + *copy_id = svn__base36toui64(&str, ++str); + if (*str != '\n') + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("next-id file corrupt")); + + return SVN_NO_ERROR; +} + +/* Get a new and unique to this transaction node-id for transaction + TXN_ID in filesystem FS. Store the new node-id in *NODE_ID_P. + Node-ids are guaranteed to be unique to this transction, but may + not necessarily be sequential. Perform all allocations in POOL. */ +static svn_error_t * +get_new_txn_node_id(svn_fs_x__id_part_t *node_id_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + apr_uint64_t node_id, copy_id; + + /* First read in the current next-ids file. */ + SVN_ERR(read_next_ids(&node_id, ©_id, fs, txn_id, pool)); + + node_id_p->revision = SVN_INVALID_REVNUM; + node_id_p->number = node_id; + + SVN_ERR(write_next_ids(fs, txn_id, ++node_id, copy_id, pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__reserve_copy_id(svn_fs_x__id_part_t *copy_id_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + apr_uint64_t node_id, copy_id; + + /* First read in the current next-ids file. */ + SVN_ERR(read_next_ids(&node_id, ©_id, fs, txn_id, pool)); + + copy_id_p->revision = SVN_INVALID_REVNUM; + copy_id_p->number = copy_id; + + SVN_ERR(write_next_ids(fs, txn_id, node_id, ++copy_id, pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__create_node(const svn_fs_id_t **id_p, + svn_fs_t *fs, + node_revision_t *noderev, + const svn_fs_x__id_part_t *copy_id, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + svn_fs_x__id_part_t node_id; + const svn_fs_id_t *id; + + /* Get a new node-id for this node. */ + SVN_ERR(get_new_txn_node_id(&node_id, fs, txn_id, pool)); + + id = svn_fs_x__id_txn_create(&node_id, copy_id, txn_id, pool); + + noderev->id = id; + + SVN_ERR(svn_fs_x__put_node_revision(fs, noderev->id, noderev, FALSE, pool)); + + *id_p = id; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__purge_txn(svn_fs_t *fs, + const char *txn_id_str, + apr_pool_t *pool) +{ + svn_fs_x__id_part_t txn_id; + SVN_ERR(svn_fs_x__id_txn_parse(&txn_id, txn_id_str)); + + /* Remove the shared transaction object associated with this transaction. */ + SVN_ERR(purge_shared_txn(fs, &txn_id, pool)); + /* Remove the directory associated with this transaction. */ + SVN_ERR(svn_io_remove_dir2(svn_fs_x__path_txn_dir(fs, &txn_id, pool), + FALSE, NULL, NULL, pool)); + + /* Delete protorev and its lock, which aren't in the txn + directory. It's OK if they don't exist (for example, if this + is post-commit and the proto-rev has been moved into + place). */ + SVN_ERR(svn_io_remove_file2( + svn_fs_x__path_txn_proto_rev(fs, &txn_id, pool), + TRUE, pool)); + SVN_ERR(svn_io_remove_file2( + svn_fs_x__path_txn_proto_rev_lock(fs, &txn_id, pool), + TRUE, pool)); + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__abort_txn(svn_fs_txn_t *txn, + apr_pool_t *pool) +{ + SVN_ERR(svn_fs__check_fs(txn->fs, TRUE)); + + /* Now, purge the transaction. */ + SVN_ERR_W(svn_fs_x__purge_txn(txn->fs, txn->id, pool), + apr_psprintf(pool, _("Transaction '%s' cleanup failed"), + txn->id)); + + return SVN_NO_ERROR; +} + +/* Assign the UNIQUIFIER member of REP based on the current state of TXN_ID + * in FS. Allocate the uniquifier in POOL. + */ +static svn_error_t * +set_uniquifier(svn_fs_t *fs, + representation_t *rep, + apr_pool_t *pool) +{ + svn_fs_x__id_part_t temp; + + SVN_ERR(get_new_txn_node_id(&temp, fs, &rep->txn_id, pool)); + rep->uniquifier.txn_id = rep->txn_id; + rep->uniquifier.number = temp.number; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__set_entry(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + node_revision_t *parent_noderev, + const char *name, + const svn_fs_id_t *id, + svn_node_kind_t kind, + apr_pool_t *pool) +{ + representation_t *rep = parent_noderev->data_rep; + const char *filename + = svn_fs_x__path_txn_node_children(fs, parent_noderev->id, pool); + apr_file_t *file; + svn_stream_t *out; + fs_x_data_t *ffd = fs->fsap_data; + apr_pool_t *subpool = svn_pool_create(pool); + + if (!rep || !svn_fs_x__id_txn_used(&rep->txn_id)) + { + apr_hash_t *entries; + + /* Before we can modify the directory, we need to dump its old + contents into a mutable representation file. */ + SVN_ERR(svn_fs_x__rep_contents_dir(&entries, fs, parent_noderev, + subpool)); + SVN_ERR(unparse_dir_entries(&entries, entries, subpool)); + SVN_ERR(svn_io_file_open(&file, filename, + APR_WRITE | APR_CREATE | APR_BUFFERED, + APR_OS_DEFAULT, pool)); + out = svn_stream_from_aprfile2(file, TRUE, pool); + SVN_ERR(svn_hash_write2(entries, out, SVN_HASH_TERMINATOR, subpool)); + + svn_pool_clear(subpool); + + /* Mark the node-rev's data rep as mutable. */ + rep = apr_pcalloc(pool, sizeof(*rep)); + rep->revision = SVN_INVALID_REVNUM; + rep->txn_id = *txn_id; + SVN_ERR(set_uniquifier(fs, rep, pool)); + parent_noderev->data_rep = rep; + SVN_ERR(svn_fs_x__put_node_revision(fs, parent_noderev->id, + parent_noderev, FALSE, pool)); + } + else + { + /* The directory rep is already mutable, so just open it for append. */ + SVN_ERR(svn_io_file_open(&file, filename, APR_WRITE | APR_APPEND, + APR_OS_DEFAULT, pool)); + out = svn_stream_from_aprfile2(file, TRUE, pool); + } + + /* if we have a directory cache for this transaction, update it */ + if (ffd->txn_dir_cache) + { + /* build parameters: (name, new entry) pair */ + const char *key = + svn_fs_x__id_unparse(parent_noderev->id, subpool)->data; + replace_baton_t baton; + + baton.name = name; + baton.new_entry = NULL; + + if (id) + { + baton.new_entry = apr_pcalloc(subpool, sizeof(*baton.new_entry)); + baton.new_entry->name = name; + baton.new_entry->kind = kind; + baton.new_entry->id = id; + } + + /* actually update the cached directory (if cached) */ + SVN_ERR(svn_cache__set_partial(ffd->txn_dir_cache, key, + svn_fs_x__replace_dir_entry, &baton, + subpool)); + } + svn_pool_clear(subpool); + + /* Append an incremental hash entry for the entry change. */ + if (id) + { + const char *val = unparse_dir_entry(kind, id, subpool); + + SVN_ERR(svn_stream_printf(out, subpool, "K %" APR_SIZE_T_FMT "\n%s\n" + "V %" APR_SIZE_T_FMT "\n%s\n", + strlen(name), name, + strlen(val), val)); + } + else + { + SVN_ERR(svn_stream_printf(out, subpool, "D %" APR_SIZE_T_FMT "\n%s\n", + strlen(name), name)); + } + + SVN_ERR(svn_io_file_close(file, subpool)); + svn_pool_destroy(subpool); + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__add_change(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + const char *path, + const svn_fs_id_t *id, + svn_fs_path_change_kind_t change_kind, + svn_boolean_t text_mod, + svn_boolean_t prop_mod, + svn_node_kind_t node_kind, + svn_revnum_t copyfrom_rev, + const char *copyfrom_path, + apr_pool_t *pool) +{ + apr_file_t *file; + svn_fs_path_change2_t *change; + apr_hash_t *changes = apr_hash_make(pool); + + SVN_ERR(svn_io_file_open(&file, + svn_fs_x__path_txn_changes(fs, txn_id, pool), + APR_APPEND | APR_WRITE | APR_CREATE + | APR_BUFFERED, APR_OS_DEFAULT, pool)); + + change = svn_fs__path_change_create_internal(id, change_kind, pool); + change->text_mod = text_mod; + change->prop_mod = prop_mod; + change->node_kind = node_kind; + change->copyfrom_rev = copyfrom_rev; + change->copyfrom_path = apr_pstrdup(pool, copyfrom_path); + + svn_hash_sets(changes, path, change); + SVN_ERR(svn_fs_x__write_changes(svn_stream_from_aprfile2(file, TRUE, pool), + fs, changes, FALSE, pool)); + + return svn_io_file_close(file, pool); +} + +/* If it is supported by the format of file system FS, store the (ITEM_INDEX, + * OFFSET) pair in the log-to-phys proto index file of transaction TXN_ID. + * Use POOL for allocations. + */ +static svn_error_t * +store_l2p_index_entry(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_off_t offset, + apr_uint64_t item_index, + apr_pool_t *pool) +{ + const char *path = svn_fs_x__path_l2p_proto_index(fs, txn_id, pool); + apr_file_t *file; + SVN_ERR(svn_fs_x__l2p_proto_index_open(&file, path, pool)); + SVN_ERR(svn_fs_x__l2p_proto_index_add_entry(file, offset, 0, + item_index, pool)); + SVN_ERR(svn_io_file_close(file, pool)); + + return SVN_NO_ERROR; +} + +/* If it is supported by the format of file system FS, store ENTRY in the + * phys-to-log proto index file of transaction TXN_ID. + * Use POOL for allocations. + */ +static svn_error_t * +store_p2l_index_entry(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + svn_fs_x__p2l_entry_t *entry, + apr_pool_t *pool) +{ + const char *path = svn_fs_x__path_p2l_proto_index(fs, txn_id, pool); + apr_file_t *file; + SVN_ERR(svn_fs_x__p2l_proto_index_open(&file, path, pool)); + SVN_ERR(svn_fs_x__p2l_proto_index_add_entry(file, entry, pool)); + SVN_ERR(svn_io_file_close(file, pool)); + + return SVN_NO_ERROR; +} + +/* Allocate an item index for the given MY_OFFSET in the transaction TXN_ID + * of file system FS and return it in *ITEM_INDEX. For old formats, it + * will simply return the offset as item index; in new formats, it will + * increment the txn's item index counter file and store the mapping in + * the proto index file. + * Use POOL for allocations. + */ +static svn_error_t * +allocate_item_index(apr_uint64_t *item_index, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_off_t my_offset, + apr_pool_t *pool) +{ + apr_file_t *file; + char buffer[SVN_INT64_BUFFER_SIZE] = { 0 }; + svn_boolean_t eof = FALSE; + apr_size_t to_write; + apr_size_t read; + apr_off_t offset = 0; + + /* read number, increment it and write it back to disk */ + SVN_ERR(svn_io_file_open(&file, + svn_fs_x__path_txn_item_index(fs, txn_id, pool), + APR_READ | APR_WRITE + | APR_CREATE | APR_BUFFERED, + APR_OS_DEFAULT, pool)); + SVN_ERR(svn_io_file_read_full2(file, buffer, sizeof(buffer)-1, + &read, &eof, pool)); + if (read) + SVN_ERR(svn_cstring_atoui64(item_index, buffer)); + else + *item_index = SVN_FS_X__ITEM_INDEX_FIRST_USER; + + to_write = svn__ui64toa(buffer, *item_index + 1); + SVN_ERR(svn_io_file_seek(file, SEEK_SET, &offset, pool)); + SVN_ERR(svn_io_file_write_full(file, buffer, to_write, NULL, pool)); + SVN_ERR(svn_io_file_close(file, pool)); + + /* write log-to-phys index */ + SVN_ERR(store_l2p_index_entry(fs, txn_id, my_offset, *item_index, + pool)); + + return SVN_NO_ERROR; +} + +/* This baton is used by the representation writing streams. It keeps + track of the checksum information as well as the total size of the + representation so far. */ +struct rep_write_baton +{ + /* The FS we are writing to. */ + svn_fs_t *fs; + + /* Actual file to which we are writing. */ + svn_stream_t *rep_stream; + + /* A stream from the delta combiner. Data written here gets + deltified, then eventually written to rep_stream. */ + svn_stream_t *delta_stream; + + /* Where is this representation header stored. */ + apr_off_t rep_offset; + + /* Start of the actual data. */ + apr_off_t delta_start; + + /* How many bytes have been written to this rep already. */ + svn_filesize_t rep_size; + + /* The node revision for which we're writing out info. */ + node_revision_t *noderev; + + /* Actual output file. */ + apr_file_t *file; + /* Lock 'cookie' used to unlock the output file once we've finished + writing to it. */ + void *lockcookie; + + svn_checksum_ctx_t *md5_checksum_ctx; + svn_checksum_ctx_t *sha1_checksum_ctx; + + apr_pool_t *pool; + + apr_pool_t *parent_pool; +}; + +/* Handler for the write method of the representation writable stream. + BATON is a rep_write_baton, DATA is the data to write, and *LEN is + the length of this data. */ +static svn_error_t * +rep_write_contents(void *baton, + const char *data, + apr_size_t *len) +{ + struct rep_write_baton *b = baton; + + SVN_ERR(svn_checksum_update(b->md5_checksum_ctx, data, *len)); + SVN_ERR(svn_checksum_update(b->sha1_checksum_ctx, data, *len)); + b->rep_size += *len; + + /* If we are writing a delta, use that stream. */ + if (b->delta_stream) + return svn_stream_write(b->delta_stream, data, len); + else + return svn_stream_write(b->rep_stream, data, len); +} + +/* Given a node-revision NODEREV in filesystem FS, return the + representation in *REP to use as the base for a text representation + delta if PROPS is FALSE. If PROPS has been set, a suitable props + base representation will be returned. Perform temporary allocations + in *POOL. */ +static svn_error_t * +choose_delta_base(representation_t **rep, + svn_fs_t *fs, + node_revision_t *noderev, + svn_boolean_t props, + apr_pool_t *pool) +{ + /* The zero-based index (counting from the "oldest" end), along NODEREVs line + * predecessors, of the node-rev we will use as delta base. */ + int count; + /* The length of the linear part of a delta chain. (Delta chains use + * skip-delta bits for the high-order bits and are linear in the low-order + * bits.) */ + int walk; + node_revision_t *base; + fs_x_data_t *ffd = fs->fsap_data; + svn_boolean_t maybe_shared_rep = FALSE; + + /* If we have no predecessors, then use the empty stream as a + base. */ + if (! noderev->predecessor_count) + { + *rep = NULL; + return SVN_NO_ERROR; + } + + /* Flip the rightmost '1' bit of the predecessor count to determine + which file rev (counting from 0) we want to use. (To see why + count & (count - 1) unsets the rightmost set bit, think about how + you decrement a binary number.) */ + count = noderev->predecessor_count; + count = count & (count - 1); + + /* We use skip delta for limiting the number of delta operations + along very long node histories. Close to HEAD however, we create + a linear history to minimize delta size. */ + walk = noderev->predecessor_count - count; + if (walk < (int)ffd->max_linear_deltification) + count = noderev->predecessor_count - 1; + + /* Finding the delta base over a very long distance can become extremely + expensive for very deep histories, possibly causing client timeouts etc. + OTOH, this is a rare operation and its gains are minimal. Lets simply + start deltification anew close every other 1000 changes or so. */ + if (walk > (int)ffd->max_deltification_walk) + { + *rep = NULL; + return SVN_NO_ERROR; + } + + /* Walk back a number of predecessors equal to the difference + between count and the original predecessor count. (For example, + if noderev has ten predecessors and we want the eighth file rev, + walk back two predecessors.) */ + base = noderev; + while ((count++) < noderev->predecessor_count) + { + svn_revnum_t base_revision; + SVN_ERR(svn_fs_x__get_node_revision(&base, fs, + base->predecessor_id, pool)); + + /* If there is a shared rep along the way, we need to limit the + * length of the deltification chain. + * + * Please note that copied nodes - such as branch directories - will + * look the same (false positive) while reps shared within the same + * revision will not be caught (false negative). + * + * Message-ID: <CA+t0gk1wzitkih3GRCLDvK-bTEm=hgppGb_7xXMtvuXDYPfL+Q@mail.gmail.com> + */ + base_revision = svn_fs_x__id_rev(base->id); + if (props) + { + if (base->prop_rep && base_revision > base->prop_rep->revision) + maybe_shared_rep = TRUE; + } + else + { + if (base->data_rep && base_revision > base->data_rep->revision) + maybe_shared_rep = TRUE; + } + } + + /* return a suitable base representation */ + *rep = props ? base->prop_rep : base->data_rep; + + /* if we encountered a shared rep, its parent chain may be different + * from the node-rev parent chain. */ + if (*rep && maybe_shared_rep) + { + int chain_length = 0; + SVN_ERR(svn_fs_x__rep_chain_length(&chain_length, *rep, fs, pool)); + + /* Some reasonable limit, depending on how acceptable longer linear + * chains are in this repo. Also, allow for some minimal chain. */ + if (chain_length >= 2 * (int)ffd->max_linear_deltification + 2) + *rep = NULL; + } + + return SVN_NO_ERROR; +} + +/* Something went wrong and the pool for the rep write is being + cleared before we've finished writing the rep. So we need + to remove the rep from the protorevfile and we need to unlock + the protorevfile. */ +static apr_status_t +rep_write_cleanup(void *data) +{ + svn_error_t *err; + struct rep_write_baton *b = data; + + /* Truncate and close the protorevfile. */ + err = svn_io_file_trunc(b->file, b->rep_offset, b->pool); + err = svn_error_compose_create(err, svn_io_file_close(b->file, b->pool)); + + /* Remove our lock regardless of any preceeding errors so that the + being_written flag is always removed and stays consistent with the + file lock which will be removed no matter what since the pool is + going away. */ + err = svn_error_compose_create(err, + unlock_proto_rev(b->fs, + svn_fs_x__id_txn_id(b->noderev->id), + b->lockcookie, b->pool)); + if (err) + { + apr_status_t rc = err->apr_err; + svn_error_clear(err); + return rc; + } + + return APR_SUCCESS; +} + + +/* Get a rep_write_baton and store it in *WB_P for the representation + indicated by NODEREV in filesystem FS. Perform allocations in + POOL. Only appropriate for file contents, not for props or + directory contents. */ +static svn_error_t * +rep_write_get_baton(struct rep_write_baton **wb_p, + svn_fs_t *fs, + node_revision_t *noderev, + apr_pool_t *pool) +{ + struct rep_write_baton *b; + apr_file_t *file; + representation_t *base_rep; + svn_stream_t *source; + svn_txdelta_window_handler_t wh; + void *whb; + int diff_version = 1; + svn_fs_x__rep_header_t header = { 0 }; + + b = apr_pcalloc(pool, sizeof(*b)); + + b->sha1_checksum_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool); + b->md5_checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); + + b->fs = fs; + b->parent_pool = pool; + b->pool = svn_pool_create(pool); + b->rep_size = 0; + b->noderev = noderev; + + /* Open the prototype rev file and seek to its end. */ + SVN_ERR(get_writable_proto_rev(&file, &b->lockcookie, + fs, svn_fs_x__id_txn_id(noderev->id), + b->pool)); + + b->file = file; + b->rep_stream = svn_stream_from_aprfile2(file, TRUE, b->pool); + + SVN_ERR(svn_fs_x__get_file_offset(&b->rep_offset, file, b->pool)); + + /* Get the base for this delta. */ + SVN_ERR(choose_delta_base(&base_rep, fs, noderev, FALSE, b->pool)); + SVN_ERR(svn_fs_x__get_contents(&source, fs, base_rep, b->pool)); + + /* Write out the rep header. */ + if (base_rep) + { + header.base_revision = base_rep->revision; + header.base_item_index = base_rep->item_index; + header.base_length = base_rep->size; + header.type = svn_fs_x__rep_delta; + } + else + { + header.type = svn_fs_x__rep_self_delta; + } + SVN_ERR(svn_fs_x__write_rep_header(&header, b->rep_stream, b->pool)); + + /* Now determine the offset of the actual svndiff data. */ + SVN_ERR(svn_fs_x__get_file_offset(&b->delta_start, file, b->pool)); + + /* Cleanup in case something goes wrong. */ + apr_pool_cleanup_register(b->pool, b, rep_write_cleanup, + apr_pool_cleanup_null); + + /* Prepare to write the svndiff data. */ + svn_txdelta_to_svndiff3(&wh, + &whb, + b->rep_stream, + diff_version, + SVN_DELTA_COMPRESSION_LEVEL_DEFAULT, + pool); + + b->delta_stream = svn_txdelta_target_push(wh, whb, source, b->pool); + + *wb_p = b; + + return SVN_NO_ERROR; +} + +/* For REP->SHA1_CHECKSUM, try to find an already existing representation + in FS and return it in *OUT_REP. If no such representation exists or + if rep sharing has been disabled for FS, NULL will be returned. Since + there may be new duplicate representations within the same uncommitted + revision, those can be passed in REPS_HASH (maps a sha1 digest onto + representation_t*), otherwise pass in NULL for REPS_HASH. + POOL will be used for allocations. The lifetime of the returned rep is + limited by both, POOL and REP lifetime. + */ +static svn_error_t * +get_shared_rep(representation_t **old_rep, + svn_fs_t *fs, + representation_t *rep, + apr_hash_t *reps_hash, + apr_pool_t *pool) +{ + svn_error_t *err; + fs_x_data_t *ffd = fs->fsap_data; + + /* Return NULL, if rep sharing has been disabled. */ + *old_rep = NULL; + if (!ffd->rep_sharing_allowed) + return SVN_NO_ERROR; + + /* Check and see if we already have a representation somewhere that's + identical to the one we just wrote out. Start with the hash lookup + because it is cheepest. */ + if (reps_hash) + *old_rep = apr_hash_get(reps_hash, + rep->sha1_digest, + APR_SHA1_DIGESTSIZE); + + /* If we haven't found anything yet, try harder and consult our DB. */ + if (*old_rep == NULL) + { + svn_checksum_t checksum; + checksum.digest = rep->sha1_digest; + checksum.kind = svn_checksum_sha1; + err = svn_fs_x__get_rep_reference(old_rep, fs, &checksum, pool); + /* ### Other error codes that we shouldn't mask out? */ + if (err == SVN_NO_ERROR) + { + if (*old_rep) + SVN_ERR(svn_fs_x__check_rep(*old_rep, fs, NULL, pool)); + } + else if (err->apr_err == SVN_ERR_FS_CORRUPT + || SVN_ERROR_IN_CATEGORY(err->apr_err, + SVN_ERR_MALFUNC_CATEGORY_START)) + { + /* Fatal error; don't mask it. + + In particular, this block is triggered when the rep-cache refers + to revisions in the future. We signal that as a corruption situation + since, once those revisions are less than youngest (because of more + commits), the rep-cache would be invalid. + */ + SVN_ERR(err); + } + else + { + /* Something's wrong with the rep-sharing index. We can continue + without rep-sharing, but warn. + */ + (fs->warning)(fs->warning_baton, err); + svn_error_clear(err); + *old_rep = NULL; + } + } + + /* look for intra-revision matches (usually data reps but not limited + to them in case props happen to look like some data rep) + */ + if (*old_rep == NULL && svn_fs_x__id_txn_used(&rep->txn_id)) + { + svn_node_kind_t kind; + const char *file_name + = svn_fs_x__path_txn_sha1(fs, &rep->txn_id, rep->sha1_digest, pool); + + /* in our txn, is there a rep file named with the wanted SHA1? + If so, read it and use that rep. + */ + SVN_ERR(svn_io_check_path(file_name, &kind, pool)); + if (kind == svn_node_file) + { + svn_stringbuf_t *rep_string; + SVN_ERR(svn_stringbuf_from_file2(&rep_string, file_name, pool)); + SVN_ERR(svn_fs_x__parse_representation(old_rep, rep_string, pool)); + } + } + + /* Add information that is missing in the cached data. */ + if (*old_rep) + { + /* Use the old rep for this content. */ + memcpy((*old_rep)->md5_digest, rep->md5_digest, sizeof(rep->md5_digest)); + (*old_rep)->uniquifier = rep->uniquifier; + } + + return SVN_NO_ERROR; +} + +/* Copy the hash sum calculation results from MD5_CTX, SHA1_CTX into REP. + * Use POOL for allocations. + */ +static svn_error_t * +digests_final(representation_t *rep, + const svn_checksum_ctx_t *md5_ctx, + const svn_checksum_ctx_t *sha1_ctx, + apr_pool_t *pool) +{ + svn_checksum_t *checksum; + + SVN_ERR(svn_checksum_final(&checksum, md5_ctx, pool)); + memcpy(rep->md5_digest, checksum->digest, svn_checksum_size(checksum)); + SVN_ERR(svn_checksum_final(&checksum, sha1_ctx, pool)); + rep->has_sha1 = checksum != NULL; + if (rep->has_sha1) + memcpy(rep->sha1_digest, checksum->digest, svn_checksum_size(checksum)); + + return SVN_NO_ERROR; +} + +/* Close handler for the representation write stream. BATON is a + rep_write_baton. Writes out a new node-rev that correctly + references the representation we just finished writing. */ +static svn_error_t * +rep_write_contents_close(void *baton) +{ + struct rep_write_baton *b = baton; + representation_t *rep; + representation_t *old_rep; + apr_off_t offset; + + rep = apr_pcalloc(b->parent_pool, sizeof(*rep)); + + /* Close our delta stream so the last bits of svndiff are written + out. */ + if (b->delta_stream) + SVN_ERR(svn_stream_close(b->delta_stream)); + + /* Determine the length of the svndiff data. */ + SVN_ERR(svn_fs_x__get_file_offset(&offset, b->file, b->pool)); + rep->size = offset - b->delta_start; + + /* Fill in the rest of the representation field. */ + rep->expanded_size = b->rep_size; + rep->txn_id = *svn_fs_x__id_txn_id(b->noderev->id); + SVN_ERR(set_uniquifier(b->fs, rep, b->pool)); + rep->revision = SVN_INVALID_REVNUM; + + /* Finalize the checksum. */ + SVN_ERR(digests_final(rep, b->md5_checksum_ctx, b->sha1_checksum_ctx, + b->parent_pool)); + + /* Check and see if we already have a representation somewhere that's + identical to the one we just wrote out. */ + SVN_ERR(get_shared_rep(&old_rep, b->fs, rep, NULL, b->parent_pool)); + + if (old_rep) + { + /* We need to erase from the protorev the data we just wrote. */ + SVN_ERR(svn_io_file_trunc(b->file, b->rep_offset, b->pool)); + + /* Use the old rep for this content. */ + b->noderev->data_rep = old_rep; + } + else + { + /* Write out our cosmetic end marker. */ + SVN_ERR(svn_stream_puts(b->rep_stream, "ENDREP\n")); + SVN_ERR(allocate_item_index(&rep->item_index, b->fs, &rep->txn_id, + b->rep_offset, b->pool)); + + b->noderev->data_rep = rep; + } + + /* Remove cleanup callback. */ + apr_pool_cleanup_kill(b->pool, b, rep_write_cleanup); + + /* Write out the new node-rev information. */ + SVN_ERR(svn_fs_x__put_node_revision(b->fs, b->noderev->id, b->noderev, + FALSE, b->pool)); + if (!old_rep) + { + svn_fs_x__p2l_entry_t entry; + svn_fs_x__id_part_t rev_item; + rev_item.revision = SVN_INVALID_REVNUM; + rev_item.number = rep->item_index; + + entry.offset = b->rep_offset; + SVN_ERR(svn_fs_x__get_file_offset(&offset, b->file, b->pool)); + entry.size = offset - b->rep_offset; + entry.type = SVN_FS_X__ITEM_TYPE_FILE_REP; + entry.item_count = 1; + entry.items = &rev_item; + + SVN_ERR(store_sha1_rep_mapping(b->fs, b->noderev, b->pool)); + SVN_ERR(store_p2l_index_entry(b->fs, &rep->txn_id, &entry, b->pool)); + } + + SVN_ERR(svn_io_file_close(b->file, b->pool)); + SVN_ERR(unlock_proto_rev(b->fs, &rep->txn_id, b->lockcookie, b->pool)); + svn_pool_destroy(b->pool); + + return SVN_NO_ERROR; +} + +/* Store a writable stream in *CONTENTS_P that will receive all data + written and store it as the file data representation referenced by + NODEREV in filesystem FS. Perform temporary allocations in + POOL. Only appropriate for file data, not props or directory + contents. */ +static svn_error_t * +set_representation(svn_stream_t **contents_p, + svn_fs_t *fs, + node_revision_t *noderev, + apr_pool_t *pool) +{ + struct rep_write_baton *wb; + + if (! svn_fs_x__id_is_txn(noderev->id)) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Attempted to write to non-transaction '%s'"), + svn_fs_x__id_unparse(noderev->id, pool)->data); + + SVN_ERR(rep_write_get_baton(&wb, fs, noderev, pool)); + + *contents_p = svn_stream_create(wb, pool); + svn_stream_set_write(*contents_p, rep_write_contents); + svn_stream_set_close(*contents_p, rep_write_contents_close); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__set_contents(svn_stream_t **stream, + svn_fs_t *fs, + node_revision_t *noderev, + apr_pool_t *pool) +{ + if (noderev->kind != svn_node_file) + return svn_error_create(SVN_ERR_FS_NOT_FILE, NULL, + _("Can't set text contents of a directory")); + + return set_representation(stream, fs, noderev, pool); +} + +svn_error_t * +svn_fs_x__create_successor(const svn_fs_id_t **new_id_p, + svn_fs_t *fs, + const svn_fs_id_t *old_idp, + node_revision_t *new_noderev, + const svn_fs_x__id_part_t *copy_id, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + const svn_fs_id_t *id; + + if (! copy_id) + copy_id = svn_fs_x__id_copy_id(old_idp); + id = svn_fs_x__id_txn_create(svn_fs_x__id_node_id(old_idp), copy_id, + txn_id, pool); + + new_noderev->id = id; + + if (! new_noderev->copyroot_path) + { + new_noderev->copyroot_path = apr_pstrdup(pool, + new_noderev->created_path); + new_noderev->copyroot_rev = svn_fs_x__id_rev(new_noderev->id); + } + + SVN_ERR(svn_fs_x__put_node_revision(fs, new_noderev->id, new_noderev, FALSE, + pool)); + + *new_id_p = id; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__set_proplist(svn_fs_t *fs, + node_revision_t *noderev, + apr_hash_t *proplist, + apr_pool_t *pool) +{ + const char *filename + = svn_fs_x__path_txn_node_props(fs, noderev->id, pool); + apr_file_t *file; + svn_stream_t *out; + + /* Dump the property list to the mutable property file. */ + SVN_ERR(svn_io_file_open(&file, filename, + APR_WRITE | APR_CREATE | APR_TRUNCATE + | APR_BUFFERED, APR_OS_DEFAULT, pool)); + out = svn_stream_from_aprfile2(file, TRUE, pool); + SVN_ERR(svn_hash_write2(proplist, out, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_io_file_close(file, pool)); + + /* Mark the node-rev's prop rep as mutable, if not already done. */ + if (!noderev->prop_rep + || !svn_fs_x__id_txn_used(&noderev->prop_rep->txn_id)) + { + noderev->prop_rep = apr_pcalloc(pool, sizeof(*noderev->prop_rep)); + noderev->prop_rep->txn_id = *svn_fs_x__id_txn_id(noderev->id); + SVN_ERR(svn_fs_x__put_node_revision(fs, noderev->id, noderev, FALSE, + pool)); + } + + return SVN_NO_ERROR; +} + +/* This baton is used by the stream created for write_hash_rep. */ +struct write_hash_baton +{ + svn_stream_t *stream; + + apr_size_t size; + + svn_checksum_ctx_t *md5_ctx; + svn_checksum_ctx_t *sha1_ctx; +}; + +/* The handler for the write_hash_rep stream. BATON is a + write_hash_baton, DATA has the data to write and *LEN is the number + of bytes to write. */ +static svn_error_t * +write_hash_handler(void *baton, + const char *data, + apr_size_t *len) +{ + struct write_hash_baton *whb = baton; + + SVN_ERR(svn_checksum_update(whb->md5_ctx, data, *len)); + SVN_ERR(svn_checksum_update(whb->sha1_ctx, data, *len)); + + SVN_ERR(svn_stream_write(whb->stream, data, len)); + whb->size += *len; + + return SVN_NO_ERROR; +} + +/* Write out the hash HASH pertaining to the NODEREV in FS as a deltified + text representation to file FILE. In the process, record the total size + and the md5 digest in REP. If rep sharing has been enabled and REPS_HASH + is not NULL, it will be used in addition to the on-disk cache to find + earlier reps with the same content. When such existing reps can be found, + we will truncate the one just written from the file and return the existing + rep. If PROPS is set, assume that we want to a props representation as + the base for our delta. Perform temporary allocations in POOL. */ +static svn_error_t * +write_hash_delta_rep(representation_t *rep, + apr_file_t *file, + apr_hash_t *hash, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + node_revision_t *noderev, + apr_hash_t *reps_hash, + int item_type, + apr_pool_t *pool) +{ + svn_txdelta_window_handler_t diff_wh; + void *diff_whb; + + svn_stream_t *file_stream; + svn_stream_t *stream; + representation_t *base_rep; + representation_t *old_rep; + svn_stream_t *source; + svn_fs_x__rep_header_t header = { 0 }; + + apr_off_t rep_end = 0; + apr_off_t delta_start = 0; + apr_off_t offset = 0; + + struct write_hash_baton *whb; + int diff_version = 1; + svn_boolean_t is_props = (item_type == SVN_FS_X__ITEM_TYPE_FILE_PROPS) + || (item_type == SVN_FS_X__ITEM_TYPE_DIR_PROPS); + + /* Get the base for this delta. */ + SVN_ERR(choose_delta_base(&base_rep, fs, noderev, is_props, pool)); + SVN_ERR(svn_fs_x__get_contents(&source, fs, base_rep, pool)); + + SVN_ERR(svn_fs_x__get_file_offset(&offset, file, pool)); + + /* Write out the rep header. */ + if (base_rep) + { + header.base_revision = base_rep->revision; + header.base_item_index = base_rep->item_index; + header.base_length = base_rep->size; + header.type = svn_fs_x__rep_delta; + } + else + { + header.type = svn_fs_x__rep_self_delta; + } + + file_stream = svn_stream_from_aprfile2(file, TRUE, pool); + SVN_ERR(svn_fs_x__write_rep_header(&header, file_stream, pool)); + SVN_ERR(svn_fs_x__get_file_offset(&delta_start, file, pool)); + + /* Prepare to write the svndiff data. */ + svn_txdelta_to_svndiff3(&diff_wh, + &diff_whb, + file_stream, + diff_version, + SVN_DELTA_COMPRESSION_LEVEL_DEFAULT, + pool); + + whb = apr_pcalloc(pool, sizeof(*whb)); + whb->stream = svn_txdelta_target_push(diff_wh, diff_whb, source, pool); + whb->size = 0; + whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); + whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool); + + /* serialize the hash */ + stream = svn_stream_create(whb, pool); + svn_stream_set_write(stream, write_hash_handler); + + SVN_ERR(svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_stream_close(whb->stream)); + + /* Store the results. */ + SVN_ERR(digests_final(rep, whb->md5_ctx, whb->sha1_ctx, pool)); + + /* Check and see if we already have a representation somewhere that's + identical to the one we just wrote out. */ + SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, pool)); + + if (old_rep) + { + /* We need to erase from the protorev the data we just wrote. */ + SVN_ERR(svn_io_file_trunc(file, offset, pool)); + + /* Use the old rep for this content. */ + memcpy(rep, old_rep, sizeof (*rep)); + } + else + { + svn_fs_x__p2l_entry_t entry; + svn_fs_x__id_part_t rev_item; + + /* Write out our cosmetic end marker. */ + SVN_ERR(svn_fs_x__get_file_offset(&rep_end, file, pool)); + SVN_ERR(svn_stream_puts(file_stream, "ENDREP\n")); + + SVN_ERR(allocate_item_index(&rep->item_index, fs, txn_id, offset, + pool)); + + rev_item.revision = SVN_INVALID_REVNUM; + rev_item.number = rep->item_index; + + entry.offset = offset; + SVN_ERR(svn_fs_x__get_file_offset(&offset, file, pool)); + entry.size = offset - entry.offset; + entry.type = item_type; + entry.item_count = 1; + entry.items = &rev_item; + + SVN_ERR(store_p2l_index_entry(fs, txn_id, &entry, pool)); + + /* update the representation */ + rep->expanded_size = whb->size; + rep->size = rep_end - delta_start; + } + + return SVN_NO_ERROR; +} + +/* Sanity check ROOT_NODEREV, a candidate for being the root node-revision + of (not yet committed) revision REV in FS. Use POOL for temporary + allocations. + + If you change this function, consider updating svn_fs_x__verify() too. + */ +static svn_error_t * +validate_root_noderev(svn_fs_t *fs, + node_revision_t *root_noderev, + svn_revnum_t rev, + apr_pool_t *pool) +{ + svn_revnum_t head_revnum = rev-1; + int head_predecessor_count; + + SVN_ERR_ASSERT(rev > 0); + + /* Compute HEAD_PREDECESSOR_COUNT. */ + { + svn_fs_root_t *head_revision; + const svn_fs_id_t *head_root_id; + node_revision_t *head_root_noderev; + + /* Get /@HEAD's noderev. */ + SVN_ERR(svn_fs_x__revision_root(&head_revision, fs, head_revnum, pool)); + SVN_ERR(svn_fs_x__node_id(&head_root_id, head_revision, "/", pool)); + SVN_ERR(svn_fs_x__get_node_revision(&head_root_noderev, fs, head_root_id, + pool)); + + head_predecessor_count = head_root_noderev->predecessor_count; + } + + /* Check that the root noderev's predecessor count equals REV. + + This kind of corruption was seen on svn.apache.org (both on + the root noderev and on other fspaths' noderevs); see + issue #4129. + + Normally (rev == root_noderev->predecessor_count), but here we + use a more roundabout check that should only trigger on new instances + of the corruption, rather then trigger on each and every new commit + to a repository that has triggered the bug somewhere in its root + noderev's history. + */ + if (root_noderev->predecessor_count != -1 + && (root_noderev->predecessor_count - head_predecessor_count) + != (rev - head_revnum)) + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("predecessor count for " + "the root node-revision is wrong: " + "found (%d+%ld != %d), committing r%ld"), + head_predecessor_count, + rev - head_revnum, /* This is equal to 1. */ + root_noderev->predecessor_count, + rev); + } + + return SVN_NO_ERROR; +} + +/* Given the potentially txn-local id PART, update that to a permanent ID + * based on the REVISION currently being written and the START_ID for that + * revision. Use the repo FORMAT to decide which implementation to use. + */ +static void +get_final_id(svn_fs_x__id_part_t *part, + svn_revnum_t revision, + apr_uint64_t start_id, + int format) +{ + if (part->revision == SVN_INVALID_REVNUM) + part->revision = revision; +} + +/* Copy a node-revision specified by id ID in fileystem FS from a + transaction into the proto-rev-file FILE. Set *NEW_ID_P to a + pointer to the new node-id which will be allocated in POOL. + If this is a directory, copy all children as well. + + START_NODE_ID and START_COPY_ID are + the first available node and copy ids for this filesystem, for older + FS formats. + + REV is the revision number that this proto-rev-file will represent. + + INITIAL_OFFSET is the offset of the proto-rev-file on entry to + commit_body. + + If REPS_TO_CACHE is not NULL, append to it a copy (allocated in + REPS_POOL) of each data rep that is new in this revision. + + If REPS_HASH is not NULL, append copies (allocated in REPS_POOL) + of the representations of each property rep that is new in this + revision. + + AT_ROOT is true if the node revision being written is the root + node-revision. It is only controls additional sanity checking + logic. + + Temporary allocations are also from POOL. */ +static svn_error_t * +write_final_rev(const svn_fs_id_t **new_id_p, + apr_file_t *file, + svn_revnum_t rev, + svn_fs_t *fs, + const svn_fs_id_t *id, + apr_uint64_t start_node_id, + apr_uint64_t start_copy_id, + apr_off_t initial_offset, + apr_array_header_t *reps_to_cache, + apr_hash_t *reps_hash, + apr_pool_t *reps_pool, + svn_boolean_t at_root, + apr_pool_t *pool) +{ + node_revision_t *noderev; + apr_off_t my_offset; + const svn_fs_id_t *new_id; + svn_fs_x__id_part_t node_id, copy_id, rev_item; + fs_x_data_t *ffd = fs->fsap_data; + const svn_fs_x__id_part_t *txn_id = svn_fs_x__id_txn_id(id); + svn_fs_x__p2l_entry_t entry; + + *new_id_p = NULL; + + /* Check to see if this is a transaction node. */ + if (! svn_fs_x__id_is_txn(id)) + return SVN_NO_ERROR; + + SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, id, pool)); + + if (noderev->kind == svn_node_dir) + { + apr_pool_t *subpool; + apr_hash_t *entries, *str_entries; + apr_array_header_t *sorted_entries; + int i; + + /* This is a directory. Write out all the children first. */ + subpool = svn_pool_create(pool); + + SVN_ERR(svn_fs_x__rep_contents_dir(&entries, fs, noderev, pool)); + /* For the sake of the repository administrator sort the entries + so that the final file is deterministic and repeatable, + however the rest of the FSX code doesn't require any + particular order here. */ + sorted_entries = svn_sort__hash(entries, svn_sort_compare_items_lexically, + pool); + for (i = 0; i < sorted_entries->nelts; ++i) + { + svn_fs_dirent_t *dirent = APR_ARRAY_IDX(sorted_entries, i, + svn_sort__item_t).value; + + svn_pool_clear(subpool); + SVN_ERR(write_final_rev(&new_id, file, rev, fs, dirent->id, + start_node_id, start_copy_id, + initial_offset, reps_to_cache, reps_hash, + reps_pool, FALSE, subpool)); + if (new_id && (svn_fs_x__id_rev(new_id) == rev)) + dirent->id = svn_fs_x__id_copy(new_id, pool); + } + svn_pool_destroy(subpool); + + if (noderev->data_rep + && svn_fs_x__id_txn_used(&noderev->data_rep->txn_id)) + { + /* Write out the contents of this directory as a text rep. */ + SVN_ERR(unparse_dir_entries(&str_entries, entries, pool)); + noderev->data_rep->revision = rev; + + SVN_ERR(write_hash_delta_rep(noderev->data_rep, file, + str_entries, fs, txn_id, noderev, + NULL, SVN_FS_X__ITEM_TYPE_DIR_REP, + pool)); + + svn_fs_x__id_txn_reset(&noderev->data_rep->txn_id); + } + } + else + { + /* This is a file. We should make sure the data rep, if it + exists in a "this" state, gets rewritten to our new revision + num. */ + + if (noderev->data_rep + && svn_fs_x__id_txn_used(&noderev->data_rep->txn_id)) + { + svn_fs_x__id_txn_reset(&noderev->data_rep->txn_id); + noderev->data_rep->revision = rev; + } + } + + /* Fix up the property reps. */ + if (noderev->prop_rep + && svn_fs_x__id_txn_used(&noderev->prop_rep->txn_id)) + { + apr_hash_t *proplist; + int item_type = noderev->kind == svn_node_dir + ? SVN_FS_X__ITEM_TYPE_DIR_PROPS + : SVN_FS_X__ITEM_TYPE_FILE_PROPS; + SVN_ERR(svn_fs_x__get_proplist(&proplist, fs, noderev, pool)); + + svn_fs_x__id_txn_reset(&noderev->prop_rep->txn_id); + noderev->prop_rep->revision = rev; + + SVN_ERR(write_hash_delta_rep(noderev->prop_rep, file, + proplist, fs, txn_id, noderev, + reps_hash, item_type, pool)); + } + + /* Convert our temporary ID into a permanent revision one. */ + node_id = *svn_fs_x__id_node_id(noderev->id); + get_final_id(&node_id, rev, start_node_id, ffd->format); + copy_id = *svn_fs_x__id_copy_id(noderev->id); + get_final_id(©_id, rev, start_copy_id, ffd->format); + + if (noderev->copyroot_rev == SVN_INVALID_REVNUM) + noderev->copyroot_rev = rev; + + SVN_ERR(svn_fs_x__get_file_offset(&my_offset, file, pool)); + if (at_root) + { + /* reference the root noderev from the log-to-phys index */ + rev_item.number = SVN_FS_X__ITEM_INDEX_ROOT_NODE; + SVN_ERR(store_l2p_index_entry(fs, txn_id, my_offset, rev_item.number, + pool)); + } + else + SVN_ERR(allocate_item_index(&rev_item.number, fs, txn_id, my_offset, + pool)); + + rev_item.revision = rev; + new_id = svn_fs_x__id_rev_create(&node_id, ©_id, &rev_item, pool); + + noderev->id = new_id; + + if (ffd->rep_sharing_allowed) + { + /* Save the data representation's hash in the rep cache. */ + if ( noderev->data_rep && noderev->kind == svn_node_file + && noderev->data_rep->revision == rev) + { + SVN_ERR_ASSERT(reps_to_cache && reps_pool); + APR_ARRAY_PUSH(reps_to_cache, representation_t *) + = svn_fs_x__rep_copy(noderev->data_rep, reps_pool); + } + + if (noderev->prop_rep && noderev->prop_rep->revision == rev) + { + /* Add new property reps to hash and on-disk cache. */ + representation_t *copy + = svn_fs_x__rep_copy(noderev->prop_rep, reps_pool); + + SVN_ERR_ASSERT(reps_to_cache && reps_pool); + APR_ARRAY_PUSH(reps_to_cache, representation_t *) = copy; + + apr_hash_set(reps_hash, + copy->sha1_digest, + APR_SHA1_DIGESTSIZE, + copy); + } + } + + /* don't serialize SHA1 for dirs to disk (waste of space) */ + if (noderev->data_rep && noderev->kind == svn_node_dir) + noderev->data_rep->has_sha1 = FALSE; + + /* don't serialize SHA1 for props to disk (waste of space) */ + if (noderev->prop_rep) + noderev->prop_rep->has_sha1 = FALSE; + + /* Workaround issue #4031: is-fresh-txn-root in revision files. */ + noderev->is_fresh_txn_root = FALSE; + + /* Write out our new node-revision. */ + if (at_root) + SVN_ERR(validate_root_noderev(fs, noderev, rev, pool)); + + SVN_ERR(svn_fs_x__write_noderev(svn_stream_from_aprfile2(file, TRUE, pool), + noderev, ffd->format, pool)); + + /* reference the root noderev from the log-to-phys index */ + rev_item.revision = SVN_INVALID_REVNUM; + + entry.offset = my_offset; + SVN_ERR(svn_fs_x__get_file_offset(&my_offset, file, pool)); + entry.size = my_offset - entry.offset; + entry.type = SVN_FS_X__ITEM_TYPE_NODEREV; + entry.item_count = 1; + entry.items = &rev_item; + + SVN_ERR(store_p2l_index_entry(fs, txn_id, &entry, pool)); + + /* Return our ID that references the revision file. */ + *new_id_p = noderev->id; + + return SVN_NO_ERROR; +} + +/* Write the changed path info from transaction TXN_ID in filesystem + FS to the permanent rev-file FILE. *OFFSET_P is set the to offset + in the file of the beginning of this information. Perform + temporary allocations in POOL. */ +static svn_error_t * +write_final_changed_path_info(apr_off_t *offset_p, + apr_file_t *file, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + apr_hash_t *changed_paths; + apr_off_t offset; + svn_fs_x__p2l_entry_t entry; + svn_fs_x__id_part_t rev_item + = {SVN_INVALID_REVNUM, SVN_FS_X__ITEM_INDEX_CHANGES}; + + SVN_ERR(svn_fs_x__get_file_offset(&offset, file, pool)); + SVN_ERR(svn_fs_x__txn_changes_fetch(&changed_paths, fs, txn_id, pool)); + SVN_ERR(svn_fs_x__write_changes(svn_stream_from_aprfile2(file, TRUE, pool), + fs, changed_paths, TRUE, pool)); + + *offset_p = offset; + + /* reference changes from the indexes */ + entry.offset = offset; + SVN_ERR(svn_fs_x__get_file_offset(&offset, file, pool)); + entry.size = offset - entry.offset; + entry.type = SVN_FS_X__ITEM_TYPE_CHANGES; + entry.item_count = 1; + entry.items = &rev_item; + + SVN_ERR(store_p2l_index_entry(fs, txn_id, &entry, pool)); + SVN_ERR(store_l2p_index_entry(fs, txn_id, entry.offset, + SVN_FS_X__ITEM_INDEX_CHANGES, pool)); + + return SVN_NO_ERROR; +} + +/* Open a new svn_fs_t handle to FS, set that handle's concept of "current + youngest revision" to NEW_REV, and call svn_fs_x__verify_root() on + NEW_REV's revision root. + + Intended to be called as the very last step in a commit before 'current' + is bumped. This implies that we are holding the write lock. */ +static svn_error_t * +verify_as_revision_before_current_plus_plus(svn_fs_t *fs, + svn_revnum_t new_rev, + apr_pool_t *pool) +{ +#ifdef SVN_DEBUG + fs_x_data_t *ffd = fs->fsap_data; + svn_fs_t *ft; /* fs++ == ft */ + svn_fs_root_t *root; + fs_x_data_t *ft_ffd; + apr_hash_t *fs_config; + + SVN_ERR_ASSERT(ffd->svn_fs_open_); + + /* make sure FT does not simply return data cached by other instances + * but actually retrieves it from disk at least once. + */ + fs_config = apr_hash_make(pool); + svn_hash_sets(fs_config, SVN_FS_CONFIG_FSFS_CACHE_NS, + svn_uuid_generate(pool)); + SVN_ERR(ffd->svn_fs_open_(&ft, fs->path, + fs_config, + pool)); + ft_ffd = ft->fsap_data; + /* Don't let FT consult rep-cache.db, either. */ + ft_ffd->rep_sharing_allowed = FALSE; + + /* Time travel! */ + ft_ffd->youngest_rev_cache = new_rev; + + SVN_ERR(svn_fs_x__revision_root(&root, ft, new_rev, pool)); + SVN_ERR_ASSERT(root->is_txn_root == FALSE && root->rev == new_rev); + SVN_ERR_ASSERT(ft_ffd->youngest_rev_cache == new_rev); + SVN_ERR(svn_fs_x__verify_root(root, pool)); +#endif /* SVN_DEBUG */ + + return SVN_NO_ERROR; +} + +/* Verify that the user registed with FS has all the locks necessary to + permit all the changes associate with TXN_NAME. + The FS write lock is assumed to be held by the caller. */ +static svn_error_t * +verify_locks(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + apr_pool_t *subpool = svn_pool_create(pool); + apr_hash_t *changes; + apr_hash_index_t *hi; + apr_array_header_t *changed_paths; + svn_stringbuf_t *last_recursed = NULL; + int i; + + /* Fetch the changes for this transaction. */ + SVN_ERR(svn_fs_x__txn_changes_fetch(&changes, fs, txn_id, pool)); + + /* Make an array of the changed paths, and sort them depth-first-ily. */ + changed_paths = apr_array_make(pool, apr_hash_count(changes) + 1, + sizeof(const char *)); + for (hi = apr_hash_first(pool, changes); hi; hi = apr_hash_next(hi)) + APR_ARRAY_PUSH(changed_paths, const char *) = svn__apr_hash_index_key(hi); + qsort(changed_paths->elts, changed_paths->nelts, + changed_paths->elt_size, svn_sort_compare_paths); + + /* Now, traverse the array of changed paths, verify locks. Note + that if we need to do a recursive verification a path, we'll skip + over children of that path when we get to them. */ + for (i = 0; i < changed_paths->nelts; i++) + { + const char *path; + svn_fs_path_change2_t *change; + svn_boolean_t recurse = TRUE; + + svn_pool_clear(subpool); + path = APR_ARRAY_IDX(changed_paths, i, const char *); + + /* If this path has already been verified as part of a recursive + check of one of its parents, no need to do it again. */ + if (last_recursed + && svn_dirent_is_child(last_recursed->data, path, subpool)) + continue; + + /* Fetch the change associated with our path. */ + change = svn_hash_gets(changes, path); + + /* What does it mean to succeed at lock verification for a given + path? For an existing file or directory getting modified + (text, props), it means we hold the lock on the file or + directory. For paths being added or removed, we need to hold + the locks for that path and any children of that path. + + WHEW! We have no reliable way to determine the node kind + of deleted items, but fortunately we are going to do a + recursive check on deleted paths regardless of their kind. */ + if (change->change_kind == svn_fs_path_change_modify) + recurse = FALSE; + SVN_ERR(svn_fs_x__allow_locked_operation(path, fs, recurse, TRUE, + subpool)); + + /* If we just did a recursive check, remember the path we + checked (so children can be skipped). */ + if (recurse) + { + if (! last_recursed) + last_recursed = svn_stringbuf_create(path, pool); + else + svn_stringbuf_set(last_recursed, path); + } + } + svn_pool_destroy(subpool); + return SVN_NO_ERROR; +} + +/* Baton used for commit_body below. */ +struct commit_baton { + svn_revnum_t *new_rev_p; + svn_fs_t *fs; + svn_fs_txn_t *txn; + svn_boolean_t set_timestamp; + apr_array_header_t *reps_to_cache; + apr_hash_t *reps_hash; + apr_pool_t *reps_pool; +}; + +/* The work-horse for svn_fs_x__commit, called with the FS write lock. + This implements the svn_fs_x__with_write_lock() 'body' callback + type. BATON is a 'struct commit_baton *'. */ +static svn_error_t * +commit_body(void *baton, apr_pool_t *pool) +{ + struct commit_baton *cb = baton; + fs_x_data_t *ffd = cb->fs->fsap_data; + const char *old_rev_filename, *rev_filename, *proto_filename; + const char *revprop_filename, *final_revprop; + const svn_fs_id_t *root_id, *new_root_id; + apr_uint64_t start_node_id = 0; + apr_uint64_t start_copy_id = 0; + svn_revnum_t old_rev, new_rev; + apr_file_t *proto_file; + void *proto_file_lockcookie; + apr_off_t initial_offset, changed_path_offset; + apr_hash_t *txnprops; + apr_array_header_t *txnprop_list; + svn_prop_t prop; + const svn_fs_x__id_part_t *txn_id = svn_fs_x__txn_get_id(cb->txn); + + /* Get the current youngest revision. */ + SVN_ERR(svn_fs_x__youngest_rev(&old_rev, cb->fs, pool)); + + /* Check to make sure this transaction is based off the most recent + revision. */ + if (cb->txn->base_rev != old_rev) + return svn_error_create(SVN_ERR_FS_TXN_OUT_OF_DATE, NULL, + _("Transaction out of date")); + + /* Locks may have been added (or stolen) between the calling of + previous svn_fs.h functions and svn_fs_commit_txn(), so we need + to re-examine every changed-path in the txn and re-verify all + discovered locks. */ + SVN_ERR(verify_locks(cb->fs, txn_id, pool)); + + /* We are going to be one better than this puny old revision. */ + new_rev = old_rev + 1; + + /* Get a write handle on the proto revision file. */ + SVN_ERR(get_writable_proto_rev(&proto_file, &proto_file_lockcookie, + cb->fs, txn_id, pool)); + SVN_ERR(svn_fs_x__get_file_offset(&initial_offset, proto_file, pool)); + + /* Write out all the node-revisions and directory contents. */ + root_id = svn_fs_x__id_txn_create_root(txn_id, pool); + SVN_ERR(write_final_rev(&new_root_id, proto_file, new_rev, cb->fs, root_id, + start_node_id, start_copy_id, initial_offset, + cb->reps_to_cache, cb->reps_hash, cb->reps_pool, + TRUE, pool)); + + /* Write the changed-path information. */ + SVN_ERR(write_final_changed_path_info(&changed_path_offset, proto_file, + cb->fs, txn_id, pool)); + + SVN_ERR(svn_io_file_flush_to_disk(proto_file, pool)); + SVN_ERR(svn_io_file_close(proto_file, pool)); + + /* We don't unlock the prototype revision file immediately to avoid a + race with another caller writing to the prototype revision file + before we commit it. */ + + /* Remove any temporary txn props representing 'flags'. */ + SVN_ERR(svn_fs_x__txn_proplist(&txnprops, cb->txn, pool)); + txnprop_list = apr_array_make(pool, 3, sizeof(svn_prop_t)); + prop.value = NULL; + + if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_OOD)) + { + prop.name = SVN_FS__PROP_TXN_CHECK_OOD; + APR_ARRAY_PUSH(txnprop_list, svn_prop_t) = prop; + } + + if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_LOCKS)) + { + prop.name = SVN_FS__PROP_TXN_CHECK_LOCKS; + APR_ARRAY_PUSH(txnprop_list, svn_prop_t) = prop; + } + + if (! apr_is_empty_array(txnprop_list)) + SVN_ERR(svn_fs_x__change_txn_props(cb->txn, txnprop_list, pool)); + + /* Create the shard for the rev and revprop file, if we're sharding and + this is the first revision of a new shard. We don't care if this + fails because the shard already existed for some reason. */ + if (new_rev % ffd->max_files_per_dir == 0) + { + /* Create the revs shard. */ + { + const char *new_dir + = svn_fs_x__path_rev_shard(cb->fs, new_rev, pool); + svn_error_t *err = svn_io_dir_make(new_dir, APR_OS_DEFAULT, pool); + if (err && !APR_STATUS_IS_EEXIST(err->apr_err)) + return svn_error_trace(err); + svn_error_clear(err); + SVN_ERR(svn_io_copy_perms(svn_dirent_join(cb->fs->path, + PATH_REVS_DIR, + pool), + new_dir, pool)); + } + + /* Create the revprops shard. */ + SVN_ERR_ASSERT(! svn_fs_x__is_packed_revprop(cb->fs, new_rev)); + { + const char *new_dir + = svn_fs_x__path_revprops_shard(cb->fs, new_rev, pool); + svn_error_t *err = svn_io_dir_make(new_dir, APR_OS_DEFAULT, pool); + if (err && !APR_STATUS_IS_EEXIST(err->apr_err)) + return svn_error_trace(err); + svn_error_clear(err); + SVN_ERR(svn_io_copy_perms(svn_dirent_join(cb->fs->path, + PATH_REVPROPS_DIR, + pool), + new_dir, pool)); + } + } + + /* Convert the index files from the proto format into their form + in their final location */ + SVN_ERR(svn_fs_x__l2p_index_create(cb->fs, + svn_fs_x__path_l2p_index(cb->fs, new_rev, pool), + svn_fs_x__path_l2p_proto_index(cb->fs, txn_id, pool), + new_rev, pool)); + SVN_ERR(svn_fs_x__p2l_index_create(cb->fs, + svn_fs_x__path_p2l_index(cb->fs, new_rev, pool), + svn_fs_x__path_p2l_proto_index(cb->fs, txn_id, pool), + new_rev, pool)); + + /* Move the finished rev file into place. */ + old_rev_filename = svn_fs_x__path_rev_absolute(cb->fs, old_rev, pool); + + rev_filename = svn_fs_x__path_rev(cb->fs, new_rev, pool); + proto_filename = svn_fs_x__path_txn_proto_rev(cb->fs, txn_id, pool); + SVN_ERR(svn_fs_x__move_into_place(proto_filename, rev_filename, + old_rev_filename, pool)); + + /* Now that we've moved the prototype revision file out of the way, + we can unlock it (since further attempts to write to the file + will fail as it no longer exists). We must do this so that we can + remove the transaction directory later. */ + SVN_ERR(unlock_proto_rev(cb->fs, txn_id, proto_file_lockcookie, pool)); + + /* Update commit time to ensure that svn:date revprops remain ordered if + requested. */ + if (cb->set_timestamp) + { + svn_string_t date; + + date.data = svn_time_to_cstring(apr_time_now(), pool); + date.len = strlen(date.data); + + SVN_ERR(svn_fs_x__change_txn_prop(cb->txn, SVN_PROP_REVISION_DATE, + &date, pool)); + } + + /* Move the revprops file into place. */ + SVN_ERR_ASSERT(! svn_fs_x__is_packed_revprop(cb->fs, new_rev)); + revprop_filename = svn_fs_x__path_txn_props(cb->fs, txn_id, pool); + final_revprop = svn_fs_x__path_revprops(cb->fs, new_rev, pool); + SVN_ERR(svn_fs_x__move_into_place(revprop_filename, final_revprop, + old_rev_filename, pool)); + + /* Update the 'current' file. */ + SVN_ERR(verify_as_revision_before_current_plus_plus(cb->fs, new_rev, pool)); + SVN_ERR(svn_fs_x__write_current(cb->fs, new_rev, pool)); + + /* At this point the new revision is committed and globally visible + so let the caller know it succeeded by giving it the new revision + number, which fulfills svn_fs_commit_txn() contract. Any errors + after this point do not change the fact that a new revision was + created. */ + *cb->new_rev_p = new_rev; + + ffd->youngest_rev_cache = new_rev; + + /* Remove this transaction directory. */ + SVN_ERR(svn_fs_x__purge_txn(cb->fs, cb->txn->id, pool)); + + return SVN_NO_ERROR; +} + +/* Add the representations in REPS_TO_CACHE (an array of representation_t *) + * to the rep-cache database of FS. */ +static svn_error_t * +write_reps_to_cache(svn_fs_t *fs, + const apr_array_header_t *reps_to_cache, + apr_pool_t *scratch_pool) +{ + int i; + + for (i = 0; i < reps_to_cache->nelts; i++) + { + representation_t *rep = APR_ARRAY_IDX(reps_to_cache, i, representation_t *); + + /* FALSE because we don't care if another parallel commit happened to + * collide with us. (Non-parallel collisions will not be detected.) */ + SVN_ERR(svn_fs_x__set_rep_reference(fs, rep, FALSE, scratch_pool)); + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__commit(svn_revnum_t *new_rev_p, + svn_fs_t *fs, + svn_fs_txn_t *txn, + svn_boolean_t set_timestamp, + apr_pool_t *pool) +{ + struct commit_baton cb; + fs_x_data_t *ffd = fs->fsap_data; + + cb.new_rev_p = new_rev_p; + cb.fs = fs; + cb.txn = txn; + cb.set_timestamp = set_timestamp; + + if (ffd->rep_sharing_allowed) + { + cb.reps_to_cache = apr_array_make(pool, 5, sizeof(representation_t *)); + cb.reps_hash = apr_hash_make(pool); + cb.reps_pool = pool; + } + else + { + cb.reps_to_cache = NULL; + cb.reps_hash = NULL; + cb.reps_pool = NULL; + } + + SVN_ERR(svn_fs_x__with_write_lock(fs, commit_body, &cb, pool)); + + /* At this point, *NEW_REV_P has been set, so errors below won't affect + the success of the commit. (See svn_fs_commit_txn().) */ + + if (ffd->rep_sharing_allowed) + { + SVN_ERR(svn_fs_x__open_rep_cache(fs, pool)); + + /* Write new entries to the rep-sharing database. + * + * We use an sqlite transaction to speed things up; + * see <http://www.sqlite.org/faq.html#q19>. + */ + /* ### A commit that touches thousands of files will starve other + (reader/writer) commits for the duration of the below call. + Maybe write in batches? */ + SVN_SQLITE__WITH_TXN( + write_reps_to_cache(fs, cb.reps_to_cache, pool), + ffd->rep_cache_db); + } + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__list_transactions(apr_array_header_t **names_p, + svn_fs_t *fs, + apr_pool_t *pool) +{ + const char *txn_dir; + apr_hash_t *dirents; + apr_hash_index_t *hi; + apr_array_header_t *names; + apr_size_t ext_len = strlen(PATH_EXT_TXN); + + names = apr_array_make(pool, 1, sizeof(const char *)); + + /* Get the transactions directory. */ + txn_dir = svn_dirent_join(fs->path, PATH_TXNS_DIR, pool); + + /* Now find a listing of this directory. */ + SVN_ERR(svn_io_get_dirents3(&dirents, txn_dir, TRUE, pool, pool)); + + /* Loop through all the entries and return anything that ends with '.txn'. */ + for (hi = apr_hash_first(pool, dirents); hi; hi = apr_hash_next(hi)) + { + const char *name = svn__apr_hash_index_key(hi); + apr_ssize_t klen = svn__apr_hash_index_klen(hi); + const char *id; + + /* The name must end with ".txn" to be considered a transaction. */ + if ((apr_size_t) klen <= ext_len + || (strcmp(name + klen - ext_len, PATH_EXT_TXN)) != 0) + continue; + + /* Truncate the ".txn" extension and store the ID. */ + id = apr_pstrndup(pool, name, strlen(name) - ext_len); + APR_ARRAY_PUSH(names, const char *) = id; + } + + *names_p = names; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__open_txn(svn_fs_txn_t **txn_p, + svn_fs_t *fs, + const char *name, + apr_pool_t *pool) +{ + svn_fs_txn_t *txn; + fs_txn_data_t *ftd; + svn_node_kind_t kind; + transaction_t *local_txn; + svn_fs_x__id_part_t txn_id; + + SVN_ERR(svn_fs_x__id_txn_parse(&txn_id, name)); + + /* First check to see if the directory exists. */ + SVN_ERR(svn_io_check_path(svn_fs_x__path_txn_dir(fs, &txn_id, pool), + &kind, pool)); + + /* Did we find it? */ + if (kind != svn_node_dir) + return svn_error_createf(SVN_ERR_FS_NO_SUCH_TRANSACTION, NULL, + _("No such transaction '%s'"), + name); + + txn = apr_pcalloc(pool, sizeof(*txn)); + ftd = apr_pcalloc(pool, sizeof(*ftd)); + ftd->txn_id = txn_id; + + /* Read in the root node of this transaction. */ + txn->id = apr_pstrdup(pool, name); + txn->fs = fs; + + SVN_ERR(svn_fs_x__get_txn(&local_txn, fs, &txn_id, pool)); + + txn->base_rev = svn_fs_x__id_rev(local_txn->base_id); + + txn->vtable = &txn_vtable; + txn->fsap_data = ftd; + *txn_p = txn; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__txn_proplist(apr_hash_t **table_p, + svn_fs_txn_t *txn, + apr_pool_t *pool) +{ + apr_hash_t *proplist = apr_hash_make(pool); + SVN_ERR(get_txn_proplist(proplist, txn->fs, svn_fs_x__txn_get_id(txn), + pool)); + *table_p = proplist; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__delete_node_revision(svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool) +{ + node_revision_t *noderev; + + SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, id, pool)); + + /* Delete any mutable property representation. */ + if (noderev->prop_rep + && svn_fs_x__id_txn_used(&noderev->prop_rep->txn_id)) + SVN_ERR(svn_io_remove_file2(svn_fs_x__path_txn_node_props(fs, id, pool), + FALSE, pool)); + + /* Delete any mutable data representation. */ + if (noderev->data_rep + && svn_fs_x__id_txn_used(&noderev->data_rep->txn_id) + && noderev->kind == svn_node_dir) + { + fs_x_data_t *ffd = fs->fsap_data; + SVN_ERR(svn_io_remove_file2(svn_fs_x__path_txn_node_children(fs, id, + pool), + FALSE, pool)); + + /* remove the corresponding entry from the cache, if such exists */ + if (ffd->txn_dir_cache) + { + const char *key = svn_fs_x__id_unparse(id, pool)->data; + SVN_ERR(svn_cache__set(ffd->txn_dir_cache, key, NULL, pool)); + } + } + + return svn_io_remove_file2(svn_fs_x__path_txn_node_rev(fs, id, pool), + FALSE, pool); +} + + + +/*** Transactions ***/ + +svn_error_t * +svn_fs_x__get_txn_ids(const svn_fs_id_t **root_id_p, + const svn_fs_id_t **base_root_id_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + transaction_t *txn; + SVN_ERR(svn_fs_x__get_txn(&txn, fs, txn_id, pool)); + *root_id_p = txn->root_id; + *base_root_id_p = txn->base_id; + return SVN_NO_ERROR; +} + + +/* Generic transaction operations. */ + +svn_error_t * +svn_fs_x__txn_prop(svn_string_t **value_p, + svn_fs_txn_t *txn, + const char *propname, + apr_pool_t *pool) +{ + apr_hash_t *table; + svn_fs_t *fs = txn->fs; + + SVN_ERR(svn_fs__check_fs(fs, TRUE)); + SVN_ERR(svn_fs_x__txn_proplist(&table, txn, pool)); + + *value_p = svn_hash_gets(table, propname); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__begin_txn(svn_fs_txn_t **txn_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_uint32_t flags, + apr_pool_t *pool) +{ + svn_string_t date; + svn_prop_t prop; + apr_array_header_t *props = apr_array_make(pool, 3, sizeof(svn_prop_t)); + + SVN_ERR(svn_fs__check_fs(fs, TRUE)); + + SVN_ERR(svn_fs_x__create_txn(txn_p, fs, rev, pool)); + + /* Put a datestamp on the newly created txn, so we always know + exactly how old it is. (This will help sysadmins identify + long-abandoned txns that may need to be manually removed.) When + a txn is promoted to a revision, this property will be + automatically overwritten with a revision datestamp. */ + date.data = svn_time_to_cstring(apr_time_now(), pool); + date.len = strlen(date.data); + + prop.name = SVN_PROP_REVISION_DATE; + prop.value = &date; + APR_ARRAY_PUSH(props, svn_prop_t) = prop; + + /* Set temporary txn props that represent the requested 'flags' + behaviors. */ + if (flags & SVN_FS_TXN_CHECK_OOD) + { + prop.name = SVN_FS__PROP_TXN_CHECK_OOD; + prop.value = svn_string_create("true", pool); + APR_ARRAY_PUSH(props, svn_prop_t) = prop; + } + + if (flags & SVN_FS_TXN_CHECK_LOCKS) + { + prop.name = SVN_FS__PROP_TXN_CHECK_LOCKS; + prop.value = svn_string_create("true", pool); + APR_ARRAY_PUSH(props, svn_prop_t) = prop; + } + + return svn_fs_x__change_txn_props(*txn_p, props, pool); +}
diff --git a/subversion/libsvn_fs_x/transaction.h b/subversion/libsvn_fs_x/transaction.h new file mode 100644 index 0000000..ed6f3b3 --- /dev/null +++ b/subversion/libsvn_fs_x/transaction.h
@@ -0,0 +1,293 @@ +/* transaction.h --- transaction-related functions of FSX + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS__TRANSACTION_H +#define SVN_LIBSVN_FS__TRANSACTION_H + +#include "fs.h" + +/* Return the transaction ID of TXN. + */ +const svn_fs_x__id_part_t * +svn_fs_x__txn_get_id(svn_fs_txn_t *txn); + +/* Obtain a write lock on the filesystem FS in a subpool of POOL, call + BODY with BATON and that subpool, destroy the subpool (releasing the write + lock) and return what BODY returned. */ +svn_error_t * +svn_fs_x__with_write_lock(svn_fs_t *fs, + svn_error_t *(*body)(void *baton, + apr_pool_t *pool), + void *baton, + apr_pool_t *pool); + +/* Store NODEREV as the node-revision for the node whose id is ID in + FS, after setting its is_fresh_txn_root to FRESH_TXN_ROOT. Do any + necessary temporary allocation in POOL. */ +svn_error_t * +svn_fs_x__put_node_revision(svn_fs_t *fs, + const svn_fs_id_t *id, + node_revision_t *noderev, + svn_boolean_t fresh_txn_root, + apr_pool_t *pool); + +/* Find the paths which were changed in transaction TXN_ID of + filesystem FS and store them in *CHANGED_PATHS_P. + Get any temporary allocations from POOL. */ +svn_error_t * +svn_fs_x__txn_changes_fetch(apr_hash_t **changed_paths_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +/* Find the paths which were changed in revision REV of filesystem FS + and store them in *CHANGED_PATHS_P. Get any temporary allocations + from POOL. */ +svn_error_t * +svn_fs_x__paths_changed(apr_hash_t **changed_paths_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +/* Create a new transaction in filesystem FS, based on revision REV, + and store it in *TXN_P. Allocate all necessary variables from + POOL. */ +svn_error_t * +svn_fs_x__create_txn(svn_fs_txn_t **txn_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +/* Set the transaction property NAME to the value VALUE in transaction + TXN. Perform temporary allocations from POOL. */ +svn_error_t * +svn_fs_x__change_txn_prop(svn_fs_txn_t *txn, + const char *name, + const svn_string_t *value, + apr_pool_t *pool); + +/* Change transaction properties in transaction TXN based on PROPS. + Perform temporary allocations from POOL. */ +svn_error_t * +svn_fs_x__change_txn_props(svn_fs_txn_t *txn, + const apr_array_header_t *props, + apr_pool_t *pool); + +/* Store a transaction record in *TXN_P for the transaction identified + by TXN_ID in filesystem FS. Allocate everything from POOL. */ +svn_error_t * +svn_fs_x__get_txn(transaction_t **txn_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +/* Return the next available copy_id in *COPY_ID for the transaction + TXN_ID in filesystem FS. Allocate space in POOL. */ +svn_error_t * +svn_fs_x__reserve_copy_id(svn_fs_x__id_part_t *copy_id_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +/* Create an entirely new mutable node in the filesystem FS, whose + node-revision is NODEREV. Set *ID_P to the new node revision's ID. + Use POOL for any temporary allocation. COPY_ID is the copy_id to + use in the node revision ID. TXN_ID is the Subversion transaction + under which this occurs. */ +svn_error_t * +svn_fs_x__create_node(const svn_fs_id_t **id_p, + svn_fs_t *fs, + node_revision_t *noderev, + const svn_fs_x__id_part_t *copy_id, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +/* Remove all references to the transaction TXN_ID from filesystem FS. + Temporary allocations are from POOL. */ +svn_error_t * +svn_fs_x__purge_txn(svn_fs_t *fs, + const char *txn_id, + apr_pool_t *pool); + +/* Abort the existing transaction TXN, performing any temporary + allocations in POOL. */ +svn_error_t * +svn_fs_x__abort_txn(svn_fs_txn_t *txn, + apr_pool_t *pool); + +/* Add or set in filesystem FS, transaction TXN_ID, in directory + PARENT_NODEREV a directory entry for NAME pointing to ID of type + KIND. Allocations are done in POOL. */ +svn_error_t * +svn_fs_x__set_entry(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + node_revision_t *parent_noderev, + const char *name, + const svn_fs_id_t *id, + svn_node_kind_t kind, + apr_pool_t *pool); + +/* Add a change to the changes record for filesystem FS in transaction + TXN_ID. Mark path PATH, having node-id ID, as changed according to + the type in CHANGE_KIND. If the text representation was changed + set TEXT_MOD to TRUE, and likewise for PROP_MOD. If this change + was the result of a copy, set COPYFROM_REV and COPYFROM_PATH to the + revision and path of the copy source, otherwise they should be set + to SVN_INVALID_REVNUM and NULL. Perform any temporary allocations + from POOL. */ +svn_error_t * +svn_fs_x__add_change(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + const char *path, + const svn_fs_id_t *id, + svn_fs_path_change_kind_t change_kind, + svn_boolean_t text_mod, + svn_boolean_t prop_mod, + svn_node_kind_t node_kind, + svn_revnum_t copyfrom_rev, + const char *copyfrom_path, + apr_pool_t *pool); + +/* Return a writable stream in *STREAM that allows storing the text + representation of node-revision NODEREV in filesystem FS. + Allocations are from POOL. */ +svn_error_t * +svn_fs_x__set_contents(svn_stream_t **stream, + svn_fs_t *fs, + node_revision_t *noderev, + apr_pool_t *pool); + +/* Create a node revision in FS which is an immediate successor of + OLD_ID, whose contents are NEW_NR. Set *NEW_ID_P to the new node + revision's ID. Use POOL for any temporary allocation. + + COPY_ID, if non-NULL, is a key into the `copies' table, and + indicates that this new node is being created as the result of a + copy operation, and specifically which operation that was. If + COPY_ID is NULL, then re-use the copy ID from the predecessor node. + + TXN_ID is the Subversion transaction under which this occurs. + + After this call, the deltification code assumes that the new node's + contents will change frequently, and will avoid representing other + nodes as deltas against this node's contents. */ +svn_error_t * +svn_fs_x__create_successor(const svn_fs_id_t **new_id_p, + svn_fs_t *fs, + const svn_fs_id_t *old_idp, + node_revision_t *new_noderev, + const svn_fs_x__id_part_t *copy_id, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +/* Write a new property list PROPLIST for node-revision NODEREV in + filesystem FS. Perform any temporary allocations in POOL. */ +svn_error_t * +svn_fs_x__set_proplist(svn_fs_t *fs, + node_revision_t *noderev, + apr_hash_t *proplist, + apr_pool_t *pool); + +/* Commit the transaction TXN in filesystem FS and return its new + revision number in *REV. If the transaction is out of date, return + the error SVN_ERR_FS_TXN_OUT_OF_DATE. Update commit time to ensure that + svn:date revprops remain ordered if SET_TIMESTAMP is non-zero. Use POOL + for temporary allocations. */ +svn_error_t * +svn_fs_x__commit(svn_revnum_t *new_rev_p, + svn_fs_t *fs, + svn_fs_txn_t *txn, + svn_boolean_t set_timestamp, + apr_pool_t *pool); + +/* Set *NAMES_P to an array of names which are all the active + transactions in filesystem FS. Allocate the array from POOL. */ +svn_error_t * +svn_fs_x__list_transactions(apr_array_header_t **names_p, + svn_fs_t *fs, + apr_pool_t *pool); + +/* Open the transaction named NAME in filesystem FS. Set *TXN_P to + * the transaction. If there is no such transaction, return +` * SVN_ERR_FS_NO_SUCH_TRANSACTION. Allocate the new transaction in + * POOL. */ +svn_error_t * +svn_fs_x__open_txn(svn_fs_txn_t **txn_p, + svn_fs_t *fs, + const char *name, + apr_pool_t *pool); + +/* Return the property list from transaction TXN and store it in + *PROPLIST. Allocate the property list from POOL. */ +svn_error_t * +svn_fs_x__txn_proplist(apr_hash_t **table_p, + svn_fs_txn_t *txn, + apr_pool_t *pool); + +/* Delete the mutable node-revision referenced by ID, along with any + mutable props or directory contents associated with it. Perform + temporary allocations in POOL. */ +svn_error_t * +svn_fs_x__delete_node_revision(svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool); + +/* Retrieve information about the Subversion transaction SVN_TXN from + the `transactions' table of FS, allocating from POOL. Set + *ROOT_ID_P to the ID of the transaction's root directory. Set + *BASE_ROOT_ID_P to the ID of the root directory of the + transaction's base revision. + + If there is no such transaction, SVN_ERR_FS_NO_SUCH_TRANSACTION is + the error returned. + + Returns SVN_ERR_FS_TRANSACTION_NOT_MUTABLE if TXN_NAME refers to a + transaction that has already been committed. + + Allocate *ROOT_ID_P and *BASE_ROOT_ID_P in POOL. */ +svn_error_t * +svn_fs_x__get_txn_ids(const svn_fs_id_t **root_id_p, + const svn_fs_id_t **base_root_id_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_name, + apr_pool_t *pool); + +/* Find the value of the property named PROPNAME in transaction TXN. + Return the contents in *VALUE_P. The contents will be allocated + from POOL. */ +svn_error_t * +svn_fs_x__txn_prop(svn_string_t **value_p, + svn_fs_txn_t *txn, + const char *propname, + apr_pool_t *pool); + +/* Begin a new transaction in filesystem FS, based on existing + revision REV. The new transaction is returned in *TXN_P. Allocate + the new transaction structure from POOL. */ +svn_error_t * +svn_fs_x__begin_txn(svn_fs_txn_t **txn_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_uint32_t flags, + apr_pool_t *pool); + +#endif \ No newline at end of file
diff --git a/subversion/libsvn_fs_x/tree.c b/subversion/libsvn_fs_x/tree.c new file mode 100644 index 0000000..bca9024 --- /dev/null +++ b/subversion/libsvn_fs_x/tree.c
@@ -0,0 +1,4358 @@ +/* tree.c : tree-like filesystem, built on DAG filesystem + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + + +/* The job of this layer is to take a filesystem with lots of node + sharing going on --- the real DAG filesystem as it appears in the + database --- and make it look and act like an ordinary tree + filesystem, with no sharing. + + We do just-in-time cloning: you can walk from some unfinished + transaction's root down into directories and files shared with + committed revisions; as soon as you try to change something, the + appropriate nodes get cloned (and parent directory entries updated) + invisibly, behind your back. Any other references you have to + nodes that have been cloned by other changes, even made by other + processes, are automatically updated to point to the right clones. */ + + +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <apr_pools.h> +#include <apr_hash.h> + +#include "svn_hash.h" +#include "svn_private_config.h" +#include "svn_pools.h" +#include "svn_error.h" +#include "svn_path.h" +#include "svn_mergeinfo.h" +#include "svn_fs.h" +#include "svn_props.h" + +#include "fs.h" +#include "dag.h" +#include "lock.h" +#include "tree.h" +#include "fs_x.h" +#include "id.h" +#include "temp_serializer.h" +#include "cached_data.h" +#include "transaction.h" +#include "pack.h" + +#include "private/svn_mergeinfo_private.h" +#include "private/svn_subr_private.h" +#include "private/svn_fs_util.h" +#include "private/svn_fspath.h" +#include "../libsvn_fs/fs-loader.h" + + +/* ### I believe this constant will become internal to reps-strings.c. + ### see the comment in window_consumer() for more information. */ + +/* ### the comment also seems to need tweaking: the log file stuff + ### is no longer an issue... */ +/* Data written to the filesystem through the svn_fs_apply_textdelta() + interface is cached in memory until the end of the data stream, or + until a size trigger is hit. Define that trigger here (in bytes). + Setting the value to 0 will result in no filesystem buffering at + all. The value only really matters when dealing with file contents + bigger than the value itself. Above that point, large values here + allow the filesystem to buffer more data in memory before flushing + to the database, which increases memory usage but greatly decreases + the amount of disk access (and log-file generation) in database. + Smaller values will limit your overall memory consumption, but can + drastically hurt throughput by necessitating more write operations + to the database (which also generates more log-files). */ +#define WRITE_BUFFER_SIZE 512000 + + + +/* The root structures. + + Why do they contain different data? Well, transactions are mutable + enough that it isn't safe to cache the DAG node for the root + directory or the hash of copyfrom data: somebody else might modify + them concurrently on disk! (Why is the DAG node cache safer than + the root DAG node? When cloning transaction DAG nodes in and out + of the cache, all of the possibly-mutable data from the + node_revision_t inside the dag_node_t is dropped.) Additionally, + revisions are immutable enough that their DAG node cache can be + kept in the FS object and shared among multiple revision root + objects. +*/ +typedef dag_node_t fs_rev_root_data_t; + +typedef struct fs_txn_root_data_t +{ + /* TXN_ID value from the main struct but as a struct instead of a string */ + svn_fs_x__id_part_t txn_id; + + /* Cache of txn DAG nodes (without their nested noderevs, because + * it's mutable). Same keys/values as ffd->rev_node_cache. */ + svn_cache__t *txn_node_cache; +} fs_txn_root_data_t; + +/* Declared here to resolve the circular dependencies. */ +static svn_error_t * get_dag(dag_node_t **dag_node_p, + svn_fs_root_t *root, + const char *path, + svn_boolean_t needs_lock_cache, + apr_pool_t *pool); + +static svn_fs_root_t *make_revision_root(svn_fs_t *fs, svn_revnum_t rev, + dag_node_t *root_dir, + apr_pool_t *pool); + +static svn_error_t *make_txn_root(svn_fs_root_t **root_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn, + svn_revnum_t base_rev, + apr_uint32_t flags, + apr_pool_t *pool); + + +/*** Node Caching ***/ + +/* 1st level cache */ + +/* An entry in the first-level cache. REVISION and PATH form the key that + will ultimately be matched. + */ +typedef struct cache_entry_t +{ + /* hash value derived from PATH, REVISION. + Used to short-circuit failed lookups. */ + apr_uint32_t hash_value; + + /* revision to which the NODE belongs */ + svn_revnum_t revision; + + /* path of the NODE */ + char *path; + + /* cached value of strlen(PATH). */ + apr_size_t path_len; + + /* the node allocated in the cache's pool. NULL for empty entries. */ + dag_node_t *node; +} cache_entry_t; + +/* Number of entries in the cache. Keep this low to keep pressure on the + CPU caches low as well. A binary value is most efficient. If we walk + a directory tree, we want enough entries to store nodes for all files + without overwriting the nodes for the parent folder. That way, there + will be no unnecessary misses (except for a few random ones caused by + hash collision). + + The actual number of instances may be higher but entries that got + overwritten are no longer visible. + */ +enum { BUCKET_COUNT = 256 }; + +/* Each pool that has received a DAG node, will hold at least on lock on + our cache to ensure that the node remains valid despite being allocated + in the cache's pool. This is the structure to represent the lock. + */ +typedef struct cache_lock_t +{ + /* pool holding the lock */ + apr_pool_t *pool; + + /* cache being locked */ + fs_x_dag_cache_t *cache; + + /* next lock. NULL at EOL */ + struct cache_lock_t *next; + + /* previous lock. NULL at list head. Only then this==cache->first_lock */ + struct cache_lock_t *prev; +} cache_lock_t; + +/* The actual cache structure. All nodes will be allocated in POOL. + When the number of INSERTIONS (i.e. objects created form that pool) + exceeds a certain threshold, the pool will be cleared and the cache + with it. + + To ensure that nodes returned from this structure remain valid, the + cache will get locked for the lifetime of the _receiving_ pools (i.e. + those in which we would allocate the node if there was no cache.). + The cache will only be cleared FIRST_LOCK is 0. + */ +struct fs_x_dag_cache_t +{ + /* fixed number of (possibly empty) cache entries */ + cache_entry_t buckets[BUCKET_COUNT]; + + /* pool used for all node allocation */ + apr_pool_t *pool; + + /* number of entries created from POOL since the last cleanup */ + apr_size_t insertions; + + /* Property lookups etc. have a very high locality (75% re-hit). + Thus, remember the last hit location for optimistic lookup. */ + apr_size_t last_hit; + + /* List of receiving pools that are still alive. */ + cache_lock_t *first_lock; +}; + +/* Cleanup function to be called when a receiving pool gets cleared. + Unlocks the cache once. + */ +static apr_status_t +unlock_cache(void *baton_void) +{ + cache_lock_t *lock = baton_void; + + /* remove lock from chain. Update the head */ + if (lock->next) + lock->next->prev = lock->prev; + if (lock->prev) + lock->prev->next = lock->next; + else + lock->cache->first_lock = lock->next; + + return APR_SUCCESS; +} + +/* Cleanup function to be called when the cache itself gets destroyed. + In that case, we must unregister all unlock requests. + */ +static apr_status_t +unregister_locks(void *baton_void) +{ + fs_x_dag_cache_t *cache = baton_void; + cache_lock_t *lock; + + for (lock = cache->first_lock; lock; lock = lock->next) + apr_pool_cleanup_kill(lock->pool, + lock, + unlock_cache); + + return APR_SUCCESS; +} + +fs_x_dag_cache_t* +svn_fs_x__create_dag_cache(apr_pool_t *pool) +{ + fs_x_dag_cache_t *result = apr_pcalloc(pool, sizeof(*result)); + result->pool = svn_pool_create(pool); + + apr_pool_cleanup_register(pool, + result, + unregister_locks, + apr_pool_cleanup_null); + + return result; +} + +/* Prevent the entries in CACHE from being destroyed, for as long as the + POOL lives. + */ +static void +lock_cache(fs_x_dag_cache_t* cache, apr_pool_t *pool) +{ + /* we only need to lock / unlock once per pool. Since we will often ask + for multiple nodes with the same pool, we can reduce the overhead. + However, if e.g. pools are being used in an alternating pattern, + we may lock the cache more than once for the same pool (and register + just as many cleanup actions). + */ + cache_lock_t *lock = cache->first_lock; + + /* try to find an existing lock for POOL. + But limit the time spent on chasing pointers. */ + int limiter = 8; + while (lock && --limiter) + if (lock->pool == pool) + return; + + /* create a new lock and put it at the beginning of the lock chain */ + lock = apr_palloc(pool, sizeof(*lock)); + lock->cache = cache; + lock->pool = pool; + lock->next = cache->first_lock; + lock->prev = NULL; + + if (cache->first_lock) + cache->first_lock->prev = lock; + cache->first_lock = lock; + + /* instruct POOL to remove the look upon cleanup */ + apr_pool_cleanup_register(pool, + lock, + unlock_cache, + apr_pool_cleanup_null); +} + +/* Clears the CACHE at regular intervals (destroying all cached nodes) + */ +static void +auto_clear_dag_cache(fs_x_dag_cache_t* cache) +{ + if (cache->first_lock == NULL && cache->insertions > BUCKET_COUNT) + { + svn_pool_clear(cache->pool); + + memset(cache->buckets, 0, sizeof(cache->buckets)); + cache->insertions = 0; + } +} + +/* For the given REVISION and PATH, return the respective entry in CACHE. + If the entry is empty, its NODE member will be NULL and the caller + may then set it to the corresponding DAG node allocated in CACHE->POOL. + */ +static cache_entry_t * +cache_lookup( fs_x_dag_cache_t *cache + , svn_revnum_t revision + , const char *path) +{ + apr_size_t i, bucket_index; + apr_size_t path_len = strlen(path); + apr_uint32_t hash_value = (apr_uint32_t)revision; + +#if SVN_UNALIGNED_ACCESS_IS_OK + /* "randomizing" / distributing factor used in our hash function */ + const apr_uint32_t factor = 0xd1f3da69; +#endif + + /* optimistic lookup: hit the same bucket again? */ + cache_entry_t *result = &cache->buckets[cache->last_hit]; + if ( (result->revision == revision) + && (result->path_len == path_len) + && !memcmp(result->path, path, path_len)) + { + return result; + } + + /* need to do a full lookup. Calculate the hash value + (HASH_VALUE has been initialized to REVISION). */ + i = 0; +#if SVN_UNALIGNED_ACCESS_IS_OK + /* We relax the dependency chain between iterations by processing + two chunks from the input per hash_value self-multiplication. + The HASH_VALUE update latency is now 1 MUL latency + 1 ADD latency + per 2 chunks instead of 1 chunk. + */ + for (; i + 8 <= path_len; i += 8) + hash_value = hash_value * factor * factor + + ( *(const apr_uint32_t*)(path + i) * factor + + *(const apr_uint32_t*)(path + i + 4)); +#endif + + for (; i < path_len; ++i) + /* Help GCC to minimize the HASH_VALUE update latency by splitting the + MUL 33 of the naive implementation: h = h * 33 + path[i]. This + shortens the dependency chain from 1 shift + 2 ADDs to 1 shift + 1 ADD. + */ + hash_value = hash_value * 32 + (hash_value + (unsigned char)path[i]); + + bucket_index = hash_value + (hash_value >> 16); + bucket_index = (bucket_index + (bucket_index >> 8)) % BUCKET_COUNT; + + /* access the corresponding bucket and remember its location */ + result = &cache->buckets[bucket_index]; + cache->last_hit = bucket_index; + + /* if it is *NOT* a match, clear the bucket, expect the caller to fill + in the node and count it as an insertion */ + if ( (result->hash_value != hash_value) + || (result->revision != revision) + || (result->path_len != path_len) + || memcmp(result->path, path, path_len)) + { + result->hash_value = hash_value; + result->revision = revision; + if (result->path_len < path_len) + result->path = apr_palloc(cache->pool, path_len + 1); + result->path_len = path_len; + memcpy(result->path, path, path_len + 1); + + result->node = NULL; + + cache->insertions++; + } + + return result; +} + +/* 2nd level cache */ + +/* Find and return the DAG node cache for ROOT and the key that + should be used for PATH. */ +static void +locate_cache(svn_cache__t **cache, + const char **key, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + if (root->is_txn_root) + { + fs_txn_root_data_t *frd = root->fsap_data; + if (cache) *cache = frd->txn_node_cache; + if (key && path) *key = path; + } + else + { + fs_x_data_t *ffd = root->fs->fsap_data; + if (cache) *cache = ffd->rev_node_cache; + if (key && path) *key + = svn_fs_x__combine_number_and_string(root->rev, path, pool); + } +} + +/* Return NODE for PATH from ROOT's node cache, or NULL if the node + isn't cached; read it from the FS. *NODE remains valid until either + POOL or the FS gets cleared or destroyed (whichever comes first). + + Since locking can be expensive and POOL may be long-living, for + nodes that will not need to survive the next call to this function, + set NEEDS_LOCK_CACHE to FALSE. */ +static svn_error_t * +dag_node_cache_get(dag_node_t **node_p, + svn_fs_root_t *root, + const char *path, + svn_boolean_t needs_lock_cache, + apr_pool_t *pool) +{ + svn_boolean_t found; + dag_node_t *node = NULL; + svn_cache__t *cache; + const char *key; + + SVN_ERR_ASSERT(*path == '/'); + + if (!root->is_txn_root) + { + /* immutable DAG node. use the global caches for it */ + + fs_x_data_t *ffd = root->fs->fsap_data; + cache_entry_t *bucket; + + auto_clear_dag_cache(ffd->dag_node_cache); + bucket = cache_lookup(ffd->dag_node_cache, root->rev, path); + if (bucket->node == NULL) + { + locate_cache(&cache, &key, root, path, pool); + SVN_ERR(svn_cache__get((void **)&node, &found, cache, key, + ffd->dag_node_cache->pool)); + if (found && node) + { + /* Patch up the FS, since this might have come from an old FS + * object. */ + svn_fs_x__dag_set_fs(node, root->fs); + bucket->node = node; + } + } + else + { + node = bucket->node; + } + + /* if we found a node, make sure it remains valid at least as long + as it would when allocated in POOL. */ + if (node && needs_lock_cache) + lock_cache(ffd->dag_node_cache, pool); + } + else + { + /* DAG is mutable / may become invalid. Use the TXN-local cache */ + + locate_cache(&cache, &key, root, path, pool); + + SVN_ERR(svn_cache__get((void **) &node, &found, cache, key, pool)); + if (found && node) + { + /* Patch up the FS, since this might have come from an old FS + * object. */ + svn_fs_x__dag_set_fs(node, root->fs); + } + } + + *node_p = node; + + return SVN_NO_ERROR; +} + + +/* Add the NODE for PATH to ROOT's node cache. */ +static svn_error_t * +dag_node_cache_set(svn_fs_root_t *root, + const char *path, + dag_node_t *node, + apr_pool_t *pool) +{ + svn_cache__t *cache; + const char *key; + + SVN_ERR_ASSERT(*path == '/'); + + /* Do *not* attempt to dup and put the node into L1. + * dup() is twice as expensive as an L2 lookup (which will set also L1). + */ + locate_cache(&cache, &key, root, path, pool); + + return svn_cache__set(cache, key, node, pool); +} + + +/* Baton for find_descendents_in_cache. */ +struct fdic_baton { + const char *path; + apr_array_header_t *list; + apr_pool_t *pool; +}; + +/* If the given item is a descendent of BATON->PATH, push + * it onto BATON->LIST (copying into BATON->POOL). Implements + * the svn_iter_apr_hash_cb_t prototype. */ +static svn_error_t * +find_descendents_in_cache(void *baton, + const void *key, + apr_ssize_t klen, + void *val, + apr_pool_t *pool) +{ + struct fdic_baton *b = baton; + const char *item_path = key; + + if (svn_fspath__skip_ancestor(b->path, item_path)) + APR_ARRAY_PUSH(b->list, const char *) = apr_pstrdup(b->pool, item_path); + + return SVN_NO_ERROR; +} + +/* Invalidate cache entries for PATH and any of its children. This + should *only* be called on a transaction root! */ +static svn_error_t * +dag_node_cache_invalidate(svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + struct fdic_baton b; + svn_cache__t *cache; + apr_pool_t *iterpool; + int i; + + b.path = path; + b.pool = svn_pool_create(pool); + b.list = apr_array_make(b.pool, 1, sizeof(const char *)); + + SVN_ERR_ASSERT(root->is_txn_root); + locate_cache(&cache, NULL, root, NULL, b.pool); + + + SVN_ERR(svn_cache__iter(NULL, cache, find_descendents_in_cache, + &b, b.pool)); + + iterpool = svn_pool_create(b.pool); + + for (i = 0; i < b.list->nelts; i++) + { + const char *descendent = APR_ARRAY_IDX(b.list, i, const char *); + svn_pool_clear(iterpool); + SVN_ERR(svn_cache__set(cache, descendent, NULL, iterpool)); + } + + svn_pool_destroy(iterpool); + svn_pool_destroy(b.pool); + return SVN_NO_ERROR; +} + + + +/* Creating transaction and revision root nodes. */ + +svn_error_t * +svn_fs_x__txn_root(svn_fs_root_t **root_p, + svn_fs_txn_t *txn, + apr_pool_t *pool) +{ + apr_uint32_t flags = 0; + apr_hash_t *txnprops; + + /* Look for the temporary txn props representing 'flags'. */ + SVN_ERR(svn_fs_x__txn_proplist(&txnprops, txn, pool)); + if (txnprops) + { + if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_OOD)) + flags |= SVN_FS_TXN_CHECK_OOD; + + if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_LOCKS)) + flags |= SVN_FS_TXN_CHECK_LOCKS; + } + + return make_txn_root(root_p, txn->fs, svn_fs_x__txn_get_id(txn), + txn->base_rev, flags, pool); +} + + +svn_error_t * +svn_fs_x__revision_root(svn_fs_root_t **root_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + dag_node_t *root_dir; + + SVN_ERR(svn_fs__check_fs(fs, TRUE)); + + SVN_ERR(svn_fs_x__dag_revision_root(&root_dir, fs, rev, pool)); + + *root_p = make_revision_root(fs, rev, root_dir, pool); + + return SVN_NO_ERROR; +} + + + +/* Getting dag nodes for roots. */ + +/* Return the transaction ID to a given transaction ROOT. */ +static const svn_fs_x__id_part_t * +root_txn_id(svn_fs_root_t *root) +{ + fs_txn_root_data_t *frd = root->fsap_data; + assert(root->is_txn_root); + + return &frd->txn_id; +} + +/* Set *NODE_P to a freshly opened dag node referring to the root + directory of ROOT, allocating from POOL. */ +static svn_error_t * +root_node(dag_node_t **node_p, + svn_fs_root_t *root, + apr_pool_t *pool) +{ + if (root->is_txn_root) + { + /* It's a transaction root. Open a fresh copy. */ + return svn_fs_x__dag_txn_root(node_p, root->fs, root_txn_id(root), + pool); + } + else + { + /* It's a revision root, so we already have its root directory + opened. */ + dag_node_t *root_dir = root->fsap_data; + *node_p = svn_fs_x__dag_dup(root_dir, pool); + return SVN_NO_ERROR; + } +} + + +/* Set *NODE_P to a mutable root directory for ROOT, cloning if + necessary, allocating in POOL. ROOT must be a transaction root. + Use ERROR_PATH in error messages. */ +static svn_error_t * +mutable_root_node(dag_node_t **node_p, + svn_fs_root_t *root, + const char *error_path, + apr_pool_t *pool) +{ + if (root->is_txn_root) + { + /* It's a transaction root. Open a fresh copy. */ + return svn_fs_x__dag_clone_root(node_p, root->fs, root_txn_id(root), + pool); + } + else + /* If it's not a transaction root, we can't change its contents. */ + return SVN_FS__ERR_NOT_MUTABLE(root->fs, root->rev, error_path); +} + + + +/* Traversing directory paths. */ + +typedef enum copy_id_inherit_t +{ + copy_id_inherit_unknown = 0, + copy_id_inherit_self, + copy_id_inherit_parent, + copy_id_inherit_new + +} copy_id_inherit_t; + +/* A linked list representing the path from a node up to a root + directory. We use this for cloning, and for operations that need + to deal with both a node and its parent directory. For example, a + `delete' operation needs to know that the node actually exists, but + also needs to change the parent directory. */ +typedef struct parent_path_t +{ + + /* A node along the path. This could be the final node, one of its + parents, or the root. Every parent path ends with an element for + the root directory. */ + dag_node_t *node; + + /* The name NODE has in its parent directory. This is zero for the + root directory, which (obviously) has no name in its parent. */ + char *entry; + + /* The parent of NODE, or zero if NODE is the root directory. */ + struct parent_path_t *parent; + + /* The copy ID inheritance style. */ + copy_id_inherit_t copy_inherit; + + /* If copy ID inheritance style is copy_id_inherit_new, this is the + path which should be implicitly copied; otherwise, this is NULL. */ + const char *copy_src_path; + +} parent_path_t; + +/* Return a text string describing the absolute path of parent_path + PARENT_PATH. It will be allocated in POOL. */ +static const char * +parent_path_path(parent_path_t *parent_path, + apr_pool_t *pool) +{ + const char *path_so_far = "/"; + if (parent_path->parent) + path_so_far = parent_path_path(parent_path->parent, pool); + return parent_path->entry + ? svn_fspath__join(path_so_far, parent_path->entry, pool) + : path_so_far; +} + + +/* Return the FS path for the parent path chain object CHILD relative + to its ANCESTOR in the same chain, allocated in POOL. */ +static const char * +parent_path_relpath(parent_path_t *child, + parent_path_t *ancestor, + apr_pool_t *pool) +{ + const char *path_so_far = ""; + parent_path_t *this_node = child; + while (this_node != ancestor) + { + assert(this_node != NULL); + path_so_far = svn_relpath_join(this_node->entry, path_so_far, pool); + this_node = this_node->parent; + } + return path_so_far; +} + + + +/* Choose a copy ID inheritance method *INHERIT_P to be used in the + event that immutable node CHILD in FS needs to be made mutable. If + the inheritance method is copy_id_inherit_new, also return a + *COPY_SRC_PATH on which to base the new copy ID (else return NULL + for that path). CHILD must have a parent (it cannot be the root + node). Allocations are taken from POOL. */ +static svn_error_t * +get_copy_inheritance(copy_id_inherit_t *inherit_p, + const char **copy_src_path, + svn_fs_t *fs, + parent_path_t *child, + apr_pool_t *pool) +{ + const svn_fs_id_t *child_id, *parent_id, *copyroot_id; + const svn_fs_x__id_part_t *child_copy_id, *parent_copy_id; + const char *id_path = NULL; + svn_fs_root_t *copyroot_root; + dag_node_t *copyroot_node; + svn_revnum_t copyroot_rev; + const char *copyroot_path; + + SVN_ERR_ASSERT(child && child->parent); + + /* Initialize some convenience variables. */ + child_id = svn_fs_x__dag_get_id(child->node); + parent_id = svn_fs_x__dag_get_id(child->parent->node); + child_copy_id = svn_fs_x__id_copy_id(child_id); + parent_copy_id = svn_fs_x__id_copy_id(parent_id); + + /* If this child is already mutable, we have nothing to do. */ + if (svn_fs_x__id_is_txn(child_id)) + { + *inherit_p = copy_id_inherit_self; + *copy_src_path = NULL; + return SVN_NO_ERROR; + } + + /* From this point on, we'll assume that the child will just take + its copy ID from its parent. */ + *inherit_p = copy_id_inherit_parent; + *copy_src_path = NULL; + + /* Special case: if the child's copy ID is '0', use the parent's + copy ID. */ + if (svn_fs_x__id_part_is_root(child_copy_id)) + return SVN_NO_ERROR; + + /* Compare the copy IDs of the child and its parent. If they are + the same, then the child is already on the same branch as the + parent, and should use the same mutability copy ID that the + parent will use. */ + if (svn_fs_x__id_part_eq(child_copy_id, parent_copy_id)) + return SVN_NO_ERROR; + + /* If the child is on the same branch that the parent is on, the + child should just use the same copy ID that the parent would use. + Else, the child needs to generate a new copy ID to use should it + need to be made mutable. We will claim that child is on the same + branch as its parent if the child itself is not a branch point, + or if it is a branch point that we are accessing via its original + copy destination path. */ + SVN_ERR(svn_fs_x__dag_get_copyroot(©root_rev, ©root_path, + child->node)); + SVN_ERR(svn_fs_x__revision_root(©root_root, fs, copyroot_rev, pool)); + SVN_ERR(get_dag(©root_node, copyroot_root, copyroot_path, FALSE, pool)); + copyroot_id = svn_fs_x__dag_get_id(copyroot_node); + + if (svn_fs_x__id_compare(copyroot_id, child_id) == -1) + return SVN_NO_ERROR; + + /* Determine if we are looking at the child via its original path or + as a subtree item of a copied tree. */ + id_path = svn_fs_x__dag_get_created_path(child->node); + if (strcmp(id_path, parent_path_path(child, pool)) == 0) + { + *inherit_p = copy_id_inherit_self; + return SVN_NO_ERROR; + } + + /* We are pretty sure that the child node is an unedited nested + branched node. When it needs to be made mutable, it should claim + a new copy ID. */ + *inherit_p = copy_id_inherit_new; + *copy_src_path = id_path; + return SVN_NO_ERROR; +} + +/* Allocate a new parent_path_t node from POOL, referring to NODE, + ENTRY, PARENT, and COPY_ID. */ +static parent_path_t * +make_parent_path(dag_node_t *node, + char *entry, + parent_path_t *parent, + apr_pool_t *pool) +{ + parent_path_t *parent_path = apr_pcalloc(pool, sizeof(*parent_path)); + parent_path->node = node; + parent_path->entry = entry; + parent_path->parent = parent; + parent_path->copy_inherit = copy_id_inherit_unknown; + parent_path->copy_src_path = NULL; + return parent_path; +} + + +/* Flags for open_path. */ +typedef enum open_path_flags_t { + + /* The last component of the PATH need not exist. (All parent + directories must exist, as usual.) If the last component doesn't + exist, simply leave the `node' member of the bottom parent_path + component zero. */ + open_path_last_optional = 1, + + /* When this flag is set, don't bother to lookup the DAG node in + our caches because we already tried this. Ignoring this flag + has no functional impact. */ + open_path_uncached = 2, + + /* The caller does not care about the parent node chain but only + the final DAG node. */ + open_path_node_only = 4 +} open_path_flags_t; + + +/* Open the node identified by PATH in ROOT, allocating in POOL. Set + *PARENT_PATH_P to a path from the node up to ROOT. The resulting + **PARENT_PATH_P value is guaranteed to contain at least one + *element, for the root directory. PATH must be in canonical form. + + If resulting *PARENT_PATH_P will eventually be made mutable and + modified, or if copy ID inheritance information is otherwise needed, + IS_TXN_PATH must be set. If IS_TXN_PATH is FALSE, no copy ID + inheritance information will be calculated for the *PARENT_PATH_P chain. + + If FLAGS & open_path_last_optional is zero, return the error + SVN_ERR_FS_NOT_FOUND if the node PATH refers to does not exist. If + non-zero, require all the parent directories to exist as normal, + but if the final path component doesn't exist, simply return a path + whose bottom `node' member is zero. This option is useful for + callers that create new nodes --- we find the parent directory for + them, and tell them whether the entry exists already. + + The remaining bits in FLAGS are hints that allow this function + to take shortcuts based on knowledge that the caller provides, + such as the caller is not actually being interested in PARENT_PATH_P, + but only in (*PARENT_PATH_P)->NODE. + + NOTE: Public interfaces which only *read* from the filesystem + should not call this function directly, but should instead use + get_dag(). +*/ +static svn_error_t * +open_path(parent_path_t **parent_path_p, + svn_fs_root_t *root, + const char *path, + int flags, + svn_boolean_t is_txn_path, + apr_pool_t *pool) +{ + svn_fs_t *fs = root->fs; + dag_node_t *here = NULL; /* The directory we're currently looking at. */ + parent_path_t *parent_path; /* The path from HERE up to the root. */ + const char *rest; /* The portion of PATH we haven't traversed yet. */ + apr_pool_t *iterpool = svn_pool_create(pool); + + /* path to the currently processed entry without trailing '/'. + We will reuse this across iterations by simply putting a NUL terminator + at the respective position and replacing that with a '/' in the next + iteration. This is correct as we assert() PATH to be canonical. */ + svn_stringbuf_t *path_so_far = svn_stringbuf_create(path, pool); + + /* callers often traverse the tree in some path-based order. That means + a sibling of PATH has been presently accessed. Try to start the lookup + directly at the parent node, if the caller did not requested the full + parent chain. */ + assert(svn_fs__is_canonical_abspath(path)); + path_so_far->len = 0; /* "" */ + if (flags & open_path_node_only) + { + const char *directory = svn_dirent_dirname(path, pool); + if (directory[1] != 0) /* root nodes are covered anyway */ + { + SVN_ERR(dag_node_cache_get(&here, root, directory, TRUE, pool)); + /* did the shortcut work? */ + if (here) + { + apr_size_t dirname_len = strlen(directory); + path_so_far->len = dirname_len; + rest = path + dirname_len + 1; + } + } + } + + /* did the shortcut work? */ + if (!here) + { + /* Make a parent_path item for the root node, using its own current + copy id. */ + SVN_ERR(root_node(&here, root, pool)); + rest = path + 1; /* skip the leading '/', it saves in iteration */ + } + + path_so_far->data[path_so_far->len] = '\0'; + parent_path = make_parent_path(here, 0, 0, pool); + parent_path->copy_inherit = copy_id_inherit_self; + + /* Whenever we are at the top of this loop: + - HERE is our current directory, + - ID is the node revision ID of HERE, + - REST is the path we're going to find in HERE, and + - PARENT_PATH includes HERE and all its parents. */ + for (;;) + { + const char *next; + char *entry; + dag_node_t *child; + + svn_pool_clear(iterpool); + + /* Parse out the next entry from the path. */ + entry = svn_fs__next_entry_name(&next, rest, pool); + + /* Update the path traversed thus far. */ + path_so_far->data[path_so_far->len] = '/'; + path_so_far->len += strlen(entry) + 1; + path_so_far->data[path_so_far->len] = '\0'; + + if (*entry == '\0') + { + /* Given the behavior of svn_fs__next_entry_name(), this + happens when the path either starts or ends with a slash. + In either case, we stay put: the current directory stays + the same, and we add nothing to the parent path. */ + child = here; + } + else + { + copy_id_inherit_t inherit; + const char *copy_path = NULL; + svn_error_t *err = SVN_NO_ERROR; + dag_node_t *cached_node = NULL; + + /* If we found a directory entry, follow it. First, we + check our node cache, and, failing that, we hit the DAG + layer. Don't bother to contact the cache for the last + element if we already know the lookup to fail for the + complete path. */ + if (next || !(flags & open_path_uncached)) + SVN_ERR(dag_node_cache_get(&cached_node, root, path_so_far->data, + TRUE, pool)); + if (cached_node) + child = cached_node; + else + err = svn_fs_x__dag_open(&child, here, entry, pool, iterpool); + + /* "file not found" requires special handling. */ + if (err && err->apr_err == SVN_ERR_FS_NOT_FOUND) + { + /* If this was the last path component, and the caller + said it was optional, then don't return an error; + just put a NULL node pointer in the path. */ + + svn_error_clear(err); + + if ((flags & open_path_last_optional) + && (! next || *next == '\0')) + { + parent_path = make_parent_path(NULL, entry, parent_path, + pool); + break; + } + else + { + /* Build a better error message than svn_fs_x__dag_open + can provide, giving the root and full path name. */ + return SVN_FS__NOT_FOUND(root, path); + } + } + + /* Other errors we return normally. */ + SVN_ERR(err); + + if (flags & open_path_node_only) + { + /* Shortcut: the caller only wan'ts the final DAG node. */ + parent_path->node = child; + } + else + { + /* Now, make a parent_path item for CHILD. */ + parent_path = make_parent_path(child, entry, parent_path, pool); + if (is_txn_path) + { + SVN_ERR(get_copy_inheritance(&inherit, ©_path, fs, + parent_path, iterpool)); + parent_path->copy_inherit = inherit; + parent_path->copy_src_path = apr_pstrdup(pool, copy_path); + } + } + + /* Cache the node we found (if it wasn't already cached). */ + if (! cached_node) + SVN_ERR(dag_node_cache_set(root, path_so_far->data, child, + iterpool)); + } + + /* Are we finished traversing the path? */ + if (! next) + break; + + /* The path isn't finished yet; we'd better be in a directory. */ + if (svn_fs_x__dag_node_kind(child) != svn_node_dir) + SVN_ERR_W(SVN_FS__ERR_NOT_DIRECTORY(fs, path_so_far->data), + apr_psprintf(iterpool, _("Failure opening '%s'"), path)); + + rest = next; + here = child; + } + + svn_pool_destroy(iterpool); + *parent_path_p = parent_path; + return SVN_NO_ERROR; +} + + +/* Make the node referred to by PARENT_PATH mutable, if it isn't + already, allocating from POOL. ROOT must be the root from which + PARENT_PATH descends. Clone any parent directories as needed. + Adjust the dag nodes in PARENT_PATH to refer to the clones. Use + ERROR_PATH in error messages. */ +static svn_error_t * +make_path_mutable(svn_fs_root_t *root, + parent_path_t *parent_path, + const char *error_path, + apr_pool_t *pool) +{ + dag_node_t *clone; + const svn_fs_x__id_part_t *txn_id = root_txn_id(root); + + /* Is the node mutable already? */ + if (svn_fs_x__dag_check_mutable(parent_path->node)) + return SVN_NO_ERROR; + + /* Are we trying to clone the root, or somebody's child node? */ + if (parent_path->parent) + { + const svn_fs_id_t *parent_id, *child_id, *copyroot_id; + svn_fs_x__id_part_t copy_id = { SVN_INVALID_REVNUM, 0 }; + svn_fs_x__id_part_t *copy_id_ptr = ©_id; + copy_id_inherit_t inherit = parent_path->copy_inherit; + const char *clone_path, *copyroot_path; + svn_revnum_t copyroot_rev; + svn_boolean_t is_parent_copyroot = FALSE; + svn_fs_root_t *copyroot_root; + dag_node_t *copyroot_node; + + /* We're trying to clone somebody's child. Make sure our parent + is mutable. */ + SVN_ERR(make_path_mutable(root, parent_path->parent, + error_path, pool)); + + switch (inherit) + { + case copy_id_inherit_parent: + parent_id = svn_fs_x__dag_get_id(parent_path->parent->node); + copy_id = *svn_fs_x__id_copy_id(parent_id); + break; + + case copy_id_inherit_new: + SVN_ERR(svn_fs_x__reserve_copy_id(©_id, root->fs, txn_id, + pool)); + break; + + case copy_id_inherit_self: + copy_id_ptr = NULL; + break; + + case copy_id_inherit_unknown: + default: + SVN_ERR_MALFUNCTION(); /* uh-oh -- somebody didn't calculate copy-ID + inheritance data. */ + } + + /* Determine what copyroot our new child node should use. */ + SVN_ERR(svn_fs_x__dag_get_copyroot(©root_rev, ©root_path, + parent_path->node)); + SVN_ERR(svn_fs_x__revision_root(©root_root, root->fs, + copyroot_rev, pool)); + SVN_ERR(get_dag(©root_node, copyroot_root, copyroot_path, + FALSE, pool)); + + child_id = svn_fs_x__dag_get_id(parent_path->node); + copyroot_id = svn_fs_x__dag_get_id(copyroot_node); + if (!svn_fs_x__id_part_eq(svn_fs_x__id_node_id(child_id), + svn_fs_x__id_node_id(copyroot_id))) + is_parent_copyroot = TRUE; + + /* Now make this node mutable. */ + clone_path = parent_path_path(parent_path->parent, pool); + SVN_ERR(svn_fs_x__dag_clone_child(&clone, + parent_path->parent->node, + clone_path, + parent_path->entry, + copy_id_ptr, txn_id, + is_parent_copyroot, + pool)); + + /* Update the path cache. */ + SVN_ERR(dag_node_cache_set(root, parent_path_path(parent_path, pool), + clone, pool)); + } + else + { + /* We're trying to clone the root directory. */ + SVN_ERR(mutable_root_node(&clone, root, error_path, pool)); + } + + /* Update the PARENT_PATH link to refer to the clone. */ + parent_path->node = clone; + + return SVN_NO_ERROR; +} + + +/* Open the node identified by PATH in ROOT. Set DAG_NODE_P to the + node we find, allocated in POOL. Return the error + SVN_ERR_FS_NOT_FOUND if this node doesn't exist. + + Since locking can be expensive and POOL may be long-living, for + nodes that will not need to survive the next call to this function, + set NEEDS_LOCK_CACHE to FALSE. */ +static svn_error_t * +get_dag(dag_node_t **dag_node_p, + svn_fs_root_t *root, + const char *path, + svn_boolean_t needs_lock_cache, + apr_pool_t *pool) +{ + parent_path_t *parent_path; + dag_node_t *node = NULL; + + /* First we look for the DAG in our cache + (if the path may be canonical). */ + if (*path == '/') + SVN_ERR(dag_node_cache_get(&node, root, path, needs_lock_cache, pool)); + + if (! node) + { + /* Canonicalize the input PATH. */ + if (! svn_fs__is_canonical_abspath(path)) + { + path = svn_fs__canonicalize_abspath(path, pool); + + /* Try again with the corrected path. */ + SVN_ERR(dag_node_cache_get(&node, root, path, needs_lock_cache, + pool)); + } + + if (! node) + { + /* Call open_path with no flags, as we want this to return an + * error if the node for which we are searching doesn't exist. */ + SVN_ERR(open_path(&parent_path, root, path, + open_path_uncached | open_path_node_only, + FALSE, pool)); + node = parent_path->node; + + /* No need to cache our find -- open_path() will do that for us. */ + } + } + + *dag_node_p = node; + return SVN_NO_ERROR; +} + + + +/* Populating the `changes' table. */ + +/* Add a change to the changes table in FS, keyed on transaction id + TXN_ID, and indicated that a change of kind CHANGE_KIND occurred on + PATH (whose node revision id is--or was, in the case of a + deletion--NODEREV_ID), and optionally that TEXT_MODs or PROP_MODs + occurred. If the change resulted from a copy, COPYFROM_REV and + COPYFROM_PATH specify under which revision and path the node was + copied from. If this was not part of a copy, COPYFROM_REV should + be SVN_INVALID_REVNUM. Do all this as part of POOL. */ +static svn_error_t * +add_change(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + const char *path, + const svn_fs_id_t *noderev_id, + svn_fs_path_change_kind_t change_kind, + svn_boolean_t text_mod, + svn_boolean_t prop_mod, + svn_node_kind_t node_kind, + svn_revnum_t copyfrom_rev, + const char *copyfrom_path, + apr_pool_t *pool) +{ + return svn_fs_x__add_change(fs, txn_id, + svn_fs__canonicalize_abspath(path, pool), + noderev_id, change_kind, text_mod, prop_mod, + node_kind, copyfrom_rev, copyfrom_path, + pool); +} + + + +/* Generic node operations. */ + +/* Get the id of a node referenced by path PATH in ROOT. Return the + id in *ID_P allocated in POOL. */ +svn_error_t * +svn_fs_x__node_id(const svn_fs_id_t **id_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + if ((! root->is_txn_root) + && (path[0] == '\0' || ((path[0] == '/') && (path[1] == '\0')))) + { + /* Optimize the case where we don't need any db access at all. + The root directory ("" or "/") node is stored in the + svn_fs_root_t object, and never changes when it's a revision + root, so we can just reach in and grab it directly. */ + dag_node_t *root_dir = root->fsap_data; + *id_p = svn_fs_x__id_copy(svn_fs_x__dag_get_id(root_dir), pool); + } + else + { + dag_node_t *node; + + SVN_ERR(get_dag(&node, root, path, FALSE, pool)); + *id_p = svn_fs_x__id_copy(svn_fs_x__dag_get_id(node), pool); + } + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__node_created_rev(svn_revnum_t *revision, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + dag_node_t *node; + + SVN_ERR(get_dag(&node, root, path, FALSE, pool)); + return svn_fs_x__dag_get_revision(revision, node, pool); +} + + +/* Set *CREATED_PATH to the path at which PATH under ROOT was created. + Return a string allocated in POOL. */ +static svn_error_t * +x_node_created_path(const char **created_path, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + dag_node_t *node; + + SVN_ERR(get_dag(&node, root, path, TRUE, pool)); + *created_path = svn_fs_x__dag_get_created_path(node); + + return SVN_NO_ERROR; +} + + +/* Set *KIND_P to the type of node located at PATH under ROOT. + Perform temporary allocations in POOL. */ +static svn_error_t * +node_kind(svn_node_kind_t *kind_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + const svn_fs_id_t *node_id; + dag_node_t *node; + + /* Get the node id. */ + SVN_ERR(svn_fs_x__node_id(&node_id, root, path, pool)); + + /* Use the node id to get the real kind. */ + SVN_ERR(svn_fs_x__dag_get_node(&node, root->fs, node_id, pool)); + *kind_p = svn_fs_x__dag_node_kind(node); + + return SVN_NO_ERROR; +} + + +/* Set *KIND_P to the type of node present at PATH under ROOT. If + PATH does not exist under ROOT, set *KIND_P to svn_node_none. Use + POOL for temporary allocation. */ +svn_error_t * +svn_fs_x__check_path(svn_node_kind_t *kind_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + svn_error_t *err = node_kind(kind_p, root, path, pool); + if (err && + ((err->apr_err == SVN_ERR_FS_NOT_FOUND) + || (err->apr_err == SVN_ERR_FS_NOT_DIRECTORY))) + { + svn_error_clear(err); + err = SVN_NO_ERROR; + *kind_p = svn_node_none; + } + + return svn_error_trace(err); +} + +/* Set *VALUE_P to the value of the property named PROPNAME of PATH in + ROOT. If the node has no property by that name, set *VALUE_P to + zero. Allocate the result in POOL. */ +static svn_error_t * +x_node_prop(svn_string_t **value_p, + svn_fs_root_t *root, + const char *path, + const char *propname, + apr_pool_t *pool) +{ + dag_node_t *node; + apr_hash_t *proplist; + + SVN_ERR(get_dag(&node, root, path, FALSE, pool)); + SVN_ERR(svn_fs_x__dag_get_proplist(&proplist, node, pool)); + *value_p = NULL; + if (proplist) + *value_p = svn_hash_gets(proplist, propname); + + return SVN_NO_ERROR; +} + + +/* Set *TABLE_P to the entire property list of PATH under ROOT, as an + APR hash table allocated in POOL. The resulting property table + maps property names to pointers to svn_string_t objects containing + the property value. */ +static svn_error_t * +x_node_proplist(apr_hash_t **table_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + apr_hash_t *table; + dag_node_t *node; + + SVN_ERR(get_dag(&node, root, path, FALSE, pool)); + SVN_ERR(svn_fs_x__dag_get_proplist(&table, node, pool)); + *table_p = table ? table : apr_hash_make(pool); + + return SVN_NO_ERROR; +} + + +static svn_error_t * +increment_mergeinfo_up_tree(parent_path_t *pp, + apr_int64_t increment, + apr_pool_t *pool) +{ + for (; pp; pp = pp->parent) + SVN_ERR(svn_fs_x__dag_increment_mergeinfo_count(pp->node, + increment, + pool)); + + return SVN_NO_ERROR; +} + +/* Change, add, or delete a node's property value. The affected node + is PATH under ROOT, the property value to modify is NAME, and VALUE + points to either a string value to set the new contents to, or NULL + if the property should be deleted. Perform temporary allocations + in POOL. */ +static svn_error_t * +x_change_node_prop(svn_fs_root_t *root, + const char *path, + const char *name, + const svn_string_t *value, + apr_pool_t *pool) +{ + parent_path_t *parent_path; + apr_hash_t *proplist; + const svn_fs_x__id_part_t *txn_id; + + if (! root->is_txn_root) + return SVN_FS__NOT_TXN(root); + txn_id = root_txn_id(root); + + path = svn_fs__canonicalize_abspath(path, pool); + SVN_ERR(open_path(&parent_path, root, path, 0, TRUE, pool)); + + /* Check (non-recursively) to see if path is locked; if so, check + that we can use it. */ + if (root->txn_flags & SVN_FS_TXN_CHECK_LOCKS) + SVN_ERR(svn_fs_x__allow_locked_operation(path, root->fs, FALSE, FALSE, + pool)); + + SVN_ERR(make_path_mutable(root, parent_path, path, pool)); + SVN_ERR(svn_fs_x__dag_get_proplist(&proplist, parent_path->node, pool)); + + /* If there's no proplist, but we're just deleting a property, exit now. */ + if ((! proplist) && (! value)) + return SVN_NO_ERROR; + + /* Now, if there's no proplist, we know we need to make one. */ + if (! proplist) + proplist = apr_hash_make(pool); + + if (strcmp(name, SVN_PROP_MERGEINFO) == 0) + { + apr_int64_t increment = 0; + svn_boolean_t had_mergeinfo; + SVN_ERR(svn_fs_x__dag_has_mergeinfo(&had_mergeinfo, parent_path->node)); + + if (value && !had_mergeinfo) + increment = 1; + else if (!value && had_mergeinfo) + increment = -1; + + if (increment != 0) + { + SVN_ERR(increment_mergeinfo_up_tree(parent_path, increment, pool)); + SVN_ERR(svn_fs_x__dag_set_has_mergeinfo(parent_path->node, + (value != NULL), pool)); + } + } + + /* Set the property. */ + svn_hash_sets(proplist, name, value); + + /* Overwrite the node's proplist. */ + SVN_ERR(svn_fs_x__dag_set_proplist(parent_path->node, proplist, + pool)); + + /* Make a record of this modification in the changes table. */ + return add_change(root->fs, txn_id, path, + svn_fs_x__dag_get_id(parent_path->node), + svn_fs_path_change_modify, FALSE, TRUE, + svn_fs_x__dag_node_kind(parent_path->node), + SVN_INVALID_REVNUM, NULL, pool); +} + + +/* Determine if the properties of two path/root combinations are + different. Set *CHANGED_P to TRUE if the properties at PATH1 under + ROOT1 differ from those at PATH2 under ROOT2, or FALSE otherwise. + Both roots must be in the same filesystem. */ +static svn_error_t * +x_props_changed(svn_boolean_t *changed_p, + svn_fs_root_t *root1, + const char *path1, + svn_fs_root_t *root2, + const char *path2, + apr_pool_t *pool) +{ + dag_node_t *node1, *node2; + + /* Check that roots are in the same fs. */ + if (root1->fs != root2->fs) + return svn_error_create + (SVN_ERR_FS_GENERAL, NULL, + _("Cannot compare property value between two different filesystems")); + + SVN_ERR(get_dag(&node1, root1, path1, TRUE, pool)); + SVN_ERR(get_dag(&node2, root2, path2, TRUE, pool)); + return svn_fs_x__dag_things_different(changed_p, NULL, node1, node2); +} + + + +/* Merges and commits. */ + +/* Set *NODE to the root node of ROOT. */ +static svn_error_t * +get_root(dag_node_t **node, svn_fs_root_t *root, apr_pool_t *pool) +{ + return get_dag(node, root, "/", TRUE, pool); +} + + +/* Set the contents of CONFLICT_PATH to PATH, and return an + SVN_ERR_FS_CONFLICT error that indicates that there was a conflict + at PATH. Perform all allocations in POOL (except the allocation of + CONFLICT_PATH, which should be handled outside this function). */ +static svn_error_t * +conflict_err(svn_stringbuf_t *conflict_path, + const char *path) +{ + svn_stringbuf_set(conflict_path, path); + return svn_error_createf(SVN_ERR_FS_CONFLICT, NULL, + _("Conflict at '%s'"), path); +} + + +/* Merge changes between ANCESTOR and SOURCE into TARGET. ANCESTOR + * and TARGET must be distinct node revisions. TARGET_PATH should + * correspond to TARGET's full path in its filesystem, and is used for + * reporting conflict location. + * + * SOURCE, TARGET, and ANCESTOR are generally directories; this + * function recursively merges the directories' contents. If any are + * files, this function simply returns an error whenever SOURCE, + * TARGET, and ANCESTOR are all distinct node revisions. + * + * If there are differences between ANCESTOR and SOURCE that conflict + * with changes between ANCESTOR and TARGET, this function returns an + * SVN_ERR_FS_CONFLICT error, and updates CONFLICT_P to the name of the + * conflicting node in TARGET, with TARGET_PATH prepended as a path. + * + * If there are no conflicting differences, CONFLICT_P is updated to + * the empty string. + * + * CONFLICT_P must point to a valid svn_stringbuf_t. + * + * Do any necessary temporary allocation in POOL. + */ +static svn_error_t * +merge(svn_stringbuf_t *conflict_p, + const char *target_path, + dag_node_t *target, + dag_node_t *source, + dag_node_t *ancestor, + const svn_fs_x__id_part_t *txn_id, + apr_int64_t *mergeinfo_increment_out, + apr_pool_t *pool) +{ + const svn_fs_id_t *source_id, *target_id, *ancestor_id; + apr_hash_t *s_entries, *t_entries, *a_entries; + apr_hash_index_t *hi; + svn_fs_t *fs; + apr_pool_t *iterpool; + apr_int64_t mergeinfo_increment = 0; + + /* Make sure everyone comes from the same filesystem. */ + fs = svn_fs_x__dag_get_fs(ancestor); + if ((fs != svn_fs_x__dag_get_fs(source)) + || (fs != svn_fs_x__dag_get_fs(target))) + { + return svn_error_create + (SVN_ERR_FS_CORRUPT, NULL, + _("Bad merge; ancestor, source, and target not all in same fs")); + } + + /* We have the same fs, now check it. */ + SVN_ERR(svn_fs__check_fs(fs, TRUE)); + + source_id = svn_fs_x__dag_get_id(source); + target_id = svn_fs_x__dag_get_id(target); + ancestor_id = svn_fs_x__dag_get_id(ancestor); + + /* It's improper to call this function with ancestor == target. */ + if (svn_fs_x__id_eq(ancestor_id, target_id)) + { + svn_string_t *id_str = svn_fs_x__id_unparse(target_id, pool); + return svn_error_createf + (SVN_ERR_FS_GENERAL, NULL, + _("Bad merge; target '%s' has id '%s', same as ancestor"), + target_path, id_str->data); + } + + svn_stringbuf_setempty(conflict_p); + + /* Base cases: + * Either no change made in source, or same change as made in target. + * Both mean nothing to merge here. + */ + if (svn_fs_x__id_eq(ancestor_id, source_id) + || (svn_fs_x__id_eq(source_id, target_id))) + return SVN_NO_ERROR; + + /* Else proceed, knowing all three are distinct node revisions. + * + * How to merge from this point: + * + * if (not all 3 are directories) + * { + * early exit with conflict; + * } + * + * // Property changes may only be made to up-to-date + * // directories, because once the client commits the prop + * // change, it bumps the directory's revision, and therefore + * // must be able to depend on there being no other changes to + * // that directory in the repository. + * if (target's property list differs from ancestor's) + * conflict; + * + * For each entry NAME in the directory ANCESTOR: + * + * Let ANCESTOR-ENTRY, SOURCE-ENTRY, and TARGET-ENTRY be the IDs of + * the name within ANCESTOR, SOURCE, and TARGET respectively. + * (Possibly null if NAME does not exist in SOURCE or TARGET.) + * + * If ANCESTOR-ENTRY == SOURCE-ENTRY, then: + * No changes were made to this entry while the transaction was in + * progress, so do nothing to the target. + * + * Else if ANCESTOR-ENTRY == TARGET-ENTRY, then: + * A change was made to this entry while the transaction was in + * process, but the transaction did not touch this entry. Replace + * TARGET-ENTRY with SOURCE-ENTRY. + * + * Else: + * Changes were made to this entry both within the transaction and + * to the repository while the transaction was in progress. They + * must be merged or declared to be in conflict. + * + * If SOURCE-ENTRY and TARGET-ENTRY are both null, that's a + * double delete; flag a conflict. + * + * If any of the three entries is of type file, declare a conflict. + * + * If either SOURCE-ENTRY or TARGET-ENTRY is not a direct + * modification of ANCESTOR-ENTRY (determine by comparing the + * node-id fields), declare a conflict. A replacement is + * incompatible with a modification or other replacement--even + * an identical replacement. + * + * Direct modifications were made to the directory ANCESTOR-ENTRY + * in both SOURCE and TARGET. Recursively merge these + * modifications. + * + * For each leftover entry NAME in the directory SOURCE: + * + * If NAME exists in TARGET, declare a conflict. Even if SOURCE and + * TARGET are adding exactly the same thing, two additions are not + * auto-mergeable with each other. + * + * Add NAME to TARGET with the entry from SOURCE. + * + * Now that we are done merging the changes from SOURCE into the + * directory TARGET, update TARGET's predecessor to be SOURCE. + */ + + if ((svn_fs_x__dag_node_kind(source) != svn_node_dir) + || (svn_fs_x__dag_node_kind(target) != svn_node_dir) + || (svn_fs_x__dag_node_kind(ancestor) != svn_node_dir)) + { + return conflict_err(conflict_p, target_path); + } + + + /* Possible early merge failure: if target and ancestor have + different property lists, then the merge should fail. + Propchanges can *only* be committed on an up-to-date directory. + ### TODO: see issue #418 about the inelegance of this. + + Another possible, similar, early merge failure: if source and + ancestor have different property lists (meaning someone else + changed directory properties while our commit transaction was + happening), the merge should fail. See issue #2751. + */ + { + node_revision_t *tgt_nr, *anc_nr, *src_nr; + + /* Get node revisions for our id's. */ + SVN_ERR(svn_fs_x__get_node_revision(&tgt_nr, fs, target_id, pool)); + SVN_ERR(svn_fs_x__get_node_revision(&anc_nr, fs, ancestor_id, pool)); + SVN_ERR(svn_fs_x__get_node_revision(&src_nr, fs, source_id, pool)); + + /* Now compare the prop-keys of the skels. Note that just because + the keys are different -doesn't- mean the proplists have + different contents. But merge() isn't concerned with contents; + it doesn't do a brute-force comparison on textual contents, so + it won't do that here either. Checking to see if the propkey + atoms are `equal' is enough. */ + if (! svn_fs_x__noderev_same_rep_key(tgt_nr->prop_rep, anc_nr->prop_rep)) + return conflict_err(conflict_p, target_path); + if (! svn_fs_x__noderev_same_rep_key(src_nr->prop_rep, anc_nr->prop_rep)) + return conflict_err(conflict_p, target_path); + } + + /* ### todo: it would be more efficient to simply check for a NULL + entries hash where necessary below than to allocate an empty hash + here, but another day, another day... */ + SVN_ERR(svn_fs_x__dag_dir_entries(&s_entries, source, pool)); + SVN_ERR(svn_fs_x__dag_dir_entries(&t_entries, target, pool)); + SVN_ERR(svn_fs_x__dag_dir_entries(&a_entries, ancestor, pool)); + + /* for each entry E in a_entries... */ + iterpool = svn_pool_create(pool); + for (hi = apr_hash_first(pool, a_entries); + hi; + hi = apr_hash_next(hi)) + { + svn_fs_dirent_t *s_entry, *t_entry, *a_entry; + const char *name; + apr_ssize_t klen; + + svn_pool_clear(iterpool); + + name = svn__apr_hash_index_key(hi); + klen = svn__apr_hash_index_klen(hi); + a_entry = svn__apr_hash_index_val(hi); + + s_entry = apr_hash_get(s_entries, name, klen); + t_entry = apr_hash_get(t_entries, name, klen); + + /* No changes were made to this entry while the transaction was + in progress, so do nothing to the target. */ + if (s_entry && svn_fs_x__id_eq(a_entry->id, s_entry->id)) + goto end; + + /* A change was made to this entry while the transaction was in + process, but the transaction did not touch this entry. */ + else if (t_entry && svn_fs_x__id_eq(a_entry->id, t_entry->id)) + { + apr_int64_t mergeinfo_start; + apr_int64_t mergeinfo_end; + + dag_node_t *t_ent_node; + SVN_ERR(svn_fs_x__dag_get_node(&t_ent_node, fs, + t_entry->id, iterpool)); + SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_start, + t_ent_node)); + mergeinfo_increment -= mergeinfo_start; + + if (s_entry) + { + dag_node_t *s_ent_node; + SVN_ERR(svn_fs_x__dag_get_node(&s_ent_node, fs, + s_entry->id, iterpool)); + + SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_end, + s_ent_node)); + mergeinfo_increment += mergeinfo_end; + + SVN_ERR(svn_fs_x__dag_set_entry(target, name, + s_entry->id, + s_entry->kind, + txn_id, + iterpool)); + } + else + { + SVN_ERR(svn_fs_x__dag_delete(target, name, txn_id, iterpool)); + } + } + + /* Changes were made to this entry both within the transaction + and to the repository while the transaction was in progress. + They must be merged or declared to be in conflict. */ + else + { + dag_node_t *s_ent_node, *t_ent_node, *a_ent_node; + const char *new_tpath; + apr_int64_t sub_mergeinfo_increment; + + /* If SOURCE-ENTRY and TARGET-ENTRY are both null, that's a + double delete; if one of them is null, that's a delete versus + a modification. In any of these cases, flag a conflict. */ + if (s_entry == NULL || t_entry == NULL) + return conflict_err(conflict_p, + svn_fspath__join(target_path, + a_entry->name, + iterpool)); + + /* If any of the three entries is of type file, flag a conflict. */ + if (s_entry->kind == svn_node_file + || t_entry->kind == svn_node_file + || a_entry->kind == svn_node_file) + return conflict_err(conflict_p, + svn_fspath__join(target_path, + a_entry->name, + iterpool)); + + /* If either SOURCE-ENTRY or TARGET-ENTRY is not a direct + modification of ANCESTOR-ENTRY, declare a conflict. */ + if (!svn_fs_x__id_part_eq(svn_fs_x__id_node_id(s_entry->id), + svn_fs_x__id_node_id(a_entry->id)) + || !svn_fs_x__id_part_eq(svn_fs_x__id_copy_id(s_entry->id), + svn_fs_x__id_copy_id(a_entry->id)) + || !svn_fs_x__id_part_eq(svn_fs_x__id_node_id(t_entry->id), + svn_fs_x__id_node_id(a_entry->id)) + || !svn_fs_x__id_part_eq(svn_fs_x__id_copy_id(t_entry->id), + svn_fs_x__id_copy_id(a_entry->id))) + return conflict_err(conflict_p, + svn_fspath__join(target_path, + a_entry->name, + iterpool)); + + /* Direct modifications were made to the directory + ANCESTOR-ENTRY in both SOURCE and TARGET. Recursively + merge these modifications. */ + SVN_ERR(svn_fs_x__dag_get_node(&s_ent_node, fs, + s_entry->id, iterpool)); + SVN_ERR(svn_fs_x__dag_get_node(&t_ent_node, fs, + t_entry->id, iterpool)); + SVN_ERR(svn_fs_x__dag_get_node(&a_ent_node, fs, + a_entry->id, iterpool)); + new_tpath = svn_fspath__join(target_path, t_entry->name, iterpool); + SVN_ERR(merge(conflict_p, new_tpath, + t_ent_node, s_ent_node, a_ent_node, + txn_id, + &sub_mergeinfo_increment, + iterpool)); + mergeinfo_increment += sub_mergeinfo_increment; + } + + /* We've taken care of any possible implications E could have. + Remove it from source_entries, so it's easy later to loop + over all the source entries that didn't exist in + ancestor_entries. */ + end: + apr_hash_set(s_entries, name, klen, NULL); + } + + /* For each entry E in source but not in ancestor */ + for (hi = apr_hash_first(pool, s_entries); + hi; + hi = apr_hash_next(hi)) + { + svn_fs_dirent_t *s_entry, *t_entry; + const char *name = svn__apr_hash_index_key(hi); + apr_ssize_t klen = svn__apr_hash_index_klen(hi); + dag_node_t *s_ent_node; + apr_int64_t mergeinfo_s; + + svn_pool_clear(iterpool); + + s_entry = svn__apr_hash_index_val(hi); + t_entry = apr_hash_get(t_entries, name, klen); + + /* If NAME exists in TARGET, declare a conflict. */ + if (t_entry) + return conflict_err(conflict_p, + svn_fspath__join(target_path, + t_entry->name, + iterpool)); + + SVN_ERR(svn_fs_x__dag_get_node(&s_ent_node, fs, + s_entry->id, iterpool)); + SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_s, s_ent_node)); + mergeinfo_increment += mergeinfo_s; + + SVN_ERR(svn_fs_x__dag_set_entry + (target, s_entry->name, s_entry->id, s_entry->kind, + txn_id, iterpool)); + } + svn_pool_destroy(iterpool); + + SVN_ERR(svn_fs_x__dag_update_ancestry(target, source, pool)); + + SVN_ERR(svn_fs_x__dag_increment_mergeinfo_count(target, + mergeinfo_increment, + pool)); + + if (mergeinfo_increment_out) + *mergeinfo_increment_out = mergeinfo_increment; + + return SVN_NO_ERROR; +} + +/* Merge changes between an ancestor and SOURCE_NODE into + TXN. The ancestor is either ANCESTOR_NODE, or if + that is null, TXN's base node. + + If the merge is successful, TXN's base will become + SOURCE_NODE, and its root node will have a new ID, a + successor of SOURCE_NODE. + + If a conflict results, update *CONFLICT to the path in the txn that + conflicted; see the CONFLICT_P parameter of merge() for details. */ +static svn_error_t * +merge_changes(dag_node_t *ancestor_node, + dag_node_t *source_node, + svn_fs_txn_t *txn, + svn_stringbuf_t *conflict, + apr_pool_t *pool) +{ + dag_node_t *txn_root_node; + svn_fs_t *fs = txn->fs; + const svn_fs_x__id_part_t *txn_id = svn_fs_x__txn_get_id(txn); + + SVN_ERR(svn_fs_x__dag_txn_root(&txn_root_node, fs, txn_id, pool)); + + if (ancestor_node == NULL) + { + SVN_ERR(svn_fs_x__dag_txn_base_root(&ancestor_node, fs, + txn_id, pool)); + } + + if (svn_fs_x__id_eq(svn_fs_x__dag_get_id(ancestor_node), + svn_fs_x__dag_get_id(txn_root_node))) + { + /* If no changes have been made in TXN since its current base, + then it can't conflict with any changes since that base. + The caller isn't supposed to call us in that case. */ + SVN_ERR_MALFUNCTION(); + } + else + SVN_ERR(merge(conflict, "/", txn_root_node, + source_node, ancestor_node, txn_id, NULL, pool)); + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__commit_txn(const char **conflict_p, + svn_revnum_t *new_rev, + svn_fs_txn_t *txn, + svn_boolean_t set_timestamp, + apr_pool_t *pool) +{ + /* How do commits work in Subversion? + * + * When you're ready to commit, here's what you have: + * + * 1. A transaction, with a mutable tree hanging off it. + * 2. A base revision, against which TXN_TREE was made. + * 3. A latest revision, which may be newer than the base rev. + * + * The problem is that if latest != base, then one can't simply + * attach the txn root as the root of the new revision, because that + * would lose all the changes between base and latest. It is also + * not acceptable to insist that base == latest; in a busy + * repository, commits happen too fast to insist that everyone keep + * their entire tree up-to-date at all times. Non-overlapping + * changes should not interfere with each other. + * + * The solution is to merge the changes between base and latest into + * the txn tree [see the function merge()]. The txn tree is the + * only one of the three trees that is mutable, so it has to be the + * one to adjust. + * + * You might have to adjust it more than once, if a new latest + * revision gets committed while you were merging in the previous + * one. For example: + * + * 1. Jane starts txn T, based at revision 6. + * 2. Someone commits (or already committed) revision 7. + * 3. Jane's starts merging the changes between 6 and 7 into T. + * 4. Meanwhile, someone commits revision 8. + * 5. Jane finishes the 6-->7 merge. T could now be committed + * against a latest revision of 7, if only that were still the + * latest. Unfortunately, 8 is now the latest, so... + * 6. Jane starts merging the changes between 7 and 8 into T. + * 7. Meanwhile, no one commits any new revisions. Whew. + * 8. Jane commits T, creating revision 9, whose tree is exactly + * T's tree, except immutable now. + * + * Lather, rinse, repeat. + */ + + svn_error_t *err = SVN_NO_ERROR; + svn_stringbuf_t *conflict = svn_stringbuf_create_empty(pool); + svn_fs_t *fs = txn->fs; + + /* Limit memory usage when the repository has a high commit rate and + needs to run the following while loop multiple times. The memory + growth without an iteration pool is very noticeable when the + transaction modifies a node that has 20,000 sibling nodes. */ + apr_pool_t *iterpool = svn_pool_create(pool); + + /* Initialize output params. */ + *new_rev = SVN_INVALID_REVNUM; + if (conflict_p) + *conflict_p = NULL; + + while (1729) + { + svn_revnum_t youngish_rev; + svn_fs_root_t *youngish_root; + dag_node_t *youngish_root_node; + + svn_pool_clear(iterpool); + + /* Get the *current* youngest revision. We call it "youngish" + because new revisions might get committed after we've + obtained it. */ + + SVN_ERR(svn_fs_x__youngest_rev(&youngish_rev, fs, iterpool)); + SVN_ERR(svn_fs_x__revision_root(&youngish_root, fs, youngish_rev, + iterpool)); + + /* Get the dag node for the youngest revision. Later we'll use + it as the SOURCE argument to a merge, and if the merge + succeeds, this youngest root node will become the new base + root for the svn txn that was the target of the merge (but + note that the youngest rev may have changed by then -- that's + why we're careful to get this root in its own bdb txn + here). */ + SVN_ERR(get_root(&youngish_root_node, youngish_root, iterpool)); + + /* Try to merge. If the merge succeeds, the base root node of + TARGET's txn will become the same as youngish_root_node, so + any future merges will only be between that node and whatever + the root node of the youngest rev is by then. */ + err = merge_changes(NULL, youngish_root_node, txn, conflict, iterpool); + if (err) + { + if ((err->apr_err == SVN_ERR_FS_CONFLICT) && conflict_p) + *conflict_p = conflict->data; + goto cleanup; + } + txn->base_rev = youngish_rev; + + /* Try to commit. */ + err = svn_fs_x__commit(new_rev, fs, txn, set_timestamp, iterpool); + if (err && (err->apr_err == SVN_ERR_FS_TXN_OUT_OF_DATE)) + { + /* Did someone else finish committing a new revision while we + were in mid-merge or mid-commit? If so, we'll need to + loop again to merge the new changes in, then try to + commit again. Or if that's not what happened, then just + return the error. */ + svn_revnum_t youngest_rev; + SVN_ERR(svn_fs_x__youngest_rev(&youngest_rev, fs, iterpool)); + if (youngest_rev == youngish_rev) + goto cleanup; + else + svn_error_clear(err); + } + else if (err) + { + goto cleanup; + } + else + { + err = SVN_NO_ERROR; + goto cleanup; + } + } + + cleanup: + + svn_fs_x__reset_txn_caches(fs); + + svn_pool_destroy(iterpool); + return svn_error_trace(err); +} + + +/* Merge changes between two nodes into a third node. Given nodes + SOURCE_PATH under SOURCE_ROOT, TARGET_PATH under TARGET_ROOT and + ANCESTOR_PATH under ANCESTOR_ROOT, modify target to contain all the + changes between the ancestor and source. If there are conflicts, + return SVN_ERR_FS_CONFLICT and set *CONFLICT_P to a textual + description of the offending changes. Perform any temporary + allocations in POOL. */ +static svn_error_t * +x_merge(const char **conflict_p, + svn_fs_root_t *source_root, + const char *source_path, + svn_fs_root_t *target_root, + const char *target_path, + svn_fs_root_t *ancestor_root, + const char *ancestor_path, + apr_pool_t *pool) +{ + dag_node_t *source, *ancestor; + svn_fs_txn_t *txn; + svn_error_t *err; + svn_stringbuf_t *conflict = svn_stringbuf_create_empty(pool); + + if (! target_root->is_txn_root) + return SVN_FS__NOT_TXN(target_root); + + /* Paranoia. */ + if ((source_root->fs != ancestor_root->fs) + || (target_root->fs != ancestor_root->fs)) + { + return svn_error_create + (SVN_ERR_FS_CORRUPT, NULL, + _("Bad merge; ancestor, source, and target not all in same fs")); + } + + /* ### kff todo: is there any compelling reason to get the nodes in + one db transaction? Right now we don't; txn_body_get_root() gets + one node at a time. This will probably need to change: + + Jim Blandy <jimb@zwingli.cygnus.com> writes: + > svn_fs_merge needs to be a single transaction, to protect it against + > people deleting parents of nodes it's working on, etc. + */ + + /* Get the ancestor node. */ + SVN_ERR(get_root(&ancestor, ancestor_root, pool)); + + /* Get the source node. */ + SVN_ERR(get_root(&source, source_root, pool)); + + /* Open a txn for the txn root into which we're merging. */ + SVN_ERR(svn_fs_x__open_txn(&txn, ancestor_root->fs, target_root->txn, + pool)); + + /* Merge changes between ANCESTOR and SOURCE into TXN. */ + err = merge_changes(ancestor, source, txn, conflict, pool); + if (err) + { + if ((err->apr_err == SVN_ERR_FS_CONFLICT) && conflict_p) + *conflict_p = conflict->data; + return svn_error_trace(err); + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__deltify(svn_fs_t *fs, + svn_revnum_t revision, + apr_pool_t *pool) +{ + /* Deltify is a no-op for fs_x. */ + + return SVN_NO_ERROR; +} + + + +/* Directories. */ + +/* Set *TABLE_P to a newly allocated APR hash table containing the + entries of the directory at PATH in ROOT. The keys of the table + are entry names, as byte strings, excluding the final null + character; the table's values are pointers to svn_fs_dirent_t + structures. Allocate the table and its contents in POOL. */ +static svn_error_t * +x_dir_entries(apr_hash_t **table_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + dag_node_t *node; + + /* Get the entries for this path in the caller's pool. */ + SVN_ERR(get_dag(&node, root, path, FALSE, pool)); + return svn_fs_x__dag_dir_entries(table_p, node, pool); +} + +static svn_error_t * +x_dir_optimal_order(apr_array_header_t **ordered_p, + svn_fs_root_t *root, + apr_hash_t *entries, + apr_pool_t *pool) +{ + *ordered_p = svn_fs_x__order_dir_entries(root->fs, entries, pool); + + return SVN_NO_ERROR; +} + +/* Create a new directory named PATH in ROOT. The new directory has + no entries, and no properties. ROOT must be the root of a + transaction, not a revision. Do any necessary temporary allocation + in POOL. */ +static svn_error_t * +x_make_dir(svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + parent_path_t *parent_path; + dag_node_t *sub_dir; + const svn_fs_x__id_part_t *txn_id = root_txn_id(root); + + path = svn_fs__canonicalize_abspath(path, pool); + SVN_ERR(open_path(&parent_path, root, path, open_path_last_optional, + TRUE, pool)); + + /* Check (recursively) to see if some lock is 'reserving' a path at + that location, or even some child-path; if so, check that we can + use it. */ + if (root->txn_flags & SVN_FS_TXN_CHECK_LOCKS) + SVN_ERR(svn_fs_x__allow_locked_operation(path, root->fs, TRUE, FALSE, + pool)); + + /* If there's already a sub-directory by that name, complain. This + also catches the case of trying to make a subdirectory named `/'. */ + if (parent_path->node) + return SVN_FS__ALREADY_EXISTS(root, path); + + /* Create the subdirectory. */ + SVN_ERR(make_path_mutable(root, parent_path->parent, path, pool)); + SVN_ERR(svn_fs_x__dag_make_dir(&sub_dir, + parent_path->parent->node, + parent_path_path(parent_path->parent, + pool), + parent_path->entry, + txn_id, + pool)); + + /* Add this directory to the path cache. */ + SVN_ERR(dag_node_cache_set(root, parent_path_path(parent_path, pool), + sub_dir, pool)); + + /* Make a record of this modification in the changes table. */ + return add_change(root->fs, txn_id, path, svn_fs_x__dag_get_id(sub_dir), + svn_fs_path_change_add, FALSE, FALSE, svn_node_dir, + SVN_INVALID_REVNUM, NULL, pool); +} + + +/* Delete the node at PATH under ROOT. ROOT must be a transaction + root. Perform temporary allocations in POOL. */ +static svn_error_t * +x_delete_node(svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + parent_path_t *parent_path; + const svn_fs_x__id_part_t *txn_id; + apr_int64_t mergeinfo_count = 0; + svn_node_kind_t kind; + + if (! root->is_txn_root) + return SVN_FS__NOT_TXN(root); + + txn_id = root_txn_id(root); + path = svn_fs__canonicalize_abspath(path, pool); + SVN_ERR(open_path(&parent_path, root, path, 0, TRUE, pool)); + kind = svn_fs_x__dag_node_kind(parent_path->node); + + /* We can't remove the root of the filesystem. */ + if (! parent_path->parent) + return svn_error_create(SVN_ERR_FS_ROOT_DIR, NULL, + _("The root directory cannot be deleted")); + + /* Check to see if path (or any child thereof) is locked; if so, + check that we can use the existing lock(s). */ + if (root->txn_flags & SVN_FS_TXN_CHECK_LOCKS) + SVN_ERR(svn_fs_x__allow_locked_operation(path, root->fs, TRUE, FALSE, + pool)); + + /* Make the parent directory mutable, and do the deletion. */ + SVN_ERR(make_path_mutable(root, parent_path->parent, path, pool)); + SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_count, + parent_path->node)); + SVN_ERR(svn_fs_x__dag_delete(parent_path->parent->node, + parent_path->entry, + txn_id, pool)); + + /* Remove this node and any children from the path cache. */ + SVN_ERR(dag_node_cache_invalidate(root, parent_path_path(parent_path, pool), + pool)); + + /* Update mergeinfo counts for parents */ + if (mergeinfo_count > 0) + SVN_ERR(increment_mergeinfo_up_tree(parent_path->parent, + -mergeinfo_count, + pool)); + + /* Make a record of this modification in the changes table. */ + return add_change(root->fs, txn_id, path, + svn_fs_x__dag_get_id(parent_path->node), + svn_fs_path_change_delete, FALSE, FALSE, kind, + SVN_INVALID_REVNUM, NULL, pool); +} + + +/* Set *SAME_P to TRUE if FS1 and FS2 have the same UUID, else set to FALSE. + Use POOL for temporary allocation only. + Note: this code is duplicated between libsvn_fs_x and libsvn_fs_base. */ +static svn_error_t * +x_same_p(svn_boolean_t *same_p, + svn_fs_t *fs1, + svn_fs_t *fs2, + apr_pool_t *pool) +{ + *same_p = ! strcmp(fs1->uuid, fs2->uuid); + return SVN_NO_ERROR; +} + +/* Copy the node at FROM_PATH under FROM_ROOT to TO_PATH under + TO_ROOT. If PRESERVE_HISTORY is set, then the copy is recorded in + the copies table. Perform temporary allocations in POOL. */ +static svn_error_t * +copy_helper(svn_fs_root_t *from_root, + const char *from_path, + svn_fs_root_t *to_root, + const char *to_path, + svn_boolean_t preserve_history, + apr_pool_t *pool) +{ + dag_node_t *from_node; + parent_path_t *to_parent_path; + const svn_fs_x__id_part_t *txn_id = root_txn_id(to_root); + svn_boolean_t same_p; + + /* Use an error check, not an assert, because even the caller cannot + guarantee that a filesystem's UUID has not changed "on the fly". */ + SVN_ERR(x_same_p(&same_p, from_root->fs, to_root->fs, pool)); + if (! same_p) + return svn_error_createf + (SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("Cannot copy between two different filesystems ('%s' and '%s')"), + from_root->fs->path, to_root->fs->path); + + if (from_root->is_txn_root) + return svn_error_create + (SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("Copy from mutable tree not currently supported")); + + /* Get the NODE for FROM_PATH in FROM_ROOT.*/ + SVN_ERR(get_dag(&from_node, from_root, from_path, TRUE, pool)); + + /* Build up the parent path from TO_PATH in TO_ROOT. If the last + component does not exist, it's not that big a deal. We'll just + make one there. */ + SVN_ERR(open_path(&to_parent_path, to_root, to_path, + open_path_last_optional, TRUE, pool)); + + /* Check to see if path (or any child thereof) is locked; if so, + check that we can use the existing lock(s). */ + if (to_root->txn_flags & SVN_FS_TXN_CHECK_LOCKS) + SVN_ERR(svn_fs_x__allow_locked_operation(to_path, to_root->fs, + TRUE, FALSE, pool)); + + /* If the destination node already exists as the same node as the + source (in other words, this operation would result in nothing + happening at all), just do nothing an return successfully, + proud that you saved yourself from a tiresome task. */ + if (to_parent_path->node && + svn_fs_x__id_eq(svn_fs_x__dag_get_id(from_node), + svn_fs_x__dag_get_id(to_parent_path->node))) + return SVN_NO_ERROR; + + if (! from_root->is_txn_root) + { + svn_fs_path_change_kind_t kind; + dag_node_t *new_node; + const char *from_canonpath; + apr_int64_t mergeinfo_start; + apr_int64_t mergeinfo_end; + + /* If TO_PATH already existed prior to the copy, note that this + operation is a replacement, not an addition. */ + if (to_parent_path->node) + { + kind = svn_fs_path_change_replace; + SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_start, + to_parent_path->node)); + } + else + { + kind = svn_fs_path_change_add; + mergeinfo_start = 0; + } + + SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_end, from_node)); + + /* Make sure the target node's parents are mutable. */ + SVN_ERR(make_path_mutable(to_root, to_parent_path->parent, + to_path, pool)); + + /* Canonicalize the copyfrom path. */ + from_canonpath = svn_fs__canonicalize_abspath(from_path, pool); + + SVN_ERR(svn_fs_x__dag_copy(to_parent_path->parent->node, + to_parent_path->entry, + from_node, + preserve_history, + from_root->rev, + from_canonpath, + txn_id, pool)); + + if (kind == svn_fs_path_change_replace) + SVN_ERR(dag_node_cache_invalidate(to_root, + parent_path_path(to_parent_path, + pool), pool)); + + if (mergeinfo_start != mergeinfo_end) + SVN_ERR(increment_mergeinfo_up_tree(to_parent_path->parent, + mergeinfo_end - mergeinfo_start, + pool)); + + /* Make a record of this modification in the changes table. */ + SVN_ERR(get_dag(&new_node, to_root, to_path, TRUE, pool)); + SVN_ERR(add_change(to_root->fs, txn_id, to_path, + svn_fs_x__dag_get_id(new_node), kind, FALSE, FALSE, + svn_fs_x__dag_node_kind(from_node), + from_root->rev, from_canonpath, pool)); + } + else + { + /* See IZ Issue #436 */ + /* Copying from transaction roots not currently available. + + ### cmpilato todo someday: make this not so. :-) Note that + when copying from mutable trees, you have to make sure that + you aren't creating a cyclic graph filesystem, and a simple + referencing operation won't cut it. Currently, we should not + be able to reach this clause, and the interface reports that + this only works from immutable trees anyway, but JimB has + stated that this requirement need not be necessary in the + future. */ + + SVN_ERR_MALFUNCTION(); + } + + return SVN_NO_ERROR; +} + + +/* Create a copy of FROM_PATH in FROM_ROOT named TO_PATH in TO_ROOT. + If FROM_PATH is a directory, copy it recursively. Temporary + allocations are from POOL.*/ +static svn_error_t * +x_copy(svn_fs_root_t *from_root, + const char *from_path, + svn_fs_root_t *to_root, + const char *to_path, + apr_pool_t *pool) +{ + return svn_error_trace(copy_helper(from_root, + svn_fs__canonicalize_abspath(from_path, + pool), + to_root, + svn_fs__canonicalize_abspath(to_path, + pool), + TRUE, pool)); +} + + +/* Create a copy of FROM_PATH in FROM_ROOT named TO_PATH in TO_ROOT. + If FROM_PATH is a directory, copy it recursively. No history is + preserved. Temporary allocations are from POOL. */ +static svn_error_t * +x_revision_link(svn_fs_root_t *from_root, + svn_fs_root_t *to_root, + const char *path, + apr_pool_t *pool) +{ + if (! to_root->is_txn_root) + return SVN_FS__NOT_TXN(to_root); + + path = svn_fs__canonicalize_abspath(path, pool); + return svn_error_trace(copy_helper(from_root, path, to_root, path, + FALSE, pool)); +} + + +/* Discover the copy ancestry of PATH under ROOT. Return a relevant + ancestor/revision combination in *PATH_P and *REV_P. Temporary + allocations are in POOL. */ +static svn_error_t * +x_copied_from(svn_revnum_t *rev_p, + const char **path_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + dag_node_t *node; + + /* There is no cached entry, look it up the old-fashioned + way. */ + SVN_ERR(get_dag(&node, root, path, TRUE, pool)); + SVN_ERR(svn_fs_x__dag_get_copyfrom_rev(rev_p, node)); + SVN_ERR(svn_fs_x__dag_get_copyfrom_path(path_p, node)); + + return SVN_NO_ERROR; +} + + + +/* Files. */ + +/* Create the empty file PATH under ROOT. Temporary allocations are + in POOL. */ +static svn_error_t * +x_make_file(svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + parent_path_t *parent_path; + dag_node_t *child; + const svn_fs_x__id_part_t *txn_id = root_txn_id(root); + + path = svn_fs__canonicalize_abspath(path, pool); + SVN_ERR(open_path(&parent_path, root, path, open_path_last_optional, + TRUE, pool)); + + /* If there's already a file by that name, complain. + This also catches the case of trying to make a file named `/'. */ + if (parent_path->node) + return SVN_FS__ALREADY_EXISTS(root, path); + + /* Check (non-recursively) to see if path is locked; if so, check + that we can use it. */ + if (root->txn_flags & SVN_FS_TXN_CHECK_LOCKS) + SVN_ERR(svn_fs_x__allow_locked_operation(path, root->fs, FALSE, FALSE, + pool)); + + /* Create the file. */ + SVN_ERR(make_path_mutable(root, parent_path->parent, path, pool)); + SVN_ERR(svn_fs_x__dag_make_file(&child, + parent_path->parent->node, + parent_path_path(parent_path->parent, + pool), + parent_path->entry, + txn_id, + pool)); + + /* Add this file to the path cache. */ + SVN_ERR(dag_node_cache_set(root, parent_path_path(parent_path, pool), child, + pool)); + + /* Make a record of this modification in the changes table. */ + return add_change(root->fs, txn_id, path, svn_fs_x__dag_get_id(child), + svn_fs_path_change_add, TRUE, FALSE, svn_node_file, + SVN_INVALID_REVNUM, NULL, pool); +} + + +/* Set *LENGTH_P to the size of the file PATH under ROOT. Temporary + allocations are in POOL. */ +static svn_error_t * +x_file_length(svn_filesize_t *length_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + dag_node_t *file; + + /* First create a dag_node_t from the root/path pair. */ + SVN_ERR(get_dag(&file, root, path, FALSE, pool)); + + /* Now fetch its length */ + return svn_fs_x__dag_file_length(length_p, file, pool); +} + + +/* Set *CHECKSUM to the checksum of type KIND for PATH under ROOT, or + NULL if that information isn't available. Temporary allocations + are from POOL. */ +static svn_error_t * +x_file_checksum(svn_checksum_t **checksum, + svn_checksum_kind_t kind, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + dag_node_t *file; + + SVN_ERR(get_dag(&file, root, path, FALSE, pool)); + return svn_fs_x__dag_file_checksum(checksum, file, kind, pool); +} + + +/* --- Machinery for svn_fs_file_contents() --- */ + +/* Set *CONTENTS to a readable stream that will return the contents of + PATH under ROOT. The stream is allocated in POOL. */ +static svn_error_t * +x_file_contents(svn_stream_t **contents, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + dag_node_t *node; + svn_stream_t *file_stream; + + /* First create a dag_node_t from the root/path pair. */ + SVN_ERR(get_dag(&node, root, path, FALSE, pool)); + + /* Then create a readable stream from the dag_node_t. */ + SVN_ERR(svn_fs_x__dag_get_contents(&file_stream, node, pool)); + + *contents = file_stream; + return SVN_NO_ERROR; +} + +/* --- End machinery for svn_fs_file_contents() --- */ + + +/* --- Machinery for svn_fs_try_process_file_contents() --- */ + +static svn_error_t * +x_try_process_file_contents(svn_boolean_t *success, + svn_fs_root_t *root, + const char *path, + svn_fs_process_contents_func_t processor, + void* baton, + apr_pool_t *pool) +{ + dag_node_t *node; + SVN_ERR(get_dag(&node, root, path, FALSE, pool)); + + return svn_fs_x__dag_try_process_file_contents(success, node, + processor, baton, pool); +} + +/* --- End machinery for svn_fs_try_process_file_contents() --- */ + + +/* --- Machinery for svn_fs_apply_textdelta() --- */ + + +/* Local baton type for all the helper functions below. */ +typedef struct txdelta_baton_t +{ + /* This is the custom-built window consumer given to us by the delta + library; it uniquely knows how to read data from our designated + "source" stream, interpret the window, and write data to our + designated "target" stream (in this case, our repos file.) */ + svn_txdelta_window_handler_t interpreter; + void *interpreter_baton; + + /* The original file info */ + svn_fs_root_t *root; + const char *path; + + /* Derived from the file info */ + dag_node_t *node; + + svn_stream_t *source_stream; + svn_stream_t *target_stream; + svn_stream_t *string_stream; + svn_stringbuf_t *target_string; + + /* MD5 digest for the base text against which a delta is to be + applied, and for the resultant fulltext, respectively. Either or + both may be null, in which case ignored. */ + svn_checksum_t *base_checksum; + svn_checksum_t *result_checksum; + + /* Pool used by db txns */ + apr_pool_t *pool; + +} txdelta_baton_t; + + +/* ### see comment in window_consumer() regarding this function. */ + +/* Helper function of generic type `svn_write_fn_t'. Implements a + writable stream which appends to an svn_stringbuf_t. */ +static svn_error_t * +write_to_string(void *baton, const char *data, apr_size_t *len) +{ + txdelta_baton_t *tb = (txdelta_baton_t *) baton; + svn_stringbuf_appendbytes(tb->target_string, data, *len); + return SVN_NO_ERROR; +} + + + +/* The main window handler returned by svn_fs_apply_textdelta. */ +static svn_error_t * +window_consumer(svn_txdelta_window_t *window, void *baton) +{ + txdelta_baton_t *tb = (txdelta_baton_t *) baton; + + /* Send the window right through to the custom window interpreter. + In theory, the interpreter will then write more data to + cb->target_string. */ + SVN_ERR(tb->interpreter(window, tb->interpreter_baton)); + + /* ### the write_to_string() callback for the txdelta's output stream + ### should be doing all the flush determination logic, not here. + ### in a drastic case, a window could generate a LOT more than the + ### maximum buffer size. we want to flush to the underlying target + ### stream much sooner (e.g. also in a streamy fashion). also, by + ### moving this logic inside the stream, the stream becomes nice + ### and encapsulated: it holds all the logic about buffering and + ### flushing. + ### + ### further: I believe the buffering should be removed from tree.c + ### the buffering should go into the target_stream itself, which + ### is defined by reps-string.c. Specifically, I think the + ### rep_write_contents() function will handle the buffering and + ### the spill to the underlying DB. by locating it there, then + ### anybody who gets a writable stream for FS content can take + ### advantage of the buffering capability. this will be important + ### when we export an FS API function for writing a fulltext into + ### the FS, rather than forcing that fulltext thru apply_textdelta. + */ + + /* Check to see if we need to purge the portion of the contents that + have been written thus far. */ + if ((! window) || (tb->target_string->len > WRITE_BUFFER_SIZE)) + { + apr_size_t len = tb->target_string->len; + SVN_ERR(svn_stream_write(tb->target_stream, + tb->target_string->data, + &len)); + svn_stringbuf_setempty(tb->target_string); + } + + /* Is the window NULL? If so, we're done. */ + if (! window) + { + /* Close the internal-use stream. ### This used to be inside of + txn_body_fulltext_finalize_edits(), but that invoked a nested + Berkeley DB transaction -- scandalous! */ + SVN_ERR(svn_stream_close(tb->target_stream)); + + SVN_ERR(svn_fs_x__dag_finalize_edits(tb->node, tb->result_checksum, + tb->pool)); + } + + return SVN_NO_ERROR; +} + +/* Helper function for fs_apply_textdelta. BATON is of type + txdelta_baton_t. */ +static svn_error_t * +apply_textdelta(void *baton, apr_pool_t *pool) +{ + txdelta_baton_t *tb = (txdelta_baton_t *) baton; + parent_path_t *parent_path; + const svn_fs_x__id_part_t *txn_id = root_txn_id(tb->root); + + /* Call open_path with no flags, as we want this to return an error + if the node for which we are searching doesn't exist. */ + SVN_ERR(open_path(&parent_path, tb->root, tb->path, 0, TRUE, pool)); + + /* Check (non-recursively) to see if path is locked; if so, check + that we can use it. */ + if (tb->root->txn_flags & SVN_FS_TXN_CHECK_LOCKS) + SVN_ERR(svn_fs_x__allow_locked_operation(tb->path, tb->root->fs, + FALSE, FALSE, pool)); + + /* Now, make sure this path is mutable. */ + SVN_ERR(make_path_mutable(tb->root, parent_path, tb->path, pool)); + tb->node = parent_path->node; + + if (tb->base_checksum) + { + svn_checksum_t *checksum; + + /* Until we finalize the node, its data_key points to the old + contents, in other words, the base text. */ + SVN_ERR(svn_fs_x__dag_file_checksum(&checksum, tb->node, + tb->base_checksum->kind, pool)); + if (!svn_checksum_match(tb->base_checksum, checksum)) + return svn_checksum_mismatch_err(tb->base_checksum, checksum, pool, + _("Base checksum mismatch on '%s'"), + tb->path); + } + + /* Make a readable "source" stream out of the current contents of + ROOT/PATH; obviously, this must done in the context of a db_txn. + The stream is returned in tb->source_stream. */ + SVN_ERR(svn_fs_x__dag_get_contents(&(tb->source_stream), + tb->node, tb->pool)); + + /* Make a writable "target" stream */ + SVN_ERR(svn_fs_x__dag_get_edit_stream(&(tb->target_stream), tb->node, + tb->pool)); + + /* Make a writable "string" stream which writes data to + tb->target_string. */ + tb->target_string = svn_stringbuf_create_empty(tb->pool); + tb->string_stream = svn_stream_create(tb, tb->pool); + svn_stream_set_write(tb->string_stream, write_to_string); + + /* Now, create a custom window handler that uses our two streams. */ + svn_txdelta_apply(tb->source_stream, + tb->string_stream, + NULL, + tb->path, + tb->pool, + &(tb->interpreter), + &(tb->interpreter_baton)); + + /* Make a record of this modification in the changes table. */ + return add_change(tb->root->fs, txn_id, tb->path, + svn_fs_x__dag_get_id(tb->node), + svn_fs_path_change_modify, TRUE, FALSE, svn_node_file, + SVN_INVALID_REVNUM, NULL, pool); +} + + +/* Set *CONTENTS_P and *CONTENTS_BATON_P to a window handler and baton + that will accept text delta windows to modify the contents of PATH + under ROOT. Allocations are in POOL. */ +static svn_error_t * +x_apply_textdelta(svn_txdelta_window_handler_t *contents_p, + void **contents_baton_p, + svn_fs_root_t *root, + const char *path, + svn_checksum_t *base_checksum, + svn_checksum_t *result_checksum, + apr_pool_t *pool) +{ + txdelta_baton_t *tb = apr_pcalloc(pool, sizeof(*tb)); + + tb->root = root; + tb->path = svn_fs__canonicalize_abspath(path, pool); + tb->pool = pool; + tb->base_checksum = svn_checksum_dup(base_checksum, pool); + tb->result_checksum = svn_checksum_dup(result_checksum, pool); + + SVN_ERR(apply_textdelta(tb, pool)); + + *contents_p = window_consumer; + *contents_baton_p = tb; + return SVN_NO_ERROR; +} + +/* --- End machinery for svn_fs_apply_textdelta() --- */ + +/* --- Machinery for svn_fs_apply_text() --- */ + +/* Baton for svn_fs_apply_text(). */ +struct text_baton_t +{ + /* The original file info */ + svn_fs_root_t *root; + const char *path; + + /* Derived from the file info */ + dag_node_t *node; + + /* The returned stream that will accept the file's new contents. */ + svn_stream_t *stream; + + /* The actual fs stream that the returned stream will write to. */ + svn_stream_t *file_stream; + + /* MD5 digest for the final fulltext written to the file. May + be null, in which case ignored. */ + svn_checksum_t *result_checksum; + + /* Pool used by db txns */ + apr_pool_t *pool; +}; + + +/* A wrapper around svn_fs_x__dag_finalize_edits, but for + * fulltext data, not text deltas. Closes BATON->file_stream. + * + * Note: If you're confused about how this function relates to another + * of similar name, think of it this way: + * + * svn_fs_apply_textdelta() ==> ... ==> txn_body_txdelta_finalize_edits() + * svn_fs_apply_text() ==> ... ==> txn_body_fulltext_finalize_edits() + */ + +/* Write function for the publically returned stream. */ +static svn_error_t * +text_stream_writer(void *baton, + const char *data, + apr_size_t *len) +{ + struct text_baton_t *tb = baton; + + /* Psst, here's some data. Pass it on to the -real- file stream. */ + return svn_stream_write(tb->file_stream, data, len); +} + +/* Close function for the publically returned stream. */ +static svn_error_t * +text_stream_closer(void *baton) +{ + struct text_baton_t *tb = baton; + + /* Close the internal-use stream. ### This used to be inside of + txn_body_fulltext_finalize_edits(), but that invoked a nested + Berkeley DB transaction -- scandalous! */ + SVN_ERR(svn_stream_close(tb->file_stream)); + + /* Need to tell fs that we're done sending text */ + return svn_fs_x__dag_finalize_edits(tb->node, tb->result_checksum, + tb->pool); +} + + +/* Helper function for fs_apply_text. BATON is of type + text_baton_t. */ +static svn_error_t * +apply_text(void *baton, apr_pool_t *pool) +{ + struct text_baton_t *tb = baton; + parent_path_t *parent_path; + const svn_fs_x__id_part_t *txn_id = root_txn_id(tb->root); + + /* Call open_path with no flags, as we want this to return an error + if the node for which we are searching doesn't exist. */ + SVN_ERR(open_path(&parent_path, tb->root, tb->path, 0, TRUE, pool)); + + /* Check (non-recursively) to see if path is locked; if so, check + that we can use it. */ + if (tb->root->txn_flags & SVN_FS_TXN_CHECK_LOCKS) + SVN_ERR(svn_fs_x__allow_locked_operation(tb->path, tb->root->fs, + FALSE, FALSE, pool)); + + /* Now, make sure this path is mutable. */ + SVN_ERR(make_path_mutable(tb->root, parent_path, tb->path, pool)); + tb->node = parent_path->node; + + /* Make a writable stream for replacing the file's text. */ + SVN_ERR(svn_fs_x__dag_get_edit_stream(&(tb->file_stream), tb->node, + tb->pool)); + + /* Create a 'returnable' stream which writes to the file_stream. */ + tb->stream = svn_stream_create(tb, tb->pool); + svn_stream_set_write(tb->stream, text_stream_writer); + svn_stream_set_close(tb->stream, text_stream_closer); + + /* Make a record of this modification in the changes table. */ + return add_change(tb->root->fs, txn_id, tb->path, + svn_fs_x__dag_get_id(tb->node), + svn_fs_path_change_modify, TRUE, FALSE, svn_node_file, + SVN_INVALID_REVNUM, NULL, pool); +} + + +/* Return a writable stream that will set the contents of PATH under + ROOT. RESULT_CHECKSUM is the MD5 checksum of the final result. + Temporary allocations are in POOL. */ +static svn_error_t * +x_apply_text(svn_stream_t **contents_p, + svn_fs_root_t *root, + const char *path, + svn_checksum_t *result_checksum, + apr_pool_t *pool) +{ + struct text_baton_t *tb = apr_pcalloc(pool, sizeof(*tb)); + + tb->root = root; + tb->path = svn_fs__canonicalize_abspath(path, pool); + tb->pool = pool; + tb->result_checksum = svn_checksum_dup(result_checksum, pool); + + SVN_ERR(apply_text(tb, pool)); + + *contents_p = tb->stream; + return SVN_NO_ERROR; +} + +/* --- End machinery for svn_fs_apply_text() --- */ + + +/* Check if the contents of PATH1 under ROOT1 are different from the + contents of PATH2 under ROOT2. If they are different set + *CHANGED_P to TRUE, otherwise set it to FALSE. */ +static svn_error_t * +x_contents_changed(svn_boolean_t *changed_p, + svn_fs_root_t *root1, + const char *path1, + svn_fs_root_t *root2, + const char *path2, + apr_pool_t *pool) +{ + dag_node_t *node1, *node2; + + /* Check that roots are in the same fs. */ + if (root1->fs != root2->fs) + return svn_error_create + (SVN_ERR_FS_GENERAL, NULL, + _("Cannot compare file contents between two different filesystems")); + + /* Check that both paths are files. */ + { + svn_node_kind_t kind; + + SVN_ERR(svn_fs_x__check_path(&kind, root1, path1, pool)); + if (kind != svn_node_file) + return svn_error_createf + (SVN_ERR_FS_GENERAL, NULL, _("'%s' is not a file"), path1); + + SVN_ERR(svn_fs_x__check_path(&kind, root2, path2, pool)); + if (kind != svn_node_file) + return svn_error_createf + (SVN_ERR_FS_GENERAL, NULL, _("'%s' is not a file"), path2); + } + + SVN_ERR(get_dag(&node1, root1, path1, TRUE, pool)); + SVN_ERR(get_dag(&node2, root2, path2, TRUE, pool)); + return svn_fs_x__dag_things_different(NULL, changed_p, node1, node2); +} + + + +/* Public interface to computing file text deltas. */ + +static svn_error_t * +x_get_file_delta_stream(svn_txdelta_stream_t **stream_p, + svn_fs_root_t *source_root, + const char *source_path, + svn_fs_root_t *target_root, + const char *target_path, + apr_pool_t *pool) +{ + dag_node_t *source_node, *target_node; + + if (source_root && source_path) + SVN_ERR(get_dag(&source_node, source_root, source_path, TRUE, pool)); + else + source_node = NULL; + SVN_ERR(get_dag(&target_node, target_root, target_path, TRUE, pool)); + + /* Create a delta stream that turns the source into the target. */ + return svn_fs_x__dag_get_file_delta_stream(stream_p, source_node, + target_node, pool); +} + + + +/* Finding Changes */ + +/* Set *CHANGED_PATHS_P to a newly allocated hash containing + descriptions of the paths changed under ROOT. The hash is keyed + with const char * paths and has svn_fs_path_change2_t * values. Use + POOL for all allocations. */ +static svn_error_t * +x_paths_changed(apr_hash_t **changed_paths_p, + svn_fs_root_t *root, + apr_pool_t *pool) +{ + if (root->is_txn_root) + return svn_fs_x__txn_changes_fetch(changed_paths_p, root->fs, + root_txn_id(root), pool); + else + return svn_fs_x__paths_changed(changed_paths_p, root->fs, root->rev, + pool); +} + + + +/* Our coolio opaque history object. */ +typedef struct fs_history_data_t +{ + /* filesystem object */ + svn_fs_t *fs; + + /* path and revision of historical location */ + const char *path; + svn_revnum_t revision; + + /* internal-use hints about where to resume the history search. */ + const char *path_hint; + svn_revnum_t rev_hint; + + /* FALSE until the first call to svn_fs_history_prev(). */ + svn_boolean_t is_interesting; +} fs_history_data_t; + +static svn_fs_history_t * +assemble_history(svn_fs_t *fs, + const char *path, + svn_revnum_t revision, + svn_boolean_t is_interesting, + const char *path_hint, + svn_revnum_t rev_hint, + apr_pool_t *pool); + + +/* Set *HISTORY_P to an opaque node history object which represents + PATH under ROOT. ROOT must be a revision root. Use POOL for all + allocations. */ +static svn_error_t * +x_node_history(svn_fs_history_t **history_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + svn_node_kind_t kind; + + /* We require a revision root. */ + if (root->is_txn_root) + return svn_error_create(SVN_ERR_FS_NOT_REVISION_ROOT, NULL, NULL); + + /* And we require that the path exist in the root. */ + SVN_ERR(svn_fs_x__check_path(&kind, root, path, pool)); + if (kind == svn_node_none) + return SVN_FS__NOT_FOUND(root, path); + + /* Okay, all seems well. Build our history object and return it. */ + *history_p = assemble_history(root->fs, + svn_fs__canonicalize_abspath(path, pool), + root->rev, FALSE, NULL, + SVN_INVALID_REVNUM, pool); + return SVN_NO_ERROR; +} + +/* Find the youngest copyroot for path PARENT_PATH or its parents in + filesystem FS, and store the copyroot in *REV_P and *PATH_P. + Perform all allocations in POOL. */ +static svn_error_t * +find_youngest_copyroot(svn_revnum_t *rev_p, + const char **path_p, + svn_fs_t *fs, + parent_path_t *parent_path, + apr_pool_t *pool) +{ + svn_revnum_t rev_mine; + svn_revnum_t rev_parent = SVN_INVALID_REVNUM; + const char *path_mine; + const char *path_parent = NULL; + + /* First find our parent's youngest copyroot. */ + if (parent_path->parent) + SVN_ERR(find_youngest_copyroot(&rev_parent, &path_parent, fs, + parent_path->parent, pool)); + + /* Find our copyroot. */ + SVN_ERR(svn_fs_x__dag_get_copyroot(&rev_mine, &path_mine, + parent_path->node)); + + /* If a parent and child were copied to in the same revision, prefer + the child copy target, since it is the copy relevant to the + history of the child. */ + if (rev_mine >= rev_parent) + { + *rev_p = rev_mine; + *path_p = path_mine; + } + else + { + *rev_p = rev_parent; + *path_p = path_parent; + } + + return SVN_NO_ERROR; +} + + +static +svn_error_t *x_closest_copy(svn_fs_root_t **root_p, + const char **path_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + svn_fs_t *fs = root->fs; + parent_path_t *parent_path, *copy_dst_parent_path; + svn_revnum_t copy_dst_rev, created_rev; + const char *copy_dst_path; + svn_fs_root_t *copy_dst_root; + dag_node_t *copy_dst_node; + svn_node_kind_t kind; + + /* Initialize return values. */ + *root_p = NULL; + *path_p = NULL; + + path = svn_fs__canonicalize_abspath(path, pool); + SVN_ERR(open_path(&parent_path, root, path, 0, FALSE, pool)); + + /* Find the youngest copyroot in the path of this node-rev, which + will indicate the target of the innermost copy affecting the + node-rev. */ + SVN_ERR(find_youngest_copyroot(©_dst_rev, ©_dst_path, + fs, parent_path, pool)); + if (copy_dst_rev == 0) /* There are no copies affecting this node-rev. */ + return SVN_NO_ERROR; + + /* It is possible that this node was created from scratch at some + revision between COPY_DST_REV and REV. Make sure that PATH + exists as of COPY_DST_REV and is related to this node-rev. */ + SVN_ERR(svn_fs_x__revision_root(©_dst_root, fs, copy_dst_rev, pool)); + SVN_ERR(svn_fs_x__check_path(&kind, copy_dst_root, path, pool)); + if (kind == svn_node_none) + return SVN_NO_ERROR; + SVN_ERR(open_path(©_dst_parent_path, copy_dst_root, path, + open_path_node_only, FALSE, pool)); + copy_dst_node = copy_dst_parent_path->node; + if (! svn_fs_x__id_check_related(svn_fs_x__dag_get_id(copy_dst_node), + svn_fs_x__dag_get_id(parent_path->node))) + return SVN_NO_ERROR; + + /* One final check must be done here. If you copy a directory and + create a new entity somewhere beneath that directory in the same + txn, then we can't claim that the copy affected the new entity. + For example, if you do: + + copy dir1 dir2 + create dir2/new-thing + commit + + then dir2/new-thing was not affected by the copy of dir1 to dir2. + We detect this situation by asking if PATH@COPY_DST_REV's + created-rev is COPY_DST_REV, and that node-revision has no + predecessors, then there is no relevant closest copy. + */ + SVN_ERR(svn_fs_x__dag_get_revision(&created_rev, copy_dst_node, pool)); + if (created_rev == copy_dst_rev) + { + const svn_fs_id_t *pred; + SVN_ERR(svn_fs_x__dag_get_predecessor_id(&pred, copy_dst_node)); + if (! pred) + return SVN_NO_ERROR; + } + + /* The copy destination checks out. Return it. */ + *root_p = copy_dst_root; + *path_p = copy_dst_path; + return SVN_NO_ERROR; +} + + +/* Set *PREV_PATH and *PREV_REV to the path and revision which + represent the location at which PATH in FS was located immediately + prior to REVISION iff there was a copy operation (to PATH or one of + its parent directories) between that previous location and + PATH@REVISION. + + If there was no such copy operation in that portion of PATH's + history, set *PREV_PATH to NULL and *PREV_REV to SVN_INVALID_REVNUM. */ +static svn_error_t * +prev_location(const char **prev_path, + svn_revnum_t *prev_rev, + svn_fs_t *fs, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + const char *copy_path, *copy_src_path, *remainder_path; + svn_fs_root_t *copy_root; + svn_revnum_t copy_src_rev; + + /* Ask about the most recent copy which affected PATH@REVISION. If + there was no such copy, we're done. */ + SVN_ERR(x_closest_copy(©_root, ©_path, root, path, pool)); + if (! copy_root) + { + *prev_rev = SVN_INVALID_REVNUM; + *prev_path = NULL; + return SVN_NO_ERROR; + } + + /* Ultimately, it's not the path of the closest copy's source that + we care about -- it's our own path's location in the copy source + revision. So we'll tack the relative path that expresses the + difference between the copy destination and our path in the copy + revision onto the copy source path to determine this information. + + In other words, if our path is "/branches/my-branch/foo/bar", and + we know that the closest relevant copy was a copy of "/trunk" to + "/branches/my-branch", then that relative path under the copy + destination is "/foo/bar". Tacking that onto the copy source + path tells us that our path was located at "/trunk/foo/bar" + before the copy. + */ + SVN_ERR(x_copied_from(©_src_rev, ©_src_path, + copy_root, copy_path, pool)); + remainder_path = svn_fspath__skip_ancestor(copy_path, path); + *prev_path = svn_fspath__join(copy_src_path, remainder_path, pool); + *prev_rev = copy_src_rev; + return SVN_NO_ERROR; +} + + +static svn_error_t * +x_node_origin_rev(svn_revnum_t *revision, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool) +{ + svn_fs_t *fs = root->fs; + const svn_fs_id_t *given_noderev_id, *cached_origin_id; + const svn_fs_x__id_part_t *node_id; + + path = svn_fs__canonicalize_abspath(path, pool); + + /* Check the cache first. */ + SVN_ERR(svn_fs_x__node_id(&given_noderev_id, root, path, pool)); + node_id = svn_fs_x__id_node_id(given_noderev_id); + + /* Is it a brand new uncommitted node or a new-style one? + * (committed old-style nodes will have a 0 revision value; + * rev 0, number 0 is rev 0 root node) */ + if (node_id->revision != 0 || node_id->number == 0) + { + *revision = node_id->revision; + return SVN_NO_ERROR; + } + + /* The root node always has ID 0, created in revision 0 and will never + use the new-style ID format. */ + if (node_id->number == 0) + { + *revision = 0; + return SVN_NO_ERROR; + } + + /* OK, it's an old-style ID? Maybe it's cached. */ + SVN_ERR(svn_fs_x__get_node_origin(&cached_origin_id, fs, node_id, pool)); + if (cached_origin_id != NULL) + { + *revision = svn_fs_x__id_rev(cached_origin_id); + return SVN_NO_ERROR; + } + + { + /* Ah well, the answer isn't in the ID itself or in the cache. + Let's actually calculate it, then. */ + svn_fs_root_t *curroot = root; + apr_pool_t *subpool = svn_pool_create(pool); + apr_pool_t *predidpool = svn_pool_create(pool); + svn_stringbuf_t *lastpath = svn_stringbuf_create(path, pool); + svn_revnum_t lastrev = SVN_INVALID_REVNUM; + dag_node_t *node; + const svn_fs_id_t *pred_id; + + /* Walk the closest-copy chain back to the first copy in our history. + + NOTE: We merely *assume* that this is faster than walking the + predecessor chain, because we *assume* that copies of parent + directories happen less often than modifications to a given item. */ + while (1) + { + svn_revnum_t currev; + const char *curpath = lastpath->data; + + svn_pool_clear(subpool); + + /* Get a root pointing to LASTREV. (The first time around, + LASTREV is invalid, but that's cool because CURROOT is + already initialized.) */ + if (SVN_IS_VALID_REVNUM(lastrev)) + SVN_ERR(svn_fs_x__revision_root(&curroot, fs, lastrev, subpool)); + + /* Find the previous location using the closest-copy shortcut. */ + SVN_ERR(prev_location(&curpath, &currev, fs, curroot, curpath, + subpool)); + if (! curpath) + break; + + /* Update our LASTPATH and LASTREV variables (which survive + SUBPOOL). */ + svn_stringbuf_set(lastpath, curpath); + lastrev = currev; + } + + /* Walk the predecessor links back to origin. */ + SVN_ERR(svn_fs_x__node_id(&pred_id, curroot, lastpath->data, predidpool)); + do + { + svn_pool_clear(subpool); + SVN_ERR(svn_fs_x__dag_get_node(&node, fs, pred_id, subpool)); + + /* Why not just fetch the predecessor ID in PREDIDPOOL? + Because svn_fs_x__dag_get_predecessor_id() doesn't + necessarily honor the passed-in pool, and might return a + value cached in the node (which is allocated in + SUBPOOL... maybe). */ + svn_pool_clear(predidpool); + SVN_ERR(svn_fs_x__dag_get_predecessor_id(&pred_id, node)); + pred_id = pred_id ? svn_fs_x__id_copy(pred_id, predidpool) : NULL; + } + while (pred_id); + + /* When we get here, NODE should be the first node-revision in our + chain. */ + SVN_ERR(svn_fs_x__dag_get_revision(revision, node, pool)); + + /* Wow, I don't want to have to do all that again. Let's cache + the result. */ + if (node_id->revision != SVN_INVALID_REVNUM) + SVN_ERR(svn_fs_x__set_node_origin(fs, node_id, + svn_fs_x__dag_get_id(node), pool)); + + svn_pool_destroy(subpool); + svn_pool_destroy(predidpool); + return SVN_NO_ERROR; + } +} + + +static svn_error_t * +history_prev(svn_fs_history_t **prev_history, + svn_fs_history_t *history, + svn_boolean_t cross_copies, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + fs_history_data_t *fhd = history->fsap_data; + const char *commit_path, *src_path, *path = fhd->path; + svn_revnum_t commit_rev, src_rev, dst_rev; + svn_revnum_t revision = fhd->revision; + svn_fs_t *fs = fhd->fs; + parent_path_t *parent_path; + dag_node_t *node; + svn_fs_root_t *root; + svn_boolean_t reported = fhd->is_interesting; + svn_revnum_t copyroot_rev; + const char *copyroot_path; + + /* Initialize our return value. */ + *prev_history = NULL; + + /* If our last history report left us hints about where to pickup + the chase, then our last report was on the destination of a + copy. If we are crossing copies, start from those locations, + otherwise, we're all done here. */ + if (fhd->path_hint && SVN_IS_VALID_REVNUM(fhd->rev_hint)) + { + reported = FALSE; + if (! cross_copies) + return SVN_NO_ERROR; + path = fhd->path_hint; + revision = fhd->rev_hint; + } + + /* Construct a ROOT for the current revision. */ + SVN_ERR(svn_fs_x__revision_root(&root, fs, revision, scratch_pool)); + + /* Open PATH/REVISION, and get its node and a bunch of other + goodies. */ + SVN_ERR(open_path(&parent_path, root, path, 0, FALSE, scratch_pool)); + node = parent_path->node; + commit_path = svn_fs_x__dag_get_created_path(node); + SVN_ERR(svn_fs_x__dag_get_revision(&commit_rev, node, scratch_pool)); + + /* The Subversion filesystem is written in such a way that a given + line of history may have at most one interesting history point + per filesystem revision. Either that node was edited (and + possibly copied), or it was copied but not edited. And a copy + source cannot be from the same revision as its destination. So, + if our history revision matches its node's commit revision, we + know that ... */ + if (revision == commit_rev) + { + if (! reported) + { + /* ... we either have not yet reported on this revision (and + need now to do so) ... */ + *prev_history = assemble_history(fs, + apr_pstrdup(result_pool, commit_path), + commit_rev, TRUE, NULL, + SVN_INVALID_REVNUM, result_pool); + return SVN_NO_ERROR; + } + else + { + /* ... or we *have* reported on this revision, and must now + progress toward this node's predecessor (unless there is + no predecessor, in which case we're all done!). */ + const svn_fs_id_t *pred_id; + + SVN_ERR(svn_fs_x__dag_get_predecessor_id(&pred_id, node)); + if (! pred_id) + return SVN_NO_ERROR; + + /* Replace NODE and friends with the information from its + predecessor. */ + SVN_ERR(svn_fs_x__dag_get_node(&node, fs, pred_id, scratch_pool)); + commit_path = svn_fs_x__dag_get_created_path(node); + SVN_ERR(svn_fs_x__dag_get_revision(&commit_rev, node, scratch_pool)); + } + } + + /* Find the youngest copyroot in the path of this node, including + itself. */ + SVN_ERR(find_youngest_copyroot(©root_rev, ©root_path, fs, + parent_path, scratch_pool)); + + /* Initialize some state variables. */ + src_path = NULL; + src_rev = SVN_INVALID_REVNUM; + dst_rev = SVN_INVALID_REVNUM; + + if (copyroot_rev > commit_rev) + { + const char *remainder_path; + const char *copy_dst, *copy_src; + svn_fs_root_t *copyroot_root; + + SVN_ERR(svn_fs_x__revision_root(©root_root, fs, copyroot_rev, + scratch_pool)); + SVN_ERR(get_dag(&node, copyroot_root, copyroot_path, FALSE, scratch_pool)); + copy_dst = svn_fs_x__dag_get_created_path(node); + + /* If our current path was the very destination of the copy, + then our new current path will be the copy source. If our + current path was instead the *child* of the destination of + the copy, then figure out its previous location by taking its + path relative to the copy destination and appending that to + the copy source. Finally, if our current path doesn't meet + one of these other criteria ... ### for now just fallback to + the old copy hunt algorithm. */ + remainder_path = svn_fspath__skip_ancestor(copy_dst, path); + + if (remainder_path) + { + /* If we get here, then our current path is the destination + of, or the child of the destination of, a copy. Fill + in the return values and get outta here. */ + SVN_ERR(svn_fs_x__dag_get_copyfrom_rev(&src_rev, node)); + SVN_ERR(svn_fs_x__dag_get_copyfrom_path(©_src, node)); + + dst_rev = copyroot_rev; + src_path = svn_fspath__join(copy_src, remainder_path, scratch_pool); + } + } + + /* If we calculated a copy source path and revision, we'll make a + 'copy-style' history object. */ + if (src_path && SVN_IS_VALID_REVNUM(src_rev)) + { + svn_boolean_t retry = FALSE; + + /* It's possible for us to find a copy location that is the same + as the history point we've just reported. If that happens, + we simply need to take another trip through this history + search. */ + if ((dst_rev == revision) && reported) + retry = TRUE; + + *prev_history = assemble_history(fs, apr_pstrdup(result_pool, path), + dst_rev, ! retry, + src_path, src_rev, result_pool); + } + else + { + *prev_history = assemble_history(fs, apr_pstrdup(result_pool, commit_path), + commit_rev, TRUE, NULL, + SVN_INVALID_REVNUM, result_pool); + } + + return SVN_NO_ERROR; +} + + +/* Implement svn_fs_history_prev, set *PREV_HISTORY_P to a new + svn_fs_history_t object that represents the predecessory of + HISTORY. If CROSS_COPIES is true, *PREV_HISTORY_P may be related + only through a copy operation. Perform all allocations in POOL. */ +static svn_error_t * +fs_history_prev(svn_fs_history_t **prev_history_p, + svn_fs_history_t *history, + svn_boolean_t cross_copies, + apr_pool_t *pool) +{ + svn_fs_history_t *prev_history = NULL; + fs_history_data_t *fhd = history->fsap_data; + svn_fs_t *fs = fhd->fs; + + /* Special case: the root directory changes in every single + revision, no exceptions. And, the root can't be the target (or + child of a target -- duh) of a copy. So, if that's our path, + then we need only decrement our revision by 1, and there you go. */ + if (strcmp(fhd->path, "/") == 0) + { + if (! fhd->is_interesting) + prev_history = assemble_history(fs, "/", fhd->revision, + 1, NULL, SVN_INVALID_REVNUM, pool); + else if (fhd->revision > 0) + prev_history = assemble_history(fs, "/", fhd->revision - 1, + 1, NULL, SVN_INVALID_REVNUM, pool); + } + else + { + prev_history = history; + + while (1) + { + SVN_ERR(history_prev(&prev_history, prev_history, cross_copies, + pool, pool)); + + if (! prev_history) + break; + fhd = prev_history->fsap_data; + if (fhd->is_interesting) + break; + } + } + + *prev_history_p = prev_history; + return SVN_NO_ERROR; +} + + +/* Set *PATH and *REVISION to the path and revision for the HISTORY + object. Use POOL for all allocations. */ +static svn_error_t * +fs_history_location(const char **path, + svn_revnum_t *revision, + svn_fs_history_t *history, + apr_pool_t *pool) +{ + fs_history_data_t *fhd = history->fsap_data; + + *path = apr_pstrdup(pool, fhd->path); + *revision = fhd->revision; + return SVN_NO_ERROR; +} + +static history_vtable_t history_vtable = { + fs_history_prev, + fs_history_location +}; + +/* Return a new history object (marked as "interesting") for PATH and + REVISION, allocated in POOL, and with its members set to the values + of the parameters provided. Note that PATH and PATH_HINT are not + duped into POOL -- it is the responsibility of the caller to ensure + that this happens. */ +static svn_fs_history_t * +assemble_history(svn_fs_t *fs, + const char *path, + svn_revnum_t revision, + svn_boolean_t is_interesting, + const char *path_hint, + svn_revnum_t rev_hint, + apr_pool_t *pool) +{ + svn_fs_history_t *history = apr_pcalloc(pool, sizeof(*history)); + fs_history_data_t *fhd = apr_pcalloc(pool, sizeof(*fhd)); + fhd->path = svn_fs__canonicalize_abspath(path, pool); + fhd->revision = revision; + fhd->is_interesting = is_interesting; + fhd->path_hint = path_hint; + fhd->rev_hint = rev_hint; + fhd->fs = fs; + + history->vtable = &history_vtable; + history->fsap_data = fhd; + return history; +} + + +/* mergeinfo queries */ + + +/* DIR_DAG is a directory DAG node which has mergeinfo in its + descendants. This function iterates over its children. For each + child with immediate mergeinfo, it adds its mergeinfo to + RESULT_CATALOG. appropriate arguments. For each child with + descendants with mergeinfo, it recurses. Note that it does *not* + call the action on the path for DIR_DAG itself. + + POOL is used for temporary allocations, including the mergeinfo + hashes passed to actions; RESULT_POOL is used for the mergeinfo added + to RESULT_CATALOG. + */ +static svn_error_t * +crawl_directory_dag_for_mergeinfo(svn_fs_root_t *root, + const char *this_path, + dag_node_t *dir_dag, + svn_mergeinfo_catalog_t result_catalog, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + apr_hash_t *entries; + apr_hash_index_t *hi; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + + SVN_ERR(svn_fs_x__dag_dir_entries(&entries, dir_dag, scratch_pool)); + + for (hi = apr_hash_first(scratch_pool, entries); + hi; + hi = apr_hash_next(hi)) + { + svn_fs_dirent_t *dirent = svn__apr_hash_index_val(hi); + const char *kid_path; + dag_node_t *kid_dag; + svn_boolean_t has_mergeinfo, go_down; + + svn_pool_clear(iterpool); + + kid_path = svn_fspath__join(this_path, dirent->name, iterpool); + SVN_ERR(get_dag(&kid_dag, root, kid_path, TRUE, iterpool)); + + SVN_ERR(svn_fs_x__dag_has_mergeinfo(&has_mergeinfo, kid_dag)); + SVN_ERR(svn_fs_x__dag_has_descendants_with_mergeinfo(&go_down, kid_dag)); + + if (has_mergeinfo) + { + /* Save this particular node's mergeinfo. */ + apr_hash_t *proplist; + svn_mergeinfo_t kid_mergeinfo; + svn_string_t *mergeinfo_string; + svn_error_t *err; + + SVN_ERR(svn_fs_x__dag_get_proplist(&proplist, kid_dag, iterpool)); + mergeinfo_string = svn_hash_gets(proplist, SVN_PROP_MERGEINFO); + if (!mergeinfo_string) + { + svn_string_t *idstr = svn_fs_x__id_unparse(dirent->id, iterpool); + return svn_error_createf + (SVN_ERR_FS_CORRUPT, NULL, + _("Node-revision #'%s' claims to have mergeinfo but doesn't"), + idstr->data); + } + + /* Issue #3896: If a node has syntactically invalid mergeinfo, then + treat it as if no mergeinfo is present rather than raising a parse + error. */ + err = svn_mergeinfo_parse(&kid_mergeinfo, + mergeinfo_string->data, + result_pool); + if (err) + { + if (err->apr_err == SVN_ERR_MERGEINFO_PARSE_ERROR) + svn_error_clear(err); + else + return svn_error_trace(err); + } + else + { + svn_hash_sets(result_catalog, apr_pstrdup(result_pool, kid_path), + kid_mergeinfo); + } + } + + if (go_down) + SVN_ERR(crawl_directory_dag_for_mergeinfo(root, + kid_path, + kid_dag, + result_catalog, + result_pool, + iterpool)); + } + + svn_pool_destroy(iterpool); + return SVN_NO_ERROR; +} + +/* Return the cache key as a combination of REV_ROOT->REV, the inheritance + flags INHERIT and ADJUST_INHERITED_MERGEINFO, and the PATH. The result + will be allocated in POOL.. + */ +static const char * +mergeinfo_cache_key(const char *path, + svn_fs_root_t *rev_root, + svn_mergeinfo_inheritance_t inherit, + svn_boolean_t adjust_inherited_mergeinfo, + apr_pool_t *pool) +{ + apr_int64_t number = rev_root->rev; + number = number * 4 + + (inherit == svn_mergeinfo_nearest_ancestor ? 2 : 0) + + (adjust_inherited_mergeinfo ? 1 : 0); + + return svn_fs_x__combine_number_and_string(number, path, pool); +} + +/* Calculates the mergeinfo for PATH under REV_ROOT using inheritance + type INHERIT. Returns it in *MERGEINFO, or NULL if there is none. + The result is allocated in RESULT_POOL; SCRATCH_POOL is + used for temporary allocations. + */ +static svn_error_t * +get_mergeinfo_for_path_internal(svn_mergeinfo_t *mergeinfo, + svn_fs_root_t *rev_root, + const char *path, + svn_mergeinfo_inheritance_t inherit, + svn_boolean_t adjust_inherited_mergeinfo, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + parent_path_t *parent_path, *nearest_ancestor; + apr_hash_t *proplist; + svn_string_t *mergeinfo_string; + + path = svn_fs__canonicalize_abspath(path, scratch_pool); + + SVN_ERR(open_path(&parent_path, rev_root, path, 0, FALSE, scratch_pool)); + + if (inherit == svn_mergeinfo_nearest_ancestor && ! parent_path->parent) + return SVN_NO_ERROR; + + if (inherit == svn_mergeinfo_nearest_ancestor) + nearest_ancestor = parent_path->parent; + else + nearest_ancestor = parent_path; + + while (TRUE) + { + svn_boolean_t has_mergeinfo; + + SVN_ERR(svn_fs_x__dag_has_mergeinfo(&has_mergeinfo, + nearest_ancestor->node)); + if (has_mergeinfo) + break; + + /* No need to loop if we're looking for explicit mergeinfo. */ + if (inherit == svn_mergeinfo_explicit) + { + return SVN_NO_ERROR; + } + + nearest_ancestor = nearest_ancestor->parent; + + /* Run out? There's no mergeinfo. */ + if (!nearest_ancestor) + { + return SVN_NO_ERROR; + } + } + + SVN_ERR(svn_fs_x__dag_get_proplist(&proplist, nearest_ancestor->node, + scratch_pool)); + mergeinfo_string = svn_hash_gets(proplist, SVN_PROP_MERGEINFO); + if (!mergeinfo_string) + return svn_error_createf + (SVN_ERR_FS_CORRUPT, NULL, + _("Node-revision '%s@%ld' claims to have mergeinfo but doesn't"), + parent_path_path(nearest_ancestor, scratch_pool), rev_root->rev); + + /* Parse the mergeinfo; store the result in *MERGEINFO. */ + { + /* Issue #3896: If a node has syntactically invalid mergeinfo, then + treat it as if no mergeinfo is present rather than raising a parse + error. */ + svn_error_t *err = svn_mergeinfo_parse(mergeinfo, + mergeinfo_string->data, + result_pool); + if (err) + { + if (err->apr_err == SVN_ERR_MERGEINFO_PARSE_ERROR) + { + svn_error_clear(err); + err = NULL; + *mergeinfo = NULL; + } + return svn_error_trace(err); + } + } + + /* If our nearest ancestor is the very path we inquired about, we + can return the mergeinfo results directly. Otherwise, we're + inheriting the mergeinfo, so we need to a) remove non-inheritable + ranges and b) telescope the merged-from paths. */ + if (adjust_inherited_mergeinfo && (nearest_ancestor != parent_path)) + { + svn_mergeinfo_t tmp_mergeinfo; + + SVN_ERR(svn_mergeinfo_inheritable2(&tmp_mergeinfo, *mergeinfo, + NULL, SVN_INVALID_REVNUM, + SVN_INVALID_REVNUM, TRUE, + scratch_pool, scratch_pool)); + SVN_ERR(svn_fs__append_to_merged_froms(mergeinfo, tmp_mergeinfo, + parent_path_relpath( + parent_path, nearest_ancestor, + scratch_pool), + result_pool)); + } + + return SVN_NO_ERROR; +} + +/* Caching wrapper around get_mergeinfo_for_path_internal(). + */ +static svn_error_t * +get_mergeinfo_for_path(svn_mergeinfo_t *mergeinfo, + svn_fs_root_t *rev_root, + const char *path, + svn_mergeinfo_inheritance_t inherit, + svn_boolean_t adjust_inherited_mergeinfo, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + fs_x_data_t *ffd = rev_root->fs->fsap_data; + const char *cache_key; + svn_boolean_t found = FALSE; + svn_stringbuf_t *mergeinfo_exists; + + *mergeinfo = NULL; + + cache_key = mergeinfo_cache_key(path, rev_root, inherit, + adjust_inherited_mergeinfo, scratch_pool); + if (ffd->mergeinfo_existence_cache) + { + SVN_ERR(svn_cache__get((void **)&mergeinfo_exists, &found, + ffd->mergeinfo_existence_cache, + cache_key, result_pool)); + if (found && mergeinfo_exists->data[0] == '1') + SVN_ERR(svn_cache__get((void **)mergeinfo, &found, + ffd->mergeinfo_cache, + cache_key, result_pool)); + } + + if (! found) + { + SVN_ERR(get_mergeinfo_for_path_internal(mergeinfo, rev_root, path, + inherit, + adjust_inherited_mergeinfo, + result_pool, scratch_pool)); + if (ffd->mergeinfo_existence_cache) + { + mergeinfo_exists = svn_stringbuf_create(*mergeinfo ? "1" : "0", + scratch_pool); + SVN_ERR(svn_cache__set(ffd->mergeinfo_existence_cache, + cache_key, mergeinfo_exists, scratch_pool)); + if (*mergeinfo) + SVN_ERR(svn_cache__set(ffd->mergeinfo_cache, + cache_key, *mergeinfo, scratch_pool)); + } + } + + return SVN_NO_ERROR; +} + +/* Adds mergeinfo for each descendant of PATH (but not PATH itself) + under ROOT to RESULT_CATALOG. Returned values are allocated in + RESULT_POOL; temporary values in POOL. */ +static svn_error_t * +add_descendant_mergeinfo(svn_mergeinfo_catalog_t result_catalog, + svn_fs_root_t *root, + const char *path, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + dag_node_t *this_dag; + svn_boolean_t go_down; + + SVN_ERR(get_dag(&this_dag, root, path, TRUE, scratch_pool)); + SVN_ERR(svn_fs_x__dag_has_descendants_with_mergeinfo(&go_down, + this_dag)); + if (go_down) + SVN_ERR(crawl_directory_dag_for_mergeinfo(root, + path, + this_dag, + result_catalog, + result_pool, + scratch_pool)); + return SVN_NO_ERROR; +} + + +/* Get the mergeinfo for a set of paths, returned in + *MERGEINFO_CATALOG. Returned values are allocated in + POOL, while temporary values are allocated in a sub-pool. */ +static svn_error_t * +get_mergeinfos_for_paths(svn_fs_root_t *root, + svn_mergeinfo_catalog_t *mergeinfo_catalog, + const apr_array_header_t *paths, + svn_mergeinfo_inheritance_t inherit, + svn_boolean_t include_descendants, + svn_boolean_t adjust_inherited_mergeinfo, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + svn_mergeinfo_catalog_t result_catalog = svn_hash__make(result_pool); + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + int i; + + for (i = 0; i < paths->nelts; i++) + { + svn_error_t *err; + svn_mergeinfo_t path_mergeinfo; + const char *path = APR_ARRAY_IDX(paths, i, const char *); + + svn_pool_clear(iterpool); + + err = get_mergeinfo_for_path(&path_mergeinfo, root, path, + inherit, adjust_inherited_mergeinfo, + result_pool, iterpool); + if (err) + { + if (err->apr_err == SVN_ERR_MERGEINFO_PARSE_ERROR) + { + svn_error_clear(err); + err = NULL; + path_mergeinfo = NULL; + } + else + { + return svn_error_trace(err); + } + } + + if (path_mergeinfo) + svn_hash_sets(result_catalog, path, path_mergeinfo); + if (include_descendants) + SVN_ERR(add_descendant_mergeinfo(result_catalog, root, path, + result_pool, scratch_pool)); + } + svn_pool_destroy(iterpool); + + *mergeinfo_catalog = result_catalog; + return SVN_NO_ERROR; +} + + +/* Implements svn_fs_get_mergeinfo. */ +static svn_error_t * +x_get_mergeinfo(svn_mergeinfo_catalog_t *catalog, + svn_fs_root_t *root, + const apr_array_header_t *paths, + svn_mergeinfo_inheritance_t inherit, + svn_boolean_t include_descendants, + svn_boolean_t adjust_inherited_mergeinfo, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + /* We require a revision root. */ + if (root->is_txn_root) + return svn_error_create(SVN_ERR_FS_NOT_REVISION_ROOT, NULL, NULL); + + /* Retrieve a path -> mergeinfo hash mapping. */ + return get_mergeinfos_for_paths(root, catalog, paths, + inherit, + include_descendants, + adjust_inherited_mergeinfo, + result_pool, scratch_pool); +} + + +/* The vtable associated with root objects. */ +static root_vtable_t root_vtable = { + x_paths_changed, + svn_fs_x__check_path, + x_node_history, + svn_fs_x__node_id, + svn_fs_x__node_created_rev, + x_node_origin_rev, + x_node_created_path, + x_delete_node, + x_copied_from, + x_closest_copy, + x_node_prop, + x_node_proplist, + x_change_node_prop, + x_props_changed, + x_dir_entries, +/* x_dir_optimal_order,*/ + x_make_dir, + x_copy, + x_revision_link, + x_file_length, + x_file_checksum, + x_file_contents, + x_try_process_file_contents, + x_make_file, + x_apply_textdelta, + x_apply_text, + x_contents_changed, + x_get_file_delta_stream, + x_merge, + x_get_mergeinfo, +}; + +/* Construct a new root object in FS, allocated from POOL. */ +static svn_fs_root_t * +make_root(svn_fs_t *fs, + apr_pool_t *pool) +{ + svn_fs_root_t *root = apr_pcalloc(pool, sizeof(*root)); + + root->fs = fs; + root->pool = pool; + root->vtable = &root_vtable; + + return root; +} + + +/* Construct a root object referring to the root of REVISION in FS, + whose root directory is ROOT_DIR. Create the new root in POOL. */ +static svn_fs_root_t * +make_revision_root(svn_fs_t *fs, + svn_revnum_t rev, + dag_node_t *root_dir, + apr_pool_t *pool) +{ + svn_fs_root_t *root = make_root(fs, pool); + + root->is_txn_root = FALSE; + root->rev = rev; + root->fsap_data = root_dir; + + return root; +} + + +/* Construct a root object referring to the root of the transaction + named TXN and based on revision BASE_REV in FS, with FLAGS to + describe transaction's behavior. Create the new root in POOL. */ +static svn_error_t * +make_txn_root(svn_fs_root_t **root_p, + svn_fs_t *fs, + const svn_fs_x__id_part_t *txn, + svn_revnum_t base_rev, + apr_uint32_t flags, + apr_pool_t *pool) +{ + svn_fs_root_t *root = make_root(fs, pool); + fs_txn_root_data_t *frd = apr_pcalloc(root->pool, sizeof(*frd)); + frd->txn_id = *txn; + + root->is_txn_root = TRUE; + root->txn = svn_fs_x__id_txn_unparse(txn, root->pool); + root->txn_flags = flags; + root->rev = base_rev; + + /* Because this cache actually tries to invalidate elements, keep + the number of elements per page down. + + Note that since dag_node_cache_invalidate uses svn_cache__iter, + this *cannot* be a memcache-based cache. */ + SVN_ERR(svn_cache__create_inprocess(&(frd->txn_node_cache), + svn_fs_x__dag_serialize, + svn_fs_x__dag_deserialize, + APR_HASH_KEY_STRING, + 32, 20, FALSE, + apr_pstrcat(pool, txn, ":TXN", + (char *)NULL), + root->pool)); + + /* Initialize transaction-local caches in FS. + + Note that we cannot put those caches in frd because that content + fs root object is not available where we would need it. */ + SVN_ERR(svn_fs_x__initialize_txn_caches(fs, root->txn, pool)); + + root->fsap_data = frd; + + *root_p = root; + return SVN_NO_ERROR; +} + + + +/* Verify. */ +static APR_INLINE const char * +stringify_node(dag_node_t *node, + apr_pool_t *pool) +{ + /* ### TODO: print some PATH@REV to it, too. */ + return svn_fs_x__id_unparse(svn_fs_x__dag_get_id(node), pool)->data; +} + +/* Check metadata sanity on NODE, and on its children. Manually verify + information for DAG nodes in revision REV, and trust the metadata + accuracy for nodes belonging to older revisions. */ +static svn_error_t * +verify_node(dag_node_t *node, + svn_revnum_t rev, + apr_pool_t *pool) +{ + svn_boolean_t has_mergeinfo; + apr_int64_t mergeinfo_count; + const svn_fs_id_t *pred_id; + svn_fs_t *fs = svn_fs_x__dag_get_fs(node); + int pred_count; + svn_node_kind_t kind; + apr_pool_t *iterpool = svn_pool_create(pool); + + /* Fetch some data. */ + SVN_ERR(svn_fs_x__dag_has_mergeinfo(&has_mergeinfo, node)); + SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_count, node)); + SVN_ERR(svn_fs_x__dag_get_predecessor_id(&pred_id, node)); + SVN_ERR(svn_fs_x__dag_get_predecessor_count(&pred_count, node)); + kind = svn_fs_x__dag_node_kind(node); + + /* Sanity check. */ + if (mergeinfo_count < 0) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Negative mergeinfo-count %" APR_INT64_T_FMT + " on node '%s'", + mergeinfo_count, stringify_node(node, iterpool)); + + /* Issue #4129. (This check will explicitly catch non-root instances too.) */ + if (pred_id) + { + dag_node_t *pred; + int pred_pred_count; + SVN_ERR(svn_fs_x__dag_get_node(&pred, fs, pred_id, iterpool)); + SVN_ERR(svn_fs_x__dag_get_predecessor_count(&pred_pred_count, pred)); + if (pred_pred_count+1 != pred_count) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Predecessor count mismatch: " + "%s has %d, but %s has %d", + stringify_node(node, iterpool), pred_count, + stringify_node(pred, iterpool), + pred_pred_count); + } + + /* Kind-dependent verifications. */ + if (kind == svn_node_none) + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Node '%s' has kind 'none'", + stringify_node(node, iterpool)); + } + if (kind == svn_node_file) + { + if (has_mergeinfo != mergeinfo_count) /* comparing int to bool */ + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "File node '%s' has inconsistent mergeinfo: " + "has_mergeinfo=%d, " + "mergeinfo_count=%" APR_INT64_T_FMT, + stringify_node(node, iterpool), + has_mergeinfo, mergeinfo_count); + } + if (kind == svn_node_dir) + { + apr_hash_t *entries; + apr_hash_index_t *hi; + apr_int64_t children_mergeinfo = 0; + + SVN_ERR(svn_fs_x__dag_dir_entries(&entries, node, pool)); + + /* Compute CHILDREN_MERGEINFO. */ + for (hi = apr_hash_first(pool, entries); + hi; + hi = apr_hash_next(hi)) + { + svn_fs_dirent_t *dirent = svn__apr_hash_index_val(hi); + dag_node_t *child; + svn_revnum_t child_rev; + apr_int64_t child_mergeinfo; + + svn_pool_clear(iterpool); + + /* Compute CHILD_REV. */ + SVN_ERR(svn_fs_x__dag_get_node(&child, fs, dirent->id, iterpool)); + SVN_ERR(svn_fs_x__dag_get_revision(&child_rev, child, iterpool)); + + if (child_rev == rev) + SVN_ERR(verify_node(child, rev, iterpool)); + + SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&child_mergeinfo, child)); + children_mergeinfo += child_mergeinfo; + } + + /* Side-effect of issue #4129. */ + if (children_mergeinfo+has_mergeinfo != mergeinfo_count) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Mergeinfo-count discrepancy on '%s': " + "expected %" APR_INT64_T_FMT "+%d, " + "counted %" APR_INT64_T_FMT, + stringify_node(node, iterpool), + mergeinfo_count, has_mergeinfo, + children_mergeinfo); + } + + svn_pool_destroy(iterpool); + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__verify_root(svn_fs_root_t *root, + apr_pool_t *pool) +{ + svn_fs_t *fs = root->fs; + dag_node_t *root_dir; + + /* Issue #4129: bogus pred-counts and minfo-cnt's on the root node-rev + (and elsewhere). This code makes more thorough checks than the + commit-time checks in validate_root_noderev(). */ + + /* Callers should disable caches by setting SVN_FS_CONFIG_FSX_CACHE_NS; + see r1462436. + + When this code is called in the library, we want to ensure we + use the on-disk data --- rather than some data that was read + in the possibly-distance past and cached since. */ + + if (root->is_txn_root) + { + fs_txn_root_data_t *frd = root->fsap_data; + SVN_ERR(svn_fs_x__dag_txn_root(&root_dir, fs, &frd->txn_id, pool)); + } + else + { + root_dir = root->fsap_data; + } + + /* Recursively verify ROOT_DIR. */ + SVN_ERR(verify_node(root_dir, root->rev, pool)); + + /* Verify explicitly the predecessor of the root. */ + { + const svn_fs_id_t *pred_id; + + /* Only r0 should have no predecessor. */ + SVN_ERR(svn_fs_x__dag_get_predecessor_id(&pred_id, root_dir)); + if (! root->is_txn_root && !!pred_id != !!root->rev) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "r%ld's root node's predecessor is " + "unexpectedly '%s'", + root->rev, + (pred_id + ? svn_fs_x__id_unparse(pred_id, pool)->data + : "(null)")); + if (root->is_txn_root && !pred_id) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Transaction '%s''s root node's predecessor is " + "unexpectedly NULL", + root->txn); + + /* Check the predecessor's revision. */ + if (pred_id) + { + svn_revnum_t pred_rev = svn_fs_x__id_rev(pred_id); + if (! root->is_txn_root && pred_rev+1 != root->rev) + /* Issue #4129. */ + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "r%ld's root node's predecessor is r%ld" + " but should be r%ld", + root->rev, pred_rev, root->rev - 1); + if (root->is_txn_root && pred_rev != root->rev) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Transaction '%s''s root node's predecessor" + " is r%ld" + " but should be r%ld", + root->txn, pred_rev, root->rev); + } + } + + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_fs_x/tree.h b/subversion/libsvn_fs_x/tree.h new file mode 100644 index 0000000..552f267 --- /dev/null +++ b/subversion/libsvn_fs_x/tree.h
@@ -0,0 +1,114 @@ +/* tree.h : internal interface to tree node functions + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS_TREE_H +#define SVN_LIBSVN_FS_TREE_H + +#include "fs.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + + + +/* In POOL, create an instance of a DAG node 1st level cache. + The POOL will be cleared at regular intervals. */ +fs_x_dag_cache_t* +svn_fs_x__create_dag_cache(apr_pool_t *pool); + +/* Set *ROOT_P to the root directory of revision REV in filesystem FS. + Allocate the structure in POOL. */ +svn_error_t *svn_fs_x__revision_root(svn_fs_root_t **root_p, svn_fs_t *fs, + svn_revnum_t rev, apr_pool_t *pool); + +/* Does nothing, but included for Subversion 1.0.x compatibility. */ +svn_error_t *svn_fs_x__deltify(svn_fs_t *fs, svn_revnum_t rev, + apr_pool_t *pool); + +/* Commit the transaction TXN as a new revision. Return the new + revision in *NEW_REV. If the transaction conflicts with other + changes return SVN_ERR_FS_CONFLICT and set *CONFLICT_P to a string + that details the cause of the conflict. + Update commit time to ensure that svn:date revprops remain ordered if + SET_TIMESTAMP is non-zero. Perform temporary allocations in POOL. */ +svn_error_t *svn_fs_x__commit_txn(const char **conflict_p, + svn_revnum_t *new_rev, svn_fs_txn_t *txn, + svn_boolean_t set_timestamp, + apr_pool_t *pool); + +/* Set ROOT_P to the root directory of transaction TXN. Allocate the + structure in POOL. */ +svn_error_t *svn_fs_x__txn_root(svn_fs_root_t **root_p, svn_fs_txn_t *txn, + apr_pool_t *pool); + + +/* Set KIND_P to the node kind of the node at PATH in ROOT. + Allocate the structure in POOL. */ +svn_error_t * +svn_fs_x__check_path(svn_node_kind_t *kind_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool); + +/* Implement root_vtable_t.node_id(). */ +svn_error_t * +svn_fs_x__node_id(const svn_fs_id_t **id_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool); + +/* Set *REVISION to the revision in which PATH under ROOT was created. + Use POOL for any temporary allocations. If PATH is in an + uncommitted transaction, *REVISION will be set to + SVN_INVALID_REVNUM. */ +svn_error_t * +svn_fs_x__node_created_rev(svn_revnum_t *revision, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool); + +/* Verify metadata for ROOT. + ### Currently only implemented for revision roots. */ +svn_error_t * +svn_fs_x__verify_root(svn_fs_root_t *root, + apr_pool_t *pool); + +svn_error_t * +svn_fs_x__info_format(int *fs_format, + svn_version_t **supports_version, + svn_fs_t *fs, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + + +svn_error_t * +svn_fs_x__info_config_files(apr_array_header_t **files, + svn_fs_t *fs, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* SVN_LIBSVN_FS_TREE_H */
diff --git a/subversion/libsvn_fs_x/util.c b/subversion/libsvn_fs_x/util.c new file mode 100644 index 0000000..10a28a2 --- /dev/null +++ b/subversion/libsvn_fs_x/util.c
@@ -0,0 +1,684 @@ +/* util.c --- utility functions for FSX repo access + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include <assert.h> + +#include "svn_ctype.h" +#include "svn_dirent_uri.h" +#include "private/svn_string_private.h" + +#include "fs_x.h" +#include "id.h" +#include "util.h" + +#include "../libsvn_fs/fs-loader.h" + +#include "svn_private_config.h" + +/* Following are defines that specify the textual elements of the + native filesystem directories and revision files. */ + +/* Notes: + +To avoid opening and closing the rev-files all the time, it would +probably be advantageous to keep each rev-file open for the +lifetime of the transaction object. I'll leave that as a later +optimization for now. + +I didn't keep track of pool lifetimes at all in this code. There +are likely some errors because of that. + +*/ + +/* Pathname helper functions */ + +/* Return TRUE is REV is packed in FS, FALSE otherwise. */ +svn_boolean_t +svn_fs_x__is_packed_rev(svn_fs_t *fs, svn_revnum_t rev) +{ + fs_x_data_t *ffd = fs->fsap_data; + + return (rev < ffd->min_unpacked_rev); +} + +/* Return TRUE is REV is packed in FS, FALSE otherwise. */ +svn_boolean_t +svn_fs_x__is_packed_revprop(svn_fs_t *fs, svn_revnum_t rev) +{ + fs_x_data_t *ffd = fs->fsap_data; + + /* rev 0 will not be packed */ + return (rev < ffd->min_unpacked_rev) && (rev != 0); +} + +svn_revnum_t +svn_fs_x__packed_base_rev(svn_fs_t *fs, svn_revnum_t rev) +{ + fs_x_data_t *ffd = fs->fsap_data; + + return rev < ffd->min_unpacked_rev + ? rev - (rev % ffd->max_files_per_dir) + : rev; +} + +svn_revnum_t +svn_fs_x__pack_size(svn_fs_t *fs, svn_revnum_t rev) +{ + fs_x_data_t *ffd = fs->fsap_data; + + return rev < ffd->min_unpacked_rev ? ffd->max_files_per_dir : 1; +} + +const char * +svn_fs_x__path_format(svn_fs_t *fs, apr_pool_t *pool) +{ + return svn_dirent_join(fs->path, PATH_FORMAT, pool); +} + +const char * +svn_fs_x__path_uuid(svn_fs_t *fs, apr_pool_t *pool) +{ + return svn_dirent_join(fs->path, PATH_UUID, pool); +} + +const char * +svn_fs_x__path_current(svn_fs_t *fs, apr_pool_t *pool) +{ + return svn_dirent_join(fs->path, PATH_CURRENT, pool); +} + +const char * +svn_fs_x__path_txn_current(svn_fs_t *fs, apr_pool_t *pool) +{ + return svn_dirent_join(fs->path, PATH_TXN_CURRENT, pool); +} + +const char * +svn_fs_x__path_txn_current_lock(svn_fs_t *fs, apr_pool_t *pool) +{ + return svn_dirent_join(fs->path, PATH_TXN_CURRENT_LOCK, pool); +} + +const char * +svn_fs_x__path_lock(svn_fs_t *fs, apr_pool_t *pool) +{ + return svn_dirent_join(fs->path, PATH_LOCK_FILE, pool); +} + +const char * +svn_fs_x__path_revprop_generation(svn_fs_t *fs, apr_pool_t *pool) +{ + return svn_dirent_join(fs->path, PATH_REVPROP_GENERATION, pool); +} + +const char * +svn_fs_x__path_rev_packed(svn_fs_t *fs, svn_revnum_t rev, const char *kind, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + assert(svn_fs_x__is_packed_rev(fs, rev)); + + return svn_dirent_join_many(pool, fs->path, PATH_REVS_DIR, + apr_psprintf(pool, + "%ld" PATH_EXT_PACKED_SHARD, + rev / ffd->max_files_per_dir), + kind, NULL); +} + +const char * +svn_fs_x__path_rev_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + return svn_dirent_join_many(pool, fs->path, PATH_REVS_DIR, + apr_psprintf(pool, "%ld", + rev / ffd->max_files_per_dir), + NULL); +} + +const char * +svn_fs_x__path_rev(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) +{ + assert(! svn_fs_x__is_packed_rev(fs, rev)); + + return svn_dirent_join(svn_fs_x__path_rev_shard(fs, rev, pool), + apr_psprintf(pool, "%ld", rev), + pool); +} + +const char * +svn_fs_x__path_l2p_index(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + return apr_psprintf(pool, "%s" PATH_EXT_L2P_INDEX, + svn_fs_x__path_rev_absolute(fs, rev, pool)); +} + +const char * +svn_fs_x__path_p2l_index(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + return apr_psprintf(pool, "%s" PATH_EXT_P2L_INDEX, + svn_fs_x__path_rev_absolute(fs, rev, pool)); +} + +const char * +svn_fs_x__path_rev_absolute(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + return ! svn_fs_x__is_packed_rev(fs, rev) + ? svn_fs_x__path_rev(fs, rev, pool) + : svn_fs_x__path_rev_packed(fs, rev, PATH_PACKED, pool); +} + +const char * +svn_fs_x__path_revprops_shard(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + + return svn_dirent_join_many(pool, fs->path, PATH_REVPROPS_DIR, + apr_psprintf(pool, "%ld", + rev / ffd->max_files_per_dir), + NULL); +} + +const char * +svn_fs_x__path_revprops_pack_shard(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + + return svn_dirent_join_many(pool, fs->path, PATH_REVPROPS_DIR, + apr_psprintf(pool, "%ld" PATH_EXT_PACKED_SHARD, + rev / ffd->max_files_per_dir), + NULL); +} + +const char * +svn_fs_x__path_revprops(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) +{ + return svn_dirent_join(svn_fs_x__path_revprops_shard(fs, rev, pool), + apr_psprintf(pool, "%ld", rev), + pool); +} + +/* Return TO_ADD appended to the C string representation of TXN_ID. + * Allocate the result in POOL. + */ +static const char * +combine_txn_id_string(const svn_fs_x__id_part_t *txn_id, + const char *to_add, + apr_pool_t *pool) +{ + return apr_pstrcat(pool, svn_fs_x__id_txn_unparse(txn_id, pool), + to_add, (char *)NULL); +} + +const char * +svn_fs_x__path_txn_dir(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + SVN_ERR_ASSERT_NO_RETURN(txn_id != NULL); + return svn_dirent_join_many(pool, fs->path, PATH_TXNS_DIR, + combine_txn_id_string(txn_id, PATH_EXT_TXN, + pool), + NULL); +} + +/* Return the name of the sha1->rep mapping file in transaction TXN_ID + * within FS for the given SHA1 checksum. Use POOL for allocations. + */ +const char * +svn_fs_x__path_txn_sha1(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + const unsigned char *sha1, + apr_pool_t *pool) +{ + svn_checksum_t checksum; + checksum.digest = sha1; + checksum.kind = svn_checksum_sha1; + + return svn_dirent_join(svn_fs_x__path_txn_dir(fs, txn_id, pool), + svn_checksum_to_cstring(&checksum, pool), + pool); +} + +const char * +svn_fs_x__path_txn_changes(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + return svn_dirent_join(svn_fs_x__path_txn_dir(fs, txn_id, pool), + PATH_CHANGES, pool); +} + +const char * +svn_fs_x__path_txn_props(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + return svn_dirent_join(svn_fs_x__path_txn_dir(fs, txn_id, pool), + PATH_TXN_PROPS, pool); +} + +const char* +svn_fs_x__path_l2p_proto_index(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + return svn_dirent_join(svn_fs_x__path_txn_dir(fs, txn_id, pool), + PATH_INDEX PATH_EXT_L2P_INDEX, pool); +} + +const char* +svn_fs_x__path_p2l_proto_index(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + return svn_dirent_join(svn_fs_x__path_txn_dir(fs, txn_id, pool), + PATH_INDEX PATH_EXT_P2L_INDEX, pool); +} + +const char * +svn_fs_x__path_txn_next_ids(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + return svn_dirent_join(svn_fs_x__path_txn_dir(fs, txn_id, pool), + PATH_NEXT_IDS, pool); +} + +const char * +svn_fs_x__path_min_unpacked_rev(svn_fs_t *fs, apr_pool_t *pool) +{ + return svn_dirent_join(fs->path, PATH_MIN_UNPACKED_REV, pool); +} + +const char * +svn_fs_x__path_txn_item_index(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + return svn_dirent_join(svn_fs_x__path_txn_dir(fs, txn_id, pool), + PATH_TXN_ITEM_INDEX, pool); +} + +const char * +svn_fs_x__path_txn_proto_rev(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + return svn_dirent_join_many(pool, fs->path, PATH_TXN_PROTOS_DIR, + combine_txn_id_string(txn_id, PATH_EXT_REV, + pool), + NULL); +} + +const char * +svn_fs_x__path_txn_proto_rev_lock(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool) +{ + return svn_dirent_join_many(pool, fs->path, PATH_TXN_PROTOS_DIR, + combine_txn_id_string(txn_id, + PATH_EXT_REV_LOCK, + pool), + NULL); +} + +const char * +svn_fs_x__path_txn_node_rev(svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool) +{ + char *filename = (char *)svn_fs_x__id_unparse(id, pool)->data; + *strrchr(filename, '.') = '\0'; + + return svn_dirent_join(svn_fs_x__path_txn_dir(fs, svn_fs_x__id_txn_id(id), + pool), + apr_psprintf(pool, PATH_PREFIX_NODE "%s", + filename), + pool); +} + +const char * +svn_fs_x__path_txn_node_props(svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool) +{ + return apr_pstrcat(pool, svn_fs_x__path_txn_node_rev(fs, id, pool), + PATH_EXT_PROPS, (char *)NULL); +} + +const char * +svn_fs_x__path_txn_node_children(svn_fs_t *fs, const svn_fs_id_t *id, apr_pool_t *pool) +{ + return apr_pstrcat(pool, svn_fs_x__path_txn_node_rev(fs, id, pool), + PATH_EXT_CHILDREN, (char *)NULL); +} + +const char * +svn_fs_x__path_node_origin(svn_fs_t *fs, + const svn_fs_x__id_part_t *node_id, + apr_pool_t *pool) +{ + char buffer[SVN_INT64_BUFFER_SIZE]; + apr_size_t len = svn__ui64tobase36(buffer, node_id->number); + + if (len > 1) + buffer[len - 1] = '\0'; + + return svn_dirent_join_many(pool, fs->path, PATH_NODE_ORIGINS_DIR, + buffer, NULL); +} + + +/* Check that BUF, a nul-terminated buffer of text from file PATH, + contains only digits at OFFSET and beyond, raising an error if not. + TITLE contains a user-visible description of the file, usually the + short file name. + + Uses POOL for temporary allocation. */ +svn_error_t * +svn_fs_x__check_file_buffer_numeric(const char *buf, apr_off_t offset, + const char *path, const char *title, + apr_pool_t *pool) +{ + const char *p; + + for (p = buf + offset; *p; p++) + if (!svn_ctype_isdigit(*p)) + return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL, + _("%s file '%s' contains unexpected non-digit '%c' within '%s'"), + title, svn_dirent_local_style(path, pool), *p, buf); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__read_min_unpacked_rev(svn_revnum_t *min_unpacked_rev, + svn_fs_t *fs, + apr_pool_t *pool) +{ + char buf[80]; + apr_file_t *file; + apr_size_t len; + + SVN_ERR(svn_io_file_open(&file, + svn_fs_x__path_min_unpacked_rev(fs, pool), + APR_READ | APR_BUFFERED, + APR_OS_DEFAULT, + pool)); + len = sizeof(buf); + SVN_ERR(svn_io_read_length_line(file, buf, &len, pool)); + SVN_ERR(svn_io_file_close(file, pool)); + + *min_unpacked_rev = SVN_STR_TO_REV(buf); + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__update_min_unpacked_rev(svn_fs_t *fs, apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + return svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, fs, pool); +} + +/* Write a file FILENAME in directory FS_PATH, containing a single line + * with the number REVNUM in ASCII decimal. Move the file into place + * atomically, overwriting any existing file. + * + * Similar to write_current(). */ +svn_error_t * +svn_fs_x__write_revnum_file(svn_fs_t *fs, + svn_revnum_t revnum, + apr_pool_t *scratch_pool) +{ + const char *final_path; + char buf[SVN_INT64_BUFFER_SIZE]; + apr_size_t len = svn__i64toa(buf, revnum); + buf[len] = '\n'; + + final_path = svn_fs_x__path_min_unpacked_rev(fs, scratch_pool); + + SVN_ERR(svn_io_write_atomic(final_path, buf, len + 1, + final_path /* copy_perms */, scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Atomically update the 'current' file to hold the specifed REV. + Perform temporary allocations in POOL. */ +svn_error_t * +svn_fs_x__write_current(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) +{ + char *buf; + const char *tmp_name, *name; + + /* Now we can just write out this line. */ + buf = apr_psprintf(pool, "%ld\n", rev); + + name = svn_fs_x__path_current(fs, pool); + SVN_ERR(svn_io_write_unique(&tmp_name, + svn_dirent_dirname(name, pool), + buf, strlen(buf), + svn_io_file_del_none, pool)); + + return svn_fs_x__move_into_place(tmp_name, name, name, pool); +} + + + +/* Read the file at PATH and return its content in *CONTENT. *CONTENT will + * not be modified unless the whole file was read successfully. + * + * ESTALE, EIO and ENOENT will not cause this function to return an error + * unless LAST_ATTEMPT has been set. If MISSING is not NULL, indicate + * missing files (ENOENT) there. + * + * Use POOL for allocations. + */ +svn_error_t * +svn_fs_x__try_stringbuf_from_file(svn_stringbuf_t **content, + svn_boolean_t *missing, + const char *path, + svn_boolean_t last_attempt, + apr_pool_t *pool) +{ + svn_error_t *err = svn_stringbuf_from_file2(content, path, pool); + if (missing) + *missing = FALSE; + + if (err) + { + *content = NULL; + + if (APR_STATUS_IS_ENOENT(err->apr_err)) + { + if (!last_attempt) + { + svn_error_clear(err); + if (missing) + *missing = TRUE; + return SVN_NO_ERROR; + } + } +#ifdef ESTALE + else if (APR_TO_OS_ERROR(err->apr_err) == ESTALE + || APR_TO_OS_ERROR(err->apr_err) == EIO) + { + if (!last_attempt) + { + svn_error_clear(err); + return SVN_NO_ERROR; + } + } +#endif + } + + return svn_error_trace(err); +} + +/* Fetch the current offset of FILE into *OFFSET_P. */ +svn_error_t * +svn_fs_x__get_file_offset(apr_off_t *offset_p, + apr_file_t *file, + apr_pool_t *pool) +{ + apr_off_t offset; + + /* Note that, for buffered files, one (possibly surprising) side-effect + of this call is to flush any unwritten data to disk. */ + offset = 0; + SVN_ERR(svn_io_file_seek(file, APR_CUR, &offset, pool)); + *offset_p = offset; + + return SVN_NO_ERROR; +} + +/* Read the 'current' file FNAME and store the contents in *BUF. + Allocations are performed in POOL. */ +svn_error_t * +svn_fs_x__read_content(svn_stringbuf_t **content, + const char *fname, + apr_pool_t *pool) +{ + int i; + *content = NULL; + + for (i = 0; !*content && (i < RECOVERABLE_RETRY_COUNT); ++i) + SVN_ERR(svn_fs_x__try_stringbuf_from_file(content, NULL, + fname, i + 1 < RECOVERABLE_RETRY_COUNT, + pool)); + + if (!*content) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Can't read '%s'"), + svn_dirent_local_style(fname, pool)); + + return SVN_NO_ERROR; +} + +/* Reads a line from STREAM and converts it to a 64 bit integer to be + * returned in *RESULT. If we encounter eof, set *HIT_EOF and leave + * *RESULT unchanged. If HIT_EOF is NULL, EOF causes an "corrupt FS" + * error return. + * SCRATCH_POOL is used for temporary allocations. + */ +svn_error_t * +svn_fs_x__read_number_from_stream(apr_int64_t *result, + svn_boolean_t *hit_eof, + svn_stream_t *stream, + apr_pool_t *scratch_pool) +{ + svn_stringbuf_t *sb; + svn_boolean_t eof; + svn_error_t *err; + + SVN_ERR(svn_stream_readline(stream, &sb, "\n", &eof, scratch_pool)); + if (hit_eof) + *hit_eof = eof; + else + if (eof) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Unexpected EOF")); + + if (!eof) + { + err = svn_cstring_atoi64(result, sb->data); + if (err) + return svn_error_createf(SVN_ERR_FS_CORRUPT, err, + _("Number '%s' invalid or too large"), + sb->data); + } + + return SVN_NO_ERROR; +} + + +/* Move a file into place from OLD_FILENAME in the transactions + directory to its final location NEW_FILENAME in the repository. On + Unix, match the permissions of the new file to the permissions of + PERMS_REFERENCE. Temporary allocations are from POOL. + + This function almost duplicates svn_io_file_move(), but it tries to + guarantee a flush. */ +svn_error_t * +svn_fs_x__move_into_place(const char *old_filename, + const char *new_filename, + const char *perms_reference, + apr_pool_t *pool) +{ + svn_error_t *err; + + SVN_ERR(svn_io_copy_perms(perms_reference, old_filename, pool)); + + /* Move the file into place. */ + err = svn_io_file_rename(old_filename, new_filename, pool); + if (err && APR_STATUS_IS_EXDEV(err->apr_err)) + { + apr_file_t *file; + + /* Can't rename across devices; fall back to copying. */ + svn_error_clear(err); + err = SVN_NO_ERROR; + SVN_ERR(svn_io_copy_file(old_filename, new_filename, TRUE, pool)); + + /* Flush the target of the copy to disk. */ + SVN_ERR(svn_io_file_open(&file, new_filename, APR_READ, + APR_OS_DEFAULT, pool)); + /* ### BH: Does this really guarantee a flush of the data written + ### via a completely different handle on all operating systems? + ### + ### Maybe we should perform the copy ourselves instead of making + ### apr do that and flush the real handle? */ + SVN_ERR(svn_io_file_flush_to_disk(file, pool)); + SVN_ERR(svn_io_file_close(file, pool)); + } + if (err) + return svn_error_trace(err); + +#ifdef __linux__ + { + /* Linux has the unusual feature that fsync() on a file is not + enough to ensure that a file's directory entries have been + flushed to disk; you have to fsync the directory as well. + On other operating systems, we'd only be asking for trouble + by trying to open and fsync a directory. */ + const char *dirname; + apr_file_t *file; + + dirname = svn_dirent_dirname(new_filename, pool); + SVN_ERR(svn_io_file_open(&file, dirname, APR_READ, APR_OS_DEFAULT, + pool)); + SVN_ERR(svn_io_file_flush_to_disk(file, pool)); + SVN_ERR(svn_io_file_close(file, pool)); + } +#endif + + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_fs_x/util.h b/subversion/libsvn_fs_x/util.h new file mode 100644 index 0000000..f2b1c6d --- /dev/null +++ b/subversion/libsvn_fs_x/util.h
@@ -0,0 +1,327 @@ +/* util.h --- utility functions for FSX repo access + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS__UTIL_H +#define SVN_LIBSVN_FS__UTIL_H + +#include "svn_fs.h" +#include "id.h" + +/* Functions for dealing with recoverable errors on mutable files + * + * Revprops, current, and txn-current files are mutable; that is, they + * change as part of normal fsx operation, in constrat to revs files, or + * the format file, which are written once at create (or upgrade) time. + * When more than one host writes to the same repository, we will + * sometimes see these recoverable errors when accesssing these files. + * + * These errors all relate to NFS, and thus we only use this retry code if + * ESTALE is defined. + * + ** ESTALE + * + * In NFS v3 and under, the server doesn't track opened files. If you + * unlink(2) or rename(2) a file held open by another process *on the + * same host*, that host's kernel typically renames the file to + * .nfsXXXX and automatically deletes that when it's no longer open, + * but this behavior is not required. + * + * For obvious reasons, this does not work *across hosts*. No one + * knows about the opened file; not the server, and not the deleting + * client. So the file vanishes, and the reader gets stale NFS file + * handle. + * + ** EIO, ENOENT + * + * Some client implementations (at least the 2.6.18.5 kernel that ships + * with Ubuntu Dapper) sometimes give spurious ENOENT (only on open) or + * even EIO errors when trying to read these files that have been renamed + * over on some other host. + * + ** Solution + * + * Try open and read of such files in try_stringbuf_from_file(). Call + * this function within a loop of RECOVERABLE_RETRY_COUNT iterations + * (though, realistically, the second try will succeed). + */ + +#define RECOVERABLE_RETRY_COUNT 10 + +/* Pathname helper functions */ + +/* Return TRUE is REV is packed in FS, FALSE otherwise. */ +svn_boolean_t +svn_fs_x__is_packed_rev(svn_fs_t *fs, + svn_revnum_t rev); + +/* Return TRUE is REV is packed in FS, FALSE otherwise. */ +svn_boolean_t +svn_fs_x__is_packed_revprop(svn_fs_t *fs, + svn_revnum_t rev); + +/* Return the revision number of the pack / rev file in FS containing REV. */ +svn_revnum_t +svn_fs_x__packed_base_rev(svn_fs_t *fs, svn_revnum_t rev); + +/* Return the number of revisions in the pack / rev file in FS that contains + * revision REV. */ +svn_revnum_t +svn_fs_x__pack_size(svn_fs_t *fs, svn_revnum_t rev); + +const char * +svn_fs_x__path_format(svn_fs_t *fs, + apr_pool_t *pool); + +const char * +svn_fs_x__path_uuid(svn_fs_t *fs, + apr_pool_t *pool); + +const char * +svn_fs_x__path_txn_current(svn_fs_t *fs, + apr_pool_t *pool); + +const char * +svn_fs_x__path_txn_current_lock(svn_fs_t *fs, + apr_pool_t *pool); + +const char * +svn_fs_x__path_lock(svn_fs_t *fs, + apr_pool_t *pool); + +const char * +svn_fs_x__path_revprop_generation(svn_fs_t *fs, + apr_pool_t *pool); + +const char * +svn_fs_x__path_rev_packed(svn_fs_t *fs, + svn_revnum_t rev, + const char *kind, + apr_pool_t *pool); + +const char * +svn_fs_x__path_rev_shard(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +const char * +svn_fs_x__path_rev(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +const char * +svn_fs_x__path_l2p_index(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +const char * +svn_fs_x__path_p2l_index(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +const char * +svn_fs_x__path_revprops_shard(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +const char * +svn_fs_x__path_revprops_pack_shard(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +const char * +svn_fs_x__path_revprops(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +const char * +svn_fs_x__path_txn_dir(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +/* Return the name of the sha1->rep mapping file in transaction TXN_ID + * within FS for the given SHA1 checksum. Use POOL for allocations. + */ +const char * +svn_fs_x__path_txn_sha1(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + const unsigned char *sha1, + apr_pool_t *pool); + +const char * +svn_fs_x__path_txn_changes(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +const char* +svn_fs_x__path_l2p_proto_index(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +const char* +svn_fs_x__path_p2l_proto_index(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +const char * +svn_fs_x__path_txn_props(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +const char * +svn_fs_x__path_txn_next_ids(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +const char * +svn_fs_x__path_min_unpacked_rev(svn_fs_t *fs, + apr_pool_t *pool); + + +const char * +svn_fs_x__path_txn_item_index(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +const char * +svn_fs_x__path_txn_proto_rev(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +const char * +svn_fs_x__path_txn_proto_rev_lock(svn_fs_t *fs, + const svn_fs_x__id_part_t *txn_id, + apr_pool_t *pool); + +const char * +svn_fs_x__path_txn_node_rev(svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool); + +const char * +svn_fs_x__path_txn_node_props(svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool); + +const char * +svn_fs_x__path_txn_node_children(svn_fs_t *fs, + const svn_fs_id_t *id, + apr_pool_t *pool); + +const char * +svn_fs_x__path_node_origin(svn_fs_t *fs, + const svn_fs_x__id_part_t *node_id, + apr_pool_t *pool); + +/* Check that BUF, a nul-terminated buffer of text from file PATH, + contains only digits at OFFSET and beyond, raising an error if not. + TITLE contains a user-visible description of the file, usually the + short file name. + + Uses POOL for temporary allocation. */ +svn_error_t * +svn_fs_x__check_file_buffer_numeric(const char *buf, + apr_off_t offset, + const char *path, + const char *title, + apr_pool_t *pool); + +svn_error_t * +svn_fs_x__read_min_unpacked_rev(svn_revnum_t *min_unpacked_rev, + svn_fs_t *fs, + apr_pool_t *pool); + +svn_error_t * +svn_fs_x__update_min_unpacked_rev(svn_fs_t *fs, + apr_pool_t *pool); + +/* Write a file FILENAME in directory FS_PATH, containing a single line + * with the number REVNUM in ASCII decimal. Move the file into place + * atomically, overwriting any existing file. + * + * Similar to write_current(). */ +svn_error_t * +svn_fs_x__write_revnum_file(svn_fs_t *fs, + svn_revnum_t revnum, + apr_pool_t *scratch_pool); + +/* Atomically update the 'current' file to hold the specifed REV. + Perform temporary allocations in POOL. */ +svn_error_t * +svn_fs_x__write_current(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool); + +/* Read the file at PATH and return its content in *CONTENT. *CONTENT will + * not be modified unless the whole file was read successfully. + * + * ESTALE, EIO and ENOENT will not cause this function to return an error + * unless LAST_ATTEMPT has been set. If MISSING is not NULL, indicate + * missing files (ENOENT) there. + * + * Use POOL for allocations. + */ +svn_error_t * +svn_fs_x__try_stringbuf_from_file(svn_stringbuf_t **content, + svn_boolean_t *missing, + const char *path, + svn_boolean_t last_attempt, + apr_pool_t *pool); + +/* Fetch the current offset of FILE into *OFFSET_P. */ +svn_error_t * +svn_fs_x__get_file_offset(apr_off_t *offset_p, + apr_file_t *file, + apr_pool_t *pool); + +/* Read the file FNAME and store the contents in *BUF. + Allocations are performed in POOL. */ +svn_error_t * +svn_fs_x__read_content(svn_stringbuf_t **content, + const char *fname, + apr_pool_t *pool); + +/* Reads a line from STREAM and converts it to a 64 bit integer to be + * returned in *RESULT. If we encounter eof, set *HIT_EOF and leave + * *RESULT unchanged. If HIT_EOF is NULL, EOF causes an "corrupt FS" + * error return. + * SCRATCH_POOL is used for temporary allocations. + */ +svn_error_t * +svn_fs_x__read_number_from_stream(apr_int64_t *result, + svn_boolean_t *hit_eof, + svn_stream_t *stream, + apr_pool_t *scratch_pool); + +/* Move a file into place from OLD_FILENAME in the transactions + directory to its final location NEW_FILENAME in the repository. On + Unix, match the permissions of the new file to the permissions of + PERMS_REFERENCE. Temporary allocations are from POOL. + + This function almost duplicates svn_io_file_move(), but it tries to + guarantee a flush. */ +svn_error_t * +svn_fs_x__move_into_place(const char *old_filename, + const char *new_filename, + const char *perms_reference, + apr_pool_t *pool); + +#endif \ No newline at end of file
diff --git a/subversion/libsvn_fs_x/verify.c b/subversion/libsvn_fs_x/verify.c new file mode 100644 index 0000000..256f47d --- /dev/null +++ b/subversion/libsvn_fs_x/verify.c
@@ -0,0 +1,396 @@ +/* verify.c --- verification of FSX filesystems + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "verify.h" +#include "fs_x.h" + +#include "cached_data.h" +#include "rep-cache.h" +#include "util.h" +#include "index.h" + +#include "../libsvn_fs/fs-loader.h" + +#include "svn_private_config.h" + + +/** Verifying. **/ + +/* Baton type expected by verify_walker(). The purpose is to reuse open + * rev / pack file handles between calls. Its contents need to be cleaned + * periodically to limit resource usage. + */ +typedef struct verify_walker_baton_t +{ + /* number of calls to verify_walker() since the last clean */ + int iteration_count; + + /* number of files opened since the last clean */ + int file_count; + + /* progress notification callback to invoke periodically (may be NULL) */ + svn_fs_progress_notify_func_t notify_func; + + /* baton to use with NOTIFY_FUNC */ + void *notify_baton; + + /* remember the last revision for which we called notify_func */ + svn_revnum_t last_notified_revision; + + /* cached hint for successive calls to svn_fs_x__check_rep() */ + void *hint; + + /* pool to use for the file handles etc. */ + apr_pool_t *pool; +} verify_walker_baton_t; + +/* Used by svn_fs_x__verify(). + Implements svn_fs_x__walk_rep_reference().walker. */ +static svn_error_t * +verify_walker(representation_t *rep, + void *baton, + svn_fs_t *fs, + apr_pool_t *scratch_pool) +{ + verify_walker_baton_t *walker_baton = baton; + void *previous_hint; + + /* notify and free resources periodically */ + if ( walker_baton->iteration_count > 1000 + || walker_baton->file_count > 16) + { + if ( walker_baton->notify_func + && rep->revision != walker_baton->last_notified_revision) + { + walker_baton->notify_func(rep->revision, + walker_baton->notify_baton, + scratch_pool); + walker_baton->last_notified_revision = rep->revision; + } + + svn_pool_clear(walker_baton->pool); + + walker_baton->iteration_count = 0; + walker_baton->file_count = 0; + walker_baton->hint = NULL; + } + + /* access the repo data */ + previous_hint = walker_baton->hint; + SVN_ERR(svn_fs_x__check_rep(rep, fs, &walker_baton->hint, + walker_baton->pool)); + + /* update resource usage counters */ + walker_baton->iteration_count++; + if (previous_hint != walker_baton->hint) + walker_baton->file_count++; + + return SVN_NO_ERROR; +} + +/* Verify the rep cache DB's consistency with our rev / pack data. + * The function signature is similar to svn_fs_x__verify. + * The values of START and END have already been auto-selected and + * verified. + */ +static svn_error_t * +verify_rep_cache(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t end, + svn_fs_progress_notify_func_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + svn_boolean_t exists; + + /* rep-cache verification. */ + SVN_ERR(svn_fs_x__exists_rep_cache(&exists, fs, pool)); + if (exists) + { + /* provide a baton to allow the reuse of open file handles between + iterations (saves 2/3 of OS level file operations). */ + verify_walker_baton_t *baton = apr_pcalloc(pool, sizeof(*baton)); + baton->pool = svn_pool_create(pool); + baton->last_notified_revision = SVN_INVALID_REVNUM; + baton->notify_func = notify_func; + baton->notify_baton = notify_baton; + + /* tell the user that we are now ready to do *something* */ + if (notify_func) + notify_func(SVN_INVALID_REVNUM, notify_baton, baton->pool); + + /* Do not attempt to walk the rep-cache database if its file does + not exist, since doing so would create it --- which may confuse + the administrator. Don't take any lock. */ + SVN_ERR(svn_fs_x__walk_rep_reference(fs, start, end, + verify_walker, baton, + cancel_func, cancel_baton, + pool)); + + /* walker resource cleanup */ + svn_pool_destroy(baton->pool); + } + + return SVN_NO_ERROR; +} + +/* Verify that for all log-to-phys index entries for revisions START to + * START + COUNT-1 in FS there is a consistent entry in the phys-to-log + * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular + * intervals. Use POOL for allocations. + */ +static svn_error_t * +compare_l2p_to_p2l_index(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t count, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + svn_revnum_t i; + apr_pool_t *iterpool = svn_pool_create(pool); + apr_array_header_t *max_ids; + + /* determine the range of items to check for each revision */ + SVN_ERR(svn_fs_x__l2p_get_max_ids(&max_ids, fs, start, count, pool)); + + /* check all items in all revisions if the given range */ + for (i = 0; i < max_ids->nelts; ++i) + { + apr_uint64_t k; + apr_uint64_t max_id = APR_ARRAY_IDX(max_ids, i, apr_uint64_t); + svn_revnum_t revision = start + i; + + for (k = 0; k < max_id; ++k) + { + apr_off_t offset; + apr_uint32_t sub_item; + svn_fs_x__id_part_t *p2l_item; + + /* get L2P entry. Ignore unused entries. */ + SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, + revision, NULL, k, iterpool)); + if (offset == -1) + continue; + + /* find the corresponding P2L entry */ + SVN_ERR(svn_fs_x__p2l_item_lookup(&p2l_item, fs, start, + offset, sub_item, iterpool)); + + if (p2l_item == NULL) + return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT, + NULL, + _("p2l index entry not found for " + "PHYS o%s:s%ld returned by " + "l2p index for LOG r%ld:i%ld"), + apr_off_t_toa(pool, offset), + (long)sub_item, revision, (long)k); + + if (p2l_item->number != k || p2l_item->revision != revision) + return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT, + NULL, + _("p2l index info LOG r%ld:i%ld" + " does not match " + "l2p index for LOG r%ld:i%ld"), + p2l_item->revision, + (long)p2l_item->number, revision, + (long)k); + + svn_pool_clear(iterpool); + } + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Verify that for all phys-to-log index entries for revisions START to + * START + COUNT-1 in FS there is a consistent entry in the log-to-phys + * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular + * intervals. Use POOL for allocations. + * + * Please note that we can only check on pack / rev file granularity and + * must only be called for a single rev / pack file. + */ +static svn_error_t * +compare_p2l_to_l2p_index(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t count, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + apr_pool_t *iterpool = svn_pool_create(pool); + apr_off_t max_offset; + apr_off_t offset = 0; + + /* get the size of the rev / pack file as covered by the P2L index */ + SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, start, pool)); + + /* for all offsets in the file, get the P2L index entries and check + them against the L2P index */ + for (offset = 0; offset < max_offset; ) + { + apr_array_header_t *entries; + svn_fs_x__p2l_entry_t *last_entry; + int i; + + /* get all entries for the current block */ + SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, start, offset, + iterpool)); + if (entries->nelts == 0) + return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_CORRUPTION, + NULL, + _("p2l does not cover offset %s" + " for revision %ld"), + apr_off_t_toa(pool, offset), start); + + /* process all entries (and later continue with the next block) */ + last_entry + = &APR_ARRAY_IDX(entries, entries->nelts-1, svn_fs_x__p2l_entry_t); + offset = last_entry->offset + last_entry->size; + + for (i = 0; i < entries->nelts; ++i) + { + apr_uint32_t k; + svn_fs_x__p2l_entry_t *entry + = &APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t); + + /* check all sub-items for consist entries in the L2P index */ + for (k = 0; k < entry->item_count; ++k) + { + apr_off_t l2p_offset; + apr_uint32_t sub_item; + svn_fs_x__id_part_t *p2l_item = &entry->items[k]; + + SVN_ERR(svn_fs_x__item_offset(&l2p_offset, &sub_item, fs, + p2l_item->revision, NULL, + p2l_item->number, iterpool)); + + if (sub_item != k || l2p_offset != entry->offset) + return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT, + NULL, + _("l2p index entry PHYS o%s:s%ld " + "does not match p2l index value " + "LOG r%ld:i%ld for PHYS o%s:s%ld"), + apr_off_t_toa(pool, l2p_offset), + (long)sub_item, + p2l_item->revision, + (long)p2l_item->number, + apr_off_t_toa(pool, entry->offset), + (long)k); + } + } + + svn_pool_clear(iterpool); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Verify that the log-to-phys indexes and phys-to-log indexes are + * consistent with each other. The function signature is similar to + * svn_fs_x__verify. + * + * The values of START and END have already been auto-selected and + * verified. You may call this for format7 or higher repos. + */ +static svn_error_t * +verify_index_consistency(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t end, + svn_fs_progress_notify_func_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_revnum_t revision, pack_start, pack_end; + apr_pool_t *iterpool = svn_pool_create(pool); + + for (revision = start; revision <= end; revision = pack_end) + { + pack_start = svn_fs_x__packed_base_rev(fs, revision); + pack_end = pack_start + svn_fs_x__pack_size(fs, revision); + + if (notify_func && (pack_start % ffd->max_files_per_dir == 0)) + notify_func(pack_start, notify_baton, iterpool); + + /* two-way index check */ + SVN_ERR(compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start, + cancel_func, cancel_baton, iterpool)); + SVN_ERR(compare_p2l_to_l2p_index(fs, pack_start, pack_end - pack_start, + cancel_func, cancel_baton, iterpool)); + + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__verify(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t end, + svn_fs_progress_notify_func_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + fs_x_data_t *ffd = fs->fsap_data; + svn_revnum_t youngest = ffd->youngest_rev_cache; /* cache is current */ + + /* Input validation. */ + if (! SVN_IS_VALID_REVNUM(start)) + start = 0; + if (! SVN_IS_VALID_REVNUM(end)) + end = youngest; + SVN_ERR(svn_fs_x__ensure_revision_exists(start, fs, pool)); + SVN_ERR(svn_fs_x__ensure_revision_exists(end, fs, pool)); + + /* log/phys index consistency. We need to check them first to make + sure we can access the rev / pack files in format7. */ + SVN_ERR(verify_index_consistency(fs, start, end, + notify_func, notify_baton, + cancel_func, cancel_baton, pool)); + + /* rep cache consistency */ + SVN_ERR(verify_rep_cache(fs, start, end, notify_func, notify_baton, + cancel_func, cancel_baton, pool)); + + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_fs_x/verify.h b/subversion/libsvn_fs_x/verify.h new file mode 100644 index 0000000..210a7a6 --- /dev/null +++ b/subversion/libsvn_fs_x/verify.h
@@ -0,0 +1,42 @@ +/* verify.h : verification interface of the native filesystem layer + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS__VERIFY_H +#define SVN_LIBSVN_FS__VERIFY_H + +#include "fs.h" + +/* Verify metadata in fsx filesystem FS. Limit the checks to revisions + * START to END where possible. Indicate progress via the optional + * NOTIFY_FUNC callback using NOTIFY_BATON. The optional CANCEL_FUNC + * will periodically be called with CANCEL_BATON to allow for preemption. + * Use POOL for temporary allocations. */ +svn_error_t *svn_fs_x__verify(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t end, + svn_fs_progress_notify_func_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool); + +#endif
diff --git a/subversion/libsvn_subr/packed_data.c b/subversion/libsvn_subr/packed_data.c new file mode 100644 index 0000000..2e368a5 --- /dev/null +++ b/subversion/libsvn_subr/packed_data.c
@@ -0,0 +1,1076 @@ +/* packed_data.c : implement the packed binary stream data structure + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include <apr_tables.h> + +#include "svn_string.h" +#include "svn_sorts.h" +#include "private/svn_string_private.h" +#include "private/svn_subr_private.h" +#include "private/svn_delta_private.h" +#include "private/svn_packed_data.h" + +#include "svn_private_config.h" + + + +/* Private int stream data referenced by svn_packed__int_stream_t. + */ +typedef struct packed_int_private_t +{ + /* First sub-stream, if any. NULL otherwise. */ + svn_packed__int_stream_t *first_substream; + + /* Last sub-stream, if any. NULL otherwise. */ + svn_packed__int_stream_t *last_substream; + + /* Current sub-stream to read from / write to, if any. NULL otherwise. + This will be initialized to FIRST_SUBSTREAM and then advanced in a + round-robin scheme after each number being read. */ + svn_packed__int_stream_t *current_substream; + + /* Number of sub-streams. */ + apr_size_t substream_count; + + /* Next (sibling) integer stream. If this is the last one, points to + the first in the list (i.e. this forms a ring list). Never NULL. */ + svn_packed__int_stream_t *next; + + /* 7b/8b encoded integer values (previously diff'ed and sign-handled, + if indicated by the flags below). The contents are disjoint from + the unparsed number buffer. May be NULL while not written to. */ + svn_stringbuf_t *packed; + + /* Initialized to 0. Latest value written to / read from PACKED. + Undefined if DIFF is FALSE. */ + apr_uint64_t last_value; + + /* Deltify data before storing it in PACKED. */ + svn_boolean_t diff; + + /* Numbers are likely to contain negative values with small absolutes. + If TRUE, store the signed bit in LSB before encoding. */ + svn_boolean_t is_signed; + + /* Number of integers in this stream. */ + apr_size_t item_count; + + /* TRUE for the last stream in a list of siblings. */ + svn_boolean_t is_last; + + /* Pool to use for allocations. */ + apr_pool_t *pool; +} packed_int_private_t; + +/* A byte sequence stream. Please note that NEXT is defined different + * from the NEXT member in integer streams. + */ +struct svn_packed__byte_stream_t +{ + /* First sub-stream, if any. NULL otherwise. */ + svn_packed__byte_stream_t *first_substream; + + /* Last sub-stream, if any. NULL otherwise. */ + svn_packed__byte_stream_t *last_substream; + + /* Next (sibling) byte sequence stream, if any. NULL otherwise. */ + svn_packed__byte_stream_t *next; + + /* Stream to store the sequence lengths. */ + svn_packed__int_stream_t *lengths_stream; + + /* It's index (relative to its parent). */ + apr_size_t lengths_stream_index; + + /* Concatenated byte sequences. */ + svn_stringbuf_t *packed; + + /* Pool to use for allocations. */ + apr_pool_t *pool; +}; + +/* The serialization root object. It references the top-level streams. + */ +struct svn_packed__data_root_t +{ + /* First top-level integer stream, if any. NULL otherwise. */ + svn_packed__int_stream_t *first_int_stream; + + /* Last top-level integer stream, if any. NULL otherwise. */ + svn_packed__int_stream_t *last_int_stream; + + /* Number of top-level integer streams. */ + apr_size_t int_stream_count; + + /* First top-level byte sequence stream, if any. NULL otherwise. */ + svn_packed__byte_stream_t *first_byte_stream; + + /* Last top-level byte sequence stream, if any. NULL otherwise. */ + svn_packed__byte_stream_t *last_byte_stream; + + /* Number of top-level byte sequence streams. */ + apr_size_t byte_stream_count; + + /* Pool to use for allocations. */ + apr_pool_t *pool; +}; + +/* Write access. */ + +svn_packed__data_root_t * +svn_packed__data_create_root(apr_pool_t *pool) +{ + svn_packed__data_root_t *root = apr_pcalloc(pool, sizeof(*root)); + root->pool = pool; + + return root; +} + +svn_packed__int_stream_t * +svn_packed__create_int_stream(svn_packed__data_root_t *root, + svn_boolean_t diff, + svn_boolean_t signed_ints) +{ + /* allocate and initialize the stream node */ + packed_int_private_t *private_data + = apr_pcalloc(root->pool, sizeof(*private_data)); + svn_packed__int_stream_t *stream + = apr_palloc(root->pool, sizeof(*stream)); + + private_data->diff = diff; + private_data->is_signed = signed_ints; + private_data->is_last = TRUE; + private_data->pool = root->pool; + + stream->buffer_used = 0; + stream->private_data = private_data; + + /* maintain the ring list */ + if (root->last_int_stream) + { + packed_int_private_t *previous_private_data + = root->last_int_stream->private_data; + previous_private_data->next = stream; + previous_private_data->is_last = FALSE; + } + else + { + root->first_int_stream = stream; + } + + root->last_int_stream = stream; + root->int_stream_count++; + + return stream; +} + +svn_packed__int_stream_t * +svn_packed__create_int_substream(svn_packed__int_stream_t *parent, + svn_boolean_t diff, + svn_boolean_t signed_ints) +{ + packed_int_private_t *parent_private = parent->private_data; + + /* allocate and initialize the stream node */ + packed_int_private_t *private_data + = apr_pcalloc(parent_private->pool, sizeof(*private_data)); + svn_packed__int_stream_t *stream + = apr_palloc(parent_private->pool, sizeof(*stream)); + + private_data->diff = diff; + private_data->is_signed = signed_ints; + private_data->is_last = TRUE; + private_data->pool = parent_private->pool; + + stream->buffer_used = 0; + stream->private_data = private_data; + + /* maintain the ring list */ + if (parent_private->last_substream) + { + packed_int_private_t *previous_private_data + = parent_private->last_substream->private_data; + previous_private_data->next = stream; + previous_private_data->is_last = FALSE; + } + else + { + parent_private->first_substream = stream; + parent_private->current_substream = stream; + } + + parent_private->last_substream = stream; + parent_private->substream_count++; + private_data->next = parent_private->first_substream; + + return stream; +} + +/* Returns a new top-level byte sequence stream for ROOT but does not + * initialize the LENGTH_STREAM member. + */ +static svn_packed__byte_stream_t * +create_bytes_stream_body(svn_packed__data_root_t *root) +{ + svn_packed__byte_stream_t *stream + = apr_pcalloc(root->pool, sizeof(*stream)); + + stream->packed = svn_stringbuf_create_empty(root->pool); + + if (root->last_byte_stream) + root->last_byte_stream->next = stream; + else + root->first_byte_stream = stream; + + root->last_byte_stream = stream; + root->byte_stream_count++; + + return stream; +} + +svn_packed__byte_stream_t * +svn_packed__create_bytes_stream(svn_packed__data_root_t *root) +{ + svn_packed__byte_stream_t *stream + = create_bytes_stream_body(root); + + stream->lengths_stream_index = root->int_stream_count; + stream->lengths_stream = svn_packed__create_int_stream(root, FALSE, FALSE); + + return stream; +} + +/* Write the 7b/8b representation of VALUE into BUFFER. BUFFER must + * provide at least 10 bytes space. + * Returns the first position behind the written data. + */ +static unsigned char * +write_packed_uint_body(unsigned char *buffer, apr_uint64_t value) +{ + while (value >= 0x80) + { + *(buffer++) = (unsigned char)((value % 0x80) + 0x80); + value /= 0x80; + } + + *(buffer++) = (unsigned char)value; + return buffer; +} + +/* Empty the unprocessed integer buffer in STREAM by either pushing the + * data to the sub-streams or writing to the packed data (in case there + * are no sub-streams). + */ +static void +svn_packed__data_flush_buffer(svn_packed__int_stream_t *stream) +{ + packed_int_private_t *private_data = stream->private_data; + apr_size_t i; + + /* if we have sub-streams, push the data down to them */ + if (private_data->current_substream) + for (i = 0; i < stream->buffer_used; ++i) + { + packed_int_private_t *current_private_data + = private_data->current_substream->private_data; + + svn_packed__add_uint(private_data->current_substream, + stream->buffer[i]); + private_data->current_substream = current_private_data->next; + } + else + { + /* pack the numbers into our local PACKED buffer */ + + /* temporary buffer, max 10 bytes required per 7b/8b encoded number */ + unsigned char local_buffer[10 * SVN__PACKED_DATA_BUFFER_SIZE]; + unsigned char *p = local_buffer; + + /* if configured, deltify numbers before packing them. + Since delta may be negative, always use the 'signed' encoding. */ + if (private_data->diff) + { + apr_uint64_t last_value = private_data->last_value; + for (i = 0; i < stream->buffer_used; ++i) + { + apr_uint64_t temp = stream->buffer[i]; + apr_int64_t diff = (apr_int64_t)(temp - last_value); + stream->buffer[i] = diff < 0 ? -1 - 2 * diff : 2 * diff; + last_value = temp; + } + + private_data->last_value = last_value; + } + + /* if configured and not already done by the deltification above, + transform to 'signed' encoding. Store the sign in the LSB and + the absolute value (-1 for negative values) in the remaining + 63 bits. */ + if (!private_data->diff && private_data->is_signed) + for (i = 0; i < stream->buffer_used; ++i) + stream->buffer[i] = (apr_int64_t)stream->buffer[i] < 0 + ? -1 - 2 * stream->buffer[i] + : 2 * stream->buffer[i]; + + /* auto-create packed data buffer. Give it some reasonable initial + size - just enough for a few tens of values. */ + if (private_data->packed == NULL) + private_data->packed + = svn_stringbuf_create_ensure(256, private_data->pool); + + /* encode numbers into our temp buffer. */ + for (i = 0; i < stream->buffer_used; ++i) + p = write_packed_uint_body(p, stream->buffer[i]); + + /* append them to the final packed data */ + svn_stringbuf_appendbytes(private_data->packed, + (char *)local_buffer, + p - local_buffer); + } + + /* maintain counters */ + private_data->item_count += stream->buffer_used; + stream->buffer_used = 0; +} + +void +svn_packed__add_uint(svn_packed__int_stream_t *stream, + apr_uint64_t value) +{ + stream->buffer[stream->buffer_used] = value; + if (++stream->buffer_used == SVN__PACKED_DATA_BUFFER_SIZE) + svn_packed__data_flush_buffer(stream); +} + +void +svn_packed__add_int(svn_packed__int_stream_t *stream, + apr_int64_t value) +{ + svn_packed__add_uint(stream, (apr_uint64_t)value); +} + +void +svn_packed__add_bytes(svn_packed__byte_stream_t *stream, + const char *data, + apr_size_t len) +{ + svn_packed__add_uint(stream->lengths_stream, len); + svn_stringbuf_appendbytes(stream->packed, data, len); +} + +/* Append the 7b/8b encoded representation of VALUE to PACKED. + */ +static void +write_packed_uint(svn_stringbuf_t* packed, apr_uint64_t value) +{ + if (value < 0x80) + { + svn_stringbuf_appendbyte(packed, (char)value); + } + else + { + unsigned char buffer[10]; + unsigned char *p = write_packed_uint_body(buffer, value); + + svn_stringbuf_appendbytes(packed, (char *)buffer, p - buffer); + } +} + +/* Recursively write the structure (config parameters, sub-streams, data + * sizes) of the STREAM and all its siblings to the TREE_STRUCT buffer. + */ +static void +write_int_stream_structure(svn_stringbuf_t* tree_struct, + svn_packed__int_stream_t* stream) +{ + while (stream) + { + /* store config parameters and number of sub-streams in 1 number */ + packed_int_private_t *private_data = stream->private_data; + write_packed_uint(tree_struct, (private_data->substream_count << 2) + + (private_data->diff ? 1 : 0) + + (private_data->is_signed ? 2 : 0)); + + /* store item count and length their of packed representation */ + svn_packed__data_flush_buffer(stream); + + write_packed_uint(tree_struct, private_data->item_count); + write_packed_uint(tree_struct, private_data->packed + ? private_data->packed->len + : 0); + + /* append all sub-stream structures */ + write_int_stream_structure(tree_struct, private_data->first_substream); + + /* continue with next sibling */ + stream = private_data->is_last ? NULL : private_data->next; + } +} + +/* Recursively write the structure (sub-streams, data sizes) of the STREAM + * and all its siblings to the TREE_STRUCT buffer. + */ +static void +write_byte_stream_structure(svn_stringbuf_t* tree_struct, + svn_packed__byte_stream_t* stream) +{ + /* for this and all siblings */ + for (; stream; stream = stream->next) + { + /* this stream's structure and size */ + write_packed_uint(tree_struct, 0); + write_packed_uint(tree_struct, stream->lengths_stream_index); + write_packed_uint(tree_struct, stream->packed->len); + + /* followed by all its sub-streams */ + write_byte_stream_structure(tree_struct, stream->first_substream); + } +} + +/* Write the 7b/8b encoded representation of VALUE to STREAM. + */ +static svn_error_t * +write_stream_uint(svn_stream_t *stream, + apr_uint64_t value) +{ + unsigned char buffer[10]; + apr_size_t count = write_packed_uint_body(buffer, value) - buffer; + + SVN_ERR(svn_stream_write(stream, (char *)buffer, &count)); + + return SVN_NO_ERROR; +} + +/* Return the total size of all packed data in STREAM, its siblings and + * all sub-streams. To get an accurate value, flush all buffers prior to + * calling this function. + */ +static apr_size_t +packed_int_stream_length(svn_packed__int_stream_t *stream) +{ + packed_int_private_t *private_data = stream->private_data; + apr_size_t result = private_data->packed ? private_data->packed->len : 0; + + stream = private_data->first_substream; + while (stream) + { + private_data = stream->private_data; + result += packed_int_stream_length(stream); + stream = private_data->is_last ? NULL : private_data->next; + } + + return result; +} + +/* Return the total size of all byte sequences data in STREAM, its siblings + * and all sub-streams. + */ +static apr_size_t +packed_byte_stream_length(svn_packed__byte_stream_t *stream) +{ + apr_size_t result = stream->packed->len; + + for (stream = stream->first_substream; stream; stream = stream->next) + result += packed_byte_stream_length(stream); + + return result; +} + +/* Append all packed data in STREAM, its siblings and all sub-streams to + * COMBINED. + */ +static void +append_int_stream(svn_packed__int_stream_t *stream, + svn_stringbuf_t *combined) +{ + packed_int_private_t *private_data = stream->private_data; + if (private_data->packed) + svn_stringbuf_appendstr(combined, private_data->packed); + + stream = private_data->first_substream; + while (stream) + { + private_data = stream->private_data; + append_int_stream(stream, combined); + stream = private_data->is_last ? NULL : private_data->next; + } +} + +/* Append all byte sequences in STREAM, its siblings and all sub-streams + * to COMBINED. + */ +static void +append_byte_stream(svn_packed__byte_stream_t *stream, + svn_stringbuf_t *combined) +{ + svn_stringbuf_appendstr(combined, stream->packed); + + for (stream = stream->first_substream; stream; stream = stream->next) + append_byte_stream(stream, combined); +} + +/* Take the binary data in UNCOMPRESSED, zip it into COMPRESSED and write + * it to STREAM. COMPRESSED simply acts as a re-usable memory buffer. + * Clear all buffers (COMPRESSED, UNCOMPRESSED) at the end of the function. + */ +static svn_error_t * +write_stream_data(svn_stream_t *stream, + svn_stringbuf_t *uncompressed, + svn_stringbuf_t *compressed) +{ + SVN_ERR(svn__compress(uncompressed, + compressed, + SVN_DELTA_COMPRESSION_LEVEL_DEFAULT)); + + SVN_ERR(write_stream_uint(stream, compressed->len)); + SVN_ERR(svn_stream_write(stream, compressed->data, &compressed->len)); + + svn_stringbuf_setempty(uncompressed); + svn_stringbuf_setempty(compressed); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_packed__data_write(svn_stream_t *stream, + svn_packed__data_root_t *root, + apr_pool_t *scratch_pool) +{ + svn_packed__int_stream_t *int_stream; + svn_packed__byte_stream_t *byte_stream; + + /* re-usable data buffers */ + svn_stringbuf_t *compressed + = svn_stringbuf_create_ensure(1024, scratch_pool); + svn_stringbuf_t *uncompressed + = svn_stringbuf_create_ensure(1024, scratch_pool); + + /* write tree structure */ + svn_stringbuf_t *tree_struct + = svn_stringbuf_create_ensure(127, scratch_pool); + + write_packed_uint(tree_struct, root->int_stream_count); + write_int_stream_structure(tree_struct, root->first_int_stream); + + write_packed_uint(tree_struct, root->byte_stream_count); + write_byte_stream_structure(tree_struct, root->first_byte_stream); + + SVN_ERR(write_stream_uint(stream, tree_struct->len)); + SVN_ERR(svn_stream_write(stream, tree_struct->data, &tree_struct->len)); + + /* flatten sub-streams, zip them and write them to disk */ + + for (int_stream = root->first_int_stream; + int_stream; + int_stream = ((packed_int_private_t*)int_stream->private_data)->next) + { + apr_size_t len = packed_int_stream_length(int_stream); + svn_stringbuf_ensure(uncompressed, len); + + append_int_stream(int_stream, uncompressed); + SVN_ERR(write_stream_data(stream, uncompressed, compressed)); + } + + for (byte_stream = root->first_byte_stream; + byte_stream; + byte_stream = byte_stream->next) + { + apr_size_t len = packed_byte_stream_length(byte_stream); + svn_stringbuf_ensure(uncompressed, len); + + append_byte_stream(byte_stream, uncompressed); + SVN_ERR(write_stream_data(stream, uncompressed, compressed)); + } + + return SVN_NO_ERROR; +} + + +/* Read access. */ + +svn_packed__int_stream_t * +svn_packed__first_int_stream(svn_packed__data_root_t *root) +{ + return root->first_int_stream; +} + +svn_packed__byte_stream_t * +svn_packed__first_byte_stream(svn_packed__data_root_t *root) +{ + return root->first_byte_stream; +} + +svn_packed__int_stream_t * +svn_packed__next_int_stream(svn_packed__int_stream_t *stream) +{ + packed_int_private_t *private_data = stream->private_data; + return private_data->is_last ? NULL : private_data->next; +} + +svn_packed__byte_stream_t * +svn_packed__next_byte_stream(svn_packed__byte_stream_t *stream) +{ + return stream->next; +} + +svn_packed__int_stream_t * +svn_packed__first_int_substream(svn_packed__int_stream_t *stream) +{ + packed_int_private_t *private_data = stream->private_data; + return private_data->first_substream; +} + +apr_size_t +svn_packed__int_count(svn_packed__int_stream_t *stream) +{ + packed_int_private_t *private_data = stream->private_data; + return private_data->item_count + stream->buffer_used; +} + +apr_size_t +svn_packed__byte_count(svn_packed__byte_stream_t *stream) +{ + return stream->packed->len; +} + +/* Read one 7b/8b encoded value from *P and return it in *RESULT. Returns + * the first position after the parsed data. + * + * Overflows will be detected in the sense that it will end parsing the + * input but the result is undefined. + */ +static unsigned char * +read_packed_uint_body(unsigned char *p, apr_uint64_t *result) +{ + if (*p < 0x80) + { + *result = *p; + } + else + { + apr_uint64_t shift = 0; + apr_uint64_t value = 0; + while (*p >= 0x80) + { + value += (apr_uint64_t)(*p & 0x7f) << shift; + ++p; + + shift += 7; + if (shift > 64) + { + /* a definite overflow. Note, that numbers of 65 .. 70 + bits will not be detected as an overflow as they don't + threaten to exceed the input buffer. */ + *result = 0; + return p; + } + } + + *result = value + ((apr_uint64_t)*p << shift); + } + + return ++p; +} + +/* Read one 7b/8b encoded value from STREAM and return it in *RESULT. + * + * Overflows will be detected in the sense that it will end parsing the + * input but the result is undefined. + */ +static svn_error_t * +read_stream_uint(svn_stream_t *stream, apr_uint64_t *result) +{ + apr_uint64_t shift = 0; + apr_uint64_t value = 0; + unsigned char c; + + do + { + apr_size_t len = 1; + SVN_ERR(svn_stream_read(stream, (char *)&c, &len)); + if (len != 1) + return svn_error_create(SVN_ERR_CORRUPT_PACKED_DATA, NULL, + _("Unexpected end of stream")); + + value += (apr_uint64_t)(c & 0x7f) << shift; + shift += 7; + if (shift > 64) + return svn_error_create(SVN_ERR_CORRUPT_PACKED_DATA, NULL, + _("Integer representation too long")); + } + while (c >= 0x80); + + *result = value; + return SVN_NO_ERROR; +} + +/* Extract and return the next integer from PACKED and make PACKED point + * to the next integer. + */ +static apr_uint64_t +read_packed_uint(svn_stringbuf_t *packed) +{ + apr_uint64_t result = 0; + unsigned char *p = (unsigned char *)packed->data; + apr_size_t read = read_packed_uint_body(p, &result) - p; + + if (read > packed->len) + read = packed->len; + + packed->data += read; + packed->blocksize -= read; + packed->len -= read; + + return result; +} + +/* Ensure that STREAM contains at least one item in its buffer. + */ +static void +svn_packed__data_fill_buffer(svn_packed__int_stream_t *stream) +{ + packed_int_private_t *private_data = stream->private_data; + apr_size_t i; + apr_size_t end = MIN(SVN__PACKED_DATA_BUFFER_SIZE, + private_data->item_count); + + /* in case, some user calls us explicitly without a good reason ... */ + if (stream->buffer_used) + return; + + /* can we get data from the sub-streams or do we have to decode it from + our local packed container? */ + if (private_data->current_substream) + for (i = end; i > 0; --i) + { + packed_int_private_t *current_private_data + = private_data->current_substream->private_data; + stream->buffer[i-1] + = svn_packed__get_uint(private_data->current_substream); + private_data->current_substream = current_private_data->next; + } + else + { + /* use this local buffer only if the packed data is shorter than this. + The goal is that we don't need to check for overflows that is not + detected by read_packed_uint_body. */ + unsigned char local_buffer[10 * SVN__PACKED_DATA_BUFFER_SIZE]; + unsigned char *p; + unsigned char *start; + apr_size_t packed_read; + + if (private_data->packed->len < sizeof(local_buffer)) + { + apr_size_t trail = sizeof(local_buffer) - private_data->packed->len; + memcpy(local_buffer, + private_data->packed->data, + private_data->packed->len); + memset(local_buffer + private_data->packed->len, 0, MIN(trail, end)); + + p = local_buffer; + } + else + p = (unsigned char *)private_data->packed->data; + + /* unpack numbers */ + start = p; + for (i = end; i > 0; --i) + p = read_packed_uint_body(p, &stream->buffer[i-1]); + + /* adjust remaining packed data buffer */ + packed_read = p - start; + private_data->packed->data += packed_read; + private_data->packed->len -= packed_read; + private_data->packed->blocksize -= packed_read; + + /* undeltify numbers, if configured */ + if (private_data->diff) + { + apr_uint64_t last_value = private_data->last_value; + for (i = end; i > 0; --i) + { + apr_uint64_t temp = stream->buffer[i-1]; + temp = (temp % 2) ? -1 - temp / 2 : temp / 2; + last_value += temp; + stream->buffer[i-1] = last_value; + } + + private_data->last_value = last_value; + } + + /* handle signed values, if configured and not handled already */ + if (!private_data->diff && private_data->is_signed) + for (i = 0; i < end; ++i) + stream->buffer[i] = (stream->buffer[i] % 2) + ? -1 - stream->buffer[i] / 2 + : stream->buffer[i] / 2; + } + + stream->buffer_used = end; + private_data->item_count -= end; +} + +apr_uint64_t +svn_packed__get_uint(svn_packed__int_stream_t *stream) +{ + if (stream->buffer_used == 0) + svn_packed__data_fill_buffer(stream); + + return stream->buffer_used ? stream->buffer[--stream->buffer_used] : 0; +} + +apr_int64_t +svn_packed__get_int(svn_packed__int_stream_t *stream) +{ + return (apr_int64_t)svn_packed__get_uint(stream); +} + +const char * +svn_packed__get_bytes(svn_packed__byte_stream_t *stream, + apr_size_t *len) +{ + const char *result = stream->packed->data; + apr_size_t count = svn_packed__get_uint(stream->lengths_stream); + + if (count > stream->packed->len) + count = stream->packed->len; + + /* advance packed buffer */ + stream->packed->data += count; + stream->packed->len -= count; + stream->packed->blocksize -= count; + + *len = count; + return result; +} + +/* Read the integer stream structure and recreate it in STREAM, including + * sub-streams, from TREE_STRUCT. + */ +static void +read_int_stream_structure(svn_stringbuf_t *tree_struct, + svn_packed__int_stream_t *stream) +{ + packed_int_private_t *private_data = stream->private_data; + apr_uint64_t value = read_packed_uint(tree_struct); + apr_size_t substream_count; + apr_size_t i; + + /* extract local parameters */ + private_data->diff = (value & 1) != 0; + private_data->is_signed = (value & 2) != 0; + substream_count = (apr_size_t)(value >> 2); + + /* read item count & packed size; allocate packed data buffer */ + private_data->item_count = (apr_size_t)read_packed_uint(tree_struct); + value = read_packed_uint(tree_struct); + if (value) + { + private_data->packed = svn_stringbuf_create_ensure((apr_size_t)value, + private_data->pool); + private_data->packed->len = (apr_size_t)value; + } + + /* add sub-streams and read their config, too */ + for (i = 0; i < substream_count; ++i) + read_int_stream_structure(tree_struct, + svn_packed__create_int_substream(stream, + FALSE, + FALSE)); +} + +/* Read the integer stream structure and recreate it in STREAM, including + * sub-streams, from TREE_STRUCT. FIRST_INT_STREAM is the integer stream + * that would correspond to lengths_stream_index 0. + */ +static void +read_byte_stream_structure(svn_stringbuf_t *tree_struct, + svn_packed__byte_stream_t *stream, + svn_packed__int_stream_t *first_int_stream) +{ + /* read parameters from the TREE_STRUCT buffer */ + apr_size_t dummy = (apr_size_t)read_packed_uint(tree_struct); + apr_size_t lengths_stream_index = (apr_size_t)read_packed_uint(tree_struct); + apr_size_t packed_size = (apr_size_t)read_packed_uint(tree_struct); + apr_size_t i; + + /* allocate byte sequence buffer size */ + svn_stringbuf_ensure(stream->packed, packed_size); + stream->packed->len = packed_size; + + /* navigate to the (already existing) lengths_stream */ + stream->lengths_stream_index = lengths_stream_index; + stream->lengths_stream = first_int_stream; + for (i = 0; i < lengths_stream_index; ++i) + { + packed_int_private_t *length_private + = stream->lengths_stream->private_data; + stream->lengths_stream = length_private->next; + } +} + +/* Read a compressed block from STREAM and uncompress it into UNCOMPRESSED. + * UNCOMPRESSED_LEN is the expected size of the stream. COMPRESSED is a + * re-used buffer for temporary data. + */ +static svn_error_t * +read_stream_data(svn_stream_t *stream, + apr_size_t uncompressed_len, + svn_stringbuf_t *uncompressed, + svn_stringbuf_t *compressed) +{ + apr_uint64_t len; + apr_size_t compressed_len; + + SVN_ERR(read_stream_uint(stream, &len)); + compressed_len = (apr_size_t)len; + + svn_stringbuf_ensure(compressed, compressed_len); + compressed->len = compressed_len; + SVN_ERR(svn_stream_read(stream, compressed->data, &compressed->len)); + compressed->data[compressed_len] = '\0'; + + SVN_ERR(svn__decompress(compressed, uncompressed, uncompressed_len)); + + return SVN_NO_ERROR; +} + +/* Read the packed contents from COMBINED, starting at *OFFSET and store + * it in STREAM. Update *OFFSET to point to the next stream's data and + * continue with the sub-streams. + */ +static void +unflatten_int_stream(svn_packed__int_stream_t *stream, + svn_stringbuf_t *combined, + apr_size_t *offset) +{ + packed_int_private_t *private_data = stream->private_data; + if (private_data->packed) + { + memcpy(private_data->packed->data, + combined->data + *offset, + private_data->packed->len); + + private_data->packed->data[private_data->packed->len] = '\0'; + *offset += private_data->packed->len; + } + + stream = private_data->first_substream; + while (stream) + { + private_data = stream->private_data; + unflatten_int_stream(stream, combined, offset); + stream = private_data->is_last ? NULL : private_data->next; + } +} + +/* Read the packed contents from COMBINED, starting at *OFFSET and store + * it in STREAM. Update *OFFSET to point to the next stream's data and + * continue with the sub-streams. + */ +static void +unflatten_byte_stream(svn_packed__byte_stream_t *stream, + svn_stringbuf_t *combined, + apr_size_t *offset) +{ + memcpy(stream->packed->data, + combined->data + *offset, + stream->packed->len); + stream->packed->data[stream->packed->len] = '\0'; + + *offset += stream->packed->len; + for (stream = stream->first_substream; stream; stream = stream->next) + unflatten_byte_stream(stream, combined, offset); +} + +svn_error_t * +svn_packed__data_read(svn_packed__data_root_t **root_p, + svn_stream_t *stream, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + apr_size_t i; + apr_size_t count; + + svn_packed__int_stream_t *int_stream; + svn_packed__byte_stream_t *byte_stream; + svn_packed__data_root_t *root = svn_packed__data_create_root(result_pool); + + svn_stringbuf_t *compressed + = svn_stringbuf_create_ensure(1024, scratch_pool); + svn_stringbuf_t *uncompressed + = svn_stringbuf_create_ensure(1024, scratch_pool); + + /* read tree structure */ + + apr_uint64_t tree_struct_size; + svn_stringbuf_t *tree_struct; + + SVN_ERR(read_stream_uint(stream, &tree_struct_size)); + tree_struct + = svn_stringbuf_create_ensure((apr_size_t)tree_struct_size, scratch_pool); + tree_struct->len = (apr_size_t)tree_struct_size; + + SVN_ERR(svn_stream_read(stream, tree_struct->data, &tree_struct->len)); + tree_struct->data[tree_struct->len] = '\0'; + + /* reconstruct tree structure */ + + count = read_packed_uint(tree_struct); + for (i = 0; i < count; ++i) + read_int_stream_structure(tree_struct, + svn_packed__create_int_stream(root, FALSE, + FALSE)); + + count = read_packed_uint(tree_struct); + for (i = 0; i < count; ++i) + read_byte_stream_structure(tree_struct, + create_bytes_stream_body(root), + root->first_int_stream); + + /* read sub-stream data from disk, unzip it and buffer it */ + + for (int_stream = root->first_int_stream; + int_stream; + int_stream = ((packed_int_private_t*)int_stream->private_data)->next) + { + apr_size_t offset = 0; + SVN_ERR(read_stream_data(stream, + packed_int_stream_length(int_stream), + uncompressed, compressed)); + unflatten_int_stream(int_stream, uncompressed, &offset); + } + + for (byte_stream = root->first_byte_stream; + byte_stream; + byte_stream = byte_stream->next) + { + apr_size_t offset = 0; + SVN_ERR(read_stream_data(stream, + packed_byte_stream_length(byte_stream), + uncompressed, compressed)); + unflatten_byte_stream(byte_stream, uncompressed, &offset); + } + + *root_p = root; + return SVN_NO_ERROR; +}
diff --git a/subversion/libsvn_subr/prefix_string.c b/subversion/libsvn_subr/prefix_string.c new file mode 100644 index 0000000..080c4fa --- /dev/null +++ b/subversion/libsvn_subr/prefix_string.c
@@ -0,0 +1,315 @@ +/* prefix_string.c --- implement strings based on a prefix tree + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include <assert.h> +#include "private/svn_string_private.h" + +/* A node in the tree represents a common prefix. The root node is the + * empty prefix. Nodes may have up to 256 sub-nodes, each starting with + * a different character (possibly '\0'). + * + * The nodes in the tree store only up to 8 chars of the respective common + * prefix, i.e. longer common prefixes must be drawn out over multiple + * hierarchy levels. This is a space <-> efficiency trade-off. + * + * Strings are the leaf nodes in the tree and use a specialized, smaller + * data structure. They may add 0 to 7 extra chars to the prefix. Both + * data types can be discerned by the last char in the data buffer. This + * must be 0 for strings (leaves) and non-0 otherwise. Please note that + * ordinary nodes have a length information so that no terminating 0 is + * required for them. + */ + +/* forward declaration */ +typedef struct node_t node_t; + +/* String type and tree leaf. + */ +struct svn_prefix_string__t +{ + /* mandatory prefix */ + node_t *prefix; + + /* 0 ..7 chars to add the the prefix. NUL-terminated. */ + char data[8]; +}; + +/* A node inside the tree, i.e. not a string and not a leaf (unless this is + * the root node). + * + * Note: keep the ordering to minimize size / alignment overhead on 64 bit + * machines. + */ +struct node_t +{ + /* pointer to the parent prefix plus the 1 .. 8 extra chars. + * Only the root will provide 0 extra chars. */ + svn_prefix_string__t key; + + /* Length of the prefix from the root down to and including this one. + * 0 for the root node. Only then will key.prefix be NULL. */ + apr_uint32_t length; + + /* Number of entries used in SUB_NODES. */ + apr_uint32_t sub_node_count; + + /* The sub-nodes, ordered by first char. node_t and svn_prefix_string__t + * may be mixed here. May be NULL. + * The number of allocated entries is always a power-of-two and only + * given implicitly by SUB_NODE_COUNT. */ + struct node_t **sub_nodes; +}; + +/* The actual tree structure. */ +struct svn_prefix_tree__t +{ + /* the common tree root (represents the empty prefix). */ + node_t *root; + + /* all sub-nodes & strings will be allocated from this pool */ + apr_pool_t *pool; +}; + +/* Return TRUE, iff NODE is a leaf node. + */ +static svn_boolean_t +is_leaf(node_t *node) +{ + return node->key.data[7] == 0; +} + +/* Ensure that the sub-nodes array of NODE within TREE has at least one + * unused entry. Re-allocate as necessary. + */ +static void +auto_realloc_sub_nodes(svn_prefix_tree__t *tree, + node_t *node) +{ + if (node->sub_node_count & (node->sub_node_count - 1)) + return; + + if (node->sub_node_count == 0) + { + node->sub_nodes = apr_pcalloc(tree->pool, sizeof(*node->sub_nodes)); + } + else + { + node_t **sub_nodes + = apr_pcalloc(tree->pool, + 2 * node->sub_node_count * sizeof(*sub_nodes)); + memcpy(sub_nodes, node->sub_nodes, + node->sub_node_count * sizeof(*sub_nodes)); + node->sub_nodes = sub_nodes; + } +} + +/* Given the COUNT pointers in the SUB_NODES array, return the location at + * which KEY is either located or would be inserted. + */ +static int +search_lower_bound(node_t **sub_nodes, + unsigned char key, + int count) +{ + int lower = 0; + int upper = count - 1; + + /* Binary search for the lowest position at which to insert KEY. */ + while (lower <= upper) + { + int current = lower + (upper - lower) / 2; + + if ((unsigned char)sub_nodes[current]->key.data[0] < key) + lower = current + 1; + else + upper = current - 1; + } + + return lower; +} + +svn_prefix_tree__t * +svn_prefix_tree__create(apr_pool_t *pool) +{ + svn_prefix_tree__t *tree = apr_pcalloc(pool, sizeof(*tree)); + tree->pool = pool; + + tree->root = apr_pcalloc(pool, sizeof(*tree->root)); + tree->root->key.data[7] = '\xff'; + + return tree; +} + +svn_prefix_string__t * +svn_prefix_string__create(svn_prefix_tree__t *tree, + const char *s) +{ + svn_prefix_string__t *new_string; + apr_size_t len = strlen(s); + node_t *node = tree->root; + node_t *new_node; + int idx; + + /* walk the existing tree until we either find S or the node at which S + * has to be inserted */ + while (TRUE) + { + node_t *sub_node; + int match = 1; + + /* index of the matching sub-node */ + idx = node->sub_node_count + ? search_lower_bound(node->sub_nodes, + (unsigned char)s[node->length], + node->sub_node_count) + : 0; + + /* any (partially) matching sub-nodes? */ + if (idx == node->sub_node_count + || node->sub_nodes[idx]->key.data[0] != s[node->length]) + break; + + sub_node = node->sub_nodes[idx]; + + /* fully matching sub-node? */ + if (is_leaf(sub_node)) + { + if (strcmp(sub_node->key.data, s + node->length) == 0) + return &sub_node->key; + } + else + { + apr_size_t sub_node_len = sub_node->length - node->length; + if (strncmp(sub_node->key.data, s + node->length, + sub_node_len) == 0) + { + node = sub_node; + continue; + } + } + + /* partial match -> split */ + while (sub_node->key.data[match] == s[node->length + match]) + ++match; + + new_node = apr_pcalloc(tree->pool, sizeof(*new_node)); + new_node->key = sub_node->key; + new_node->length = node->length + match; + new_node->key.data[7] = '\xff'; + new_node->sub_node_count = 1; + new_node->sub_nodes = apr_palloc(tree->pool, sizeof(node_t *)); + new_node->sub_nodes[0] = sub_node; + + memcpy(sub_node->key.data, sub_node->key.data + match, 8 - match); + + /* replace old sub-node with new one and continue lookup */ + sub_node->key.prefix = new_node; + node->sub_nodes[idx] = new_node; + node = new_node; + } + + /* add sub-node(s) and final string */ + while (node->length + 7 < len) + { + new_node = apr_pcalloc(tree->pool, sizeof(*new_node)); + new_node->key.prefix = node; + new_node->length = node->length + 8; + memcpy(new_node->key.data, s + node->length, 8); + + auto_realloc_sub_nodes(tree, node); + memmove(node->sub_nodes + idx + 1, node->sub_nodes + idx, + (node->sub_node_count - idx) * sizeof(node_t *)); + + /* replace old sub-node with new one and continue lookup */ + node->sub_nodes[idx] = new_node; + node->sub_node_count++; + node = new_node; + idx = 0; + } + + new_string = apr_pcalloc(tree->pool, sizeof(*new_string)); + new_string->prefix = node; + memcpy(new_string->data, s + node->length, len - node->length); + + auto_realloc_sub_nodes(tree, node); + memmove(node->sub_nodes + idx + 1, node->sub_nodes + idx, + (node->sub_node_count - idx) * sizeof(node_t *)); + + node->sub_nodes[idx] = (node_t *)new_string; + node->sub_node_count++; + return new_string; +} + +svn_string_t * +svn_prefix_string__expand(const svn_prefix_string__t *s, + apr_pool_t *pool) +{ + apr_size_t s_len = strlen(s->data); + apr_size_t len = s->prefix->length + s_len; + char *buffer = apr_palloc(pool, len + 1); + + svn_string_t *result = apr_pcalloc(pool, sizeof(result)); + result->data = buffer; + result->len = len; + buffer[len] = '\0'; + + while (s->prefix) + { + memcpy(buffer + s->prefix->length, s->data, len - s->prefix->length); + len = s->prefix->length; + s = &s->prefix->key; + } + + return result; +} + +int +svn_prefix_string__compare(const svn_prefix_string__t *lhs, + const svn_prefix_string__t *rhs) +{ + const node_t *lhs_parent = lhs->prefix; + const node_t *rhs_parent = rhs->prefix; + + if (lhs == rhs) + return 0; + + /* find the common root */ + while (lhs_parent != rhs_parent) + { + if (lhs_parent->length <= rhs_parent->length) + { + rhs = &rhs_parent->key; + rhs_parent = rhs_parent->key.prefix; + } + else if (rhs_parent->length <= lhs_parent->length) + { + lhs = &lhs_parent->key; + lhs_parent = lhs_parent->key.prefix; + } + + /* same tree? */ + assert(lhs_parent && rhs_parent); + } + + /* at the common root, strings will differ in the first follow-up char */ + return (int)(unsigned char)lhs->data[0] - (int)(unsigned char)rhs->data[0]; +}
diff --git a/subversion/libsvn_subr/sorts.c b/subversion/libsvn_subr/sorts.c index 3979d50..da004d5 100644 --- a/subversion/libsvn_subr/sorts.c +++ b/subversion/libsvn_subr/sorts.c
@@ -309,3 +309,148 @@ } } } + +/* Our priority queue data structure: + * Simply remember the constructor parameters. + */ +struct svn_priority_queue__t +{ + /* the queue elements, ordered as a heap according to COMPARE_FUNC */ + apr_array_header_t *elements; + + /* predicate used to order the heap */ + int (*compare_func)(const void *, const void *); +}; + +/* Return TRUE, if heap element number LHS in QUEUE is smaller than element + * number RHS according to QUEUE->COMPARE_FUNC + */ +static int +heap_is_less(svn_priority_queue__t *queue, + apr_size_t lhs, + apr_size_t rhs) +{ + char *lhs_value = queue->elements->elts + lhs * queue->elements->elt_size; + char *rhs_value = queue->elements->elts + rhs * queue->elements->elt_size; + + assert(lhs < queue->elements->nelts); + assert(rhs < queue->elements->nelts); + return queue->compare_func((void *)lhs_value, (void *)rhs_value) < 0; +} + +/* Exchange elements number LHS and RHS in QUEUE. + */ +static void +heap_swap(svn_priority_queue__t *queue, + apr_size_t lhs, + apr_size_t rhs) +{ + int i; + char *lhs_value = queue->elements->elts + lhs * queue->elements->elt_size; + char *rhs_value = queue->elements->elts + rhs * queue->elements->elt_size; + + for (i = 0; i < queue->elements->elt_size; ++i) + { + char temp = lhs_value[i]; + lhs_value[i] = rhs_value[i]; + rhs_value[i] = temp; + } +} + +/* Move element number IDX to lower indexes until the heap criterion is + * fulfilled again. + */ +static void +heap_bubble_down(svn_priority_queue__t *queue, + int idx) +{ + while (idx > 0 && heap_is_less(queue, idx, (idx - 1) / 2)) + { + heap_swap(queue, idx, (idx - 1) / 2); + idx = (idx - 1) / 2; + } +} + +/* Move element number IDX to higher indexes until the heap criterion is + * fulfilled again. + */ +static void +heap_bubble_up(svn_priority_queue__t *queue, + int idx) +{ + while (2 * idx + 2 < queue->elements->nelts) + { + int child = heap_is_less(queue, 2 * idx + 1, 2 * idx + 2) + ? 2 * idx + 1 + : 2 * idx + 2; + + if (heap_is_less(queue, idx, child)) + return; + + heap_swap(queue, idx, child); + idx = child; + } + + if ( 2 * idx + 1 < queue->elements->nelts + && heap_is_less(queue, 2 * idx + 1, idx)) + heap_swap(queue, 2 * idx + 1, idx); +} + +svn_priority_queue__t * +svn_priority_queue__create(apr_array_header_t *elements, + int (*compare_func)(const void *, const void *)) +{ + int i; + + svn_priority_queue__t *queue = apr_pcalloc(elements->pool, sizeof(*queue)); + queue->elements = elements; + queue->compare_func = compare_func; + + for (i = elements->nelts / 2; i >= 0; --i) + heap_bubble_up(queue, i); + + return queue; +} + +apr_size_t +svn_priority_queue__size(svn_priority_queue__t *queue) +{ + return queue->elements->nelts; +} + +void * +svn_priority_queue__peek(svn_priority_queue__t *queue) +{ + return queue->elements->nelts ? queue->elements->elts : NULL; +} + +void +svn_priority_queue__update(svn_priority_queue__t *queue) +{ + heap_bubble_up(queue, 0); +} + +void +svn_priority_queue__pop(svn_priority_queue__t *queue) +{ + if (queue->elements->nelts) + { + memcpy(queue->elements->elts, + queue->elements->elts + (queue->elements->nelts - 1) + * queue->elements->elt_size, + queue->elements->elt_size); + --queue->elements->nelts; + heap_bubble_up(queue, 0); + } +} + +void +svn_priority_queue__push(svn_priority_queue__t *queue, + const void *element) +{ + /* we cannot duplicate elements due to potential array re-allocs */ + assert(element && element != queue->elements->elts); + + memcpy(apr_array_push(queue->elements), element, queue->elements->elt_size); + heap_bubble_down(queue, queue->elements->nelts - 1); +} \ No newline at end of file
diff --git a/subversion/libsvn_subr/spillbuf.c b/subversion/libsvn_subr/spillbuf.c index 8b48d47..703596c 100644 --- a/subversion/libsvn_subr/spillbuf.c +++ b/subversion/libsvn_subr/spillbuf.c
@@ -92,7 +92,7 @@ struct svn_spillbuf_reader_t { /* Embed the spill-buffer within the reader. */ - struct svn_spillbuf_t buf; + struct svn_spillbuf_t *buf; /* When we read content from the underlying spillbuf, these fields store the ptr/len pair. The ptr will be incremented as we "read" out of this @@ -527,7 +527,7 @@ apr_pool_t *result_pool) { svn_spillbuf_reader_t *sbr = apr_pcalloc(result_pool, sizeof(*sbr)); - init_spillbuf(&sbr->buf, blocksize, maxsize, result_pool); + sbr->buf = svn_spillbuf__create(blocksize, maxsize, result_pool); return sbr; } @@ -569,7 +569,7 @@ if (reader->sb_len == 0) { SVN_ERR(svn_spillbuf__read(&reader->sb_ptr, &reader->sb_len, - &reader->buf, + reader->buf, scratch_pool)); /* We've run out of content, so return with whatever has @@ -628,7 +628,8 @@ if (reader->sb_len > 0) { if (reader->save_ptr == NULL) - reader->save_ptr = apr_palloc(reader->buf.pool, reader->buf.blocksize); + reader->save_ptr = apr_palloc(reader->buf->pool, + reader->buf->blocksize); memcpy(reader->save_ptr, reader->sb_ptr, reader->sb_len); reader->save_len = reader->sb_len; @@ -638,7 +639,7 @@ reader->sb_len = 0; } - return svn_error_trace(svn_spillbuf__write(&reader->buf, data, len, + return svn_error_trace(svn_spillbuf__write(reader->buf, data, len, scratch_pool)); } @@ -677,14 +678,14 @@ svn_stream_t * -svn_stream__from_spillbuf(apr_size_t blocksize, - apr_size_t maxsize, +svn_stream__from_spillbuf(svn_spillbuf_t *buf, apr_pool_t *result_pool) { svn_stream_t *stream; struct spillbuf_baton *sb = apr_palloc(result_pool, sizeof(*sb)); - sb->reader = svn_spillbuf__reader_create(blocksize, maxsize, result_pool); + sb->reader = apr_pcalloc(result_pool, sizeof(*sb->reader)); + sb->reader->buf = buf; sb->scratch_pool = svn_pool_create(result_pool); stream = svn_stream_create(sb, result_pool);
diff --git a/subversion/libsvn_subr/stream.c b/subversion/libsvn_subr/stream.c index 93a4c42..8a54bf1 100644 --- a/subversion/libsvn_subr/stream.c +++ b/subversion/libsvn_subr/stream.c
@@ -1657,7 +1657,7 @@ } -/* These are somewhat arbirary, if we ever get good empirical data as to +/* These are somewhat arbitrary, if we ever get good empirical data as to actually valid values, feel free to update them. */ #define BUFFER_BLOCK_SIZE 1024 #define BUFFER_MAX_SIZE 100000 @@ -1665,7 +1665,9 @@ svn_stream_t * svn_stream_buffered(apr_pool_t *result_pool) { - return svn_stream__from_spillbuf(BUFFER_BLOCK_SIZE, BUFFER_MAX_SIZE, + return svn_stream__from_spillbuf(svn_spillbuf__create(BUFFER_BLOCK_SIZE, + BUFFER_MAX_SIZE, + result_pool), result_pool); }
diff --git a/subversion/libsvn_subr/string.c b/subversion/libsvn_subr/string.c index 332ba52..e49348a 100644 --- a/subversion/libsvn_subr/string.c +++ b/subversion/libsvn_subr/string.c
@@ -606,6 +606,21 @@ to null-terminate. */ } +void +svn_stringbuf_appendfill(svn_stringbuf_t *str, + char byte, + apr_size_t count) +{ + apr_size_t new_len = str->len + count; + svn_stringbuf_ensure(str, new_len); + + memset(str->data + str->len, byte, count); + + /* update buffer length and always NUL-terminate it */ + str->len = new_len; + str->data[new_len] = '\0'; +} + void svn_stringbuf_appendstr(svn_stringbuf_t *targetstr,
diff --git a/subversion/svn_private_config.hw b/subversion/svn_private_config.hw index 134ca31..ec911ff 100644 --- a/subversion/svn_private_config.hw +++ b/subversion/svn_private_config.hw
@@ -57,6 +57,9 @@ /* Link fs fs library into the fs library */ #define SVN_LIBSVN_FS_LINKS_FS_FS +/* Link fs fs library into the fs library */ +#define SVN_LIBSVN_FS_LINKS_FS_X + /* Link local repos access library to client */ #define SVN_LIBSVN_CLIENT_LINKS_RA_LOCAL
diff --git a/subversion/svnadmin/svnadmin.c b/subversion/svnadmin/svnadmin.c index e10bb2f..a111e60 100644 --- a/subversion/svnadmin/svnadmin.c +++ b/subversion/svnadmin/svnadmin.c
@@ -243,7 +243,9 @@ N_("set repos UUID to that found in stream, if any")}, {"fs-type", svnadmin__fs_type, 1, - N_("type of repository: 'fsfs' (default) or 'bdb'")}, + N_("type of repository:\n" + " 'fsfs' (default), 'bdb' or 'fsx'\n" + " CAUTION: FSX is for EXPERIMENTAL use only!")}, {"parent-dir", svnadmin__parent_dir, 1, N_("load at specified directory in repository")}, @@ -419,7 +421,7 @@ ("usage: svnadmin pack REPOS_PATH\n\n" "Possibly compact the repository into a more efficient storage model.\n" "This may not apply to all repositories, in which case, exit.\n"), - {'q'} }, + {'q', 'M'} }, {"recover", subcommand_recover, {0}, N_ ("usage: svnadmin recover REPOS_PATH\n\n" @@ -698,18 +700,39 @@ svn_hash_sets(fs_config, SVN_FS_CONFIG_PRE_1_6_COMPATIBLE, "1"); if (! svn_version__at_least(opt_state->compatible_version, 1, 8, 0)) svn_hash_sets(fs_config, SVN_FS_CONFIG_PRE_1_8_COMPATIBLE, "1"); + /* In 1.9, we figured out that we didn't have to keep extending this + madness indefinitely. */ + svn_hash_sets(fs_config, SVN_FS_CONFIG_COMPATIBLE_VERSION, + apr_psprintf(pool, "%d.%d.%d%s%s", + opt_state->compatible_version->major, + opt_state->compatible_version->minor, + opt_state->compatible_version->patch, + opt_state->compatible_version->tag + ? "-" : "", + opt_state->compatible_version->tag)); } - if (opt_state->compatible_version - && ! svn_version__at_least(opt_state->compatible_version, 1, 1, 0) - /* ### TODO: this NULL check hard-codes knowledge of the library's - default fs-type value */ - && (opt_state->fs_type == NULL - || !strcmp(opt_state->fs_type, SVN_FS_TYPE_FSFS))) + if (opt_state->compatible_version) { - return svn_error_create(SVN_ERR_CL_ARG_PARSING_ERROR, NULL, - _("Repositories compatible with 1.0.x must use " - "--fs-type=bdb")); + if (! svn_version__at_least(opt_state->compatible_version, 1, 1, 0) + /* ### TODO: this NULL check hard-codes knowledge of the library's + default fs-type value */ + && (opt_state->fs_type == NULL + || !strcmp(opt_state->fs_type, SVN_FS_TYPE_FSFS))) + { + return svn_error_create(SVN_ERR_CL_ARG_PARSING_ERROR, NULL, + _("Repositories compatible with 1.0.x must " + "use --fs-type=bdb")); + } + + if (! svn_version__at_least(opt_state->compatible_version, 1, 9, 0) + && opt_state->fs_type && !strcmp(opt_state->fs_type, SVN_FS_TYPE_FSX)) + { + return svn_error_createf(SVN_ERR_CL_ARG_PARSING_ERROR, NULL, + _("Repositories compatible with 1.8.x or " + "earlier cannot use --fs-type=%s"), + SVN_FS_TYPE_FSX); + } } SVN_ERR(svn_repos_create(&repos, opt_state->repository_path,
diff --git a/subversion/tests/cmdline/svnadmin_tests.py b/subversion/tests/cmdline/svnadmin_tests.py index dfcd84a..59dbbc1 100755 --- a/subversion/tests/cmdline/svnadmin_tests.py +++ b/subversion/tests/cmdline/svnadmin_tests.py
@@ -61,8 +61,7 @@ if origerr or backerr or origout != backout: raise svntest.Failure -def check_hotcopy_fsfs(src, dst): - "Verify that the SRC FSFS repository has been correctly copied to DST." +def check_hotcopy_fsfs_fsx(src, dst): # Walk the source and compare all files to the destination for src_dirpath, src_dirs, src_files in os.walk(src): # Verify that the current directory exists in the destination @@ -146,6 +145,14 @@ f1.close() f2.close() +def check_hotcopy_fsfs(src, dst): + "Verify that the SRC FSFS repository has been correctly copied to DST." + check_hotcopy_fsfs_fsx(src, dst) + +def check_hotcopy_fsx(src, dst): + "Verify that the SRC FSX repository has been correctly copied to DST." + check_hotcopy_fsfs_fsx(src, dst) + #---------------------------------------------------------------------- # How we currently test 'svnadmin' -- @@ -457,8 +464,10 @@ if svntest.main.is_fs_type_fsfs(): check_hotcopy_fsfs(sbox.repo_dir, backup_dir) - else: + if svntest.main.is_fs_type_bdb(): check_hotcopy_bdb(sbox.repo_dir, backup_dir) + if svntest.main.is_fs_type_fsx(): + check_hotcopy_fsx(sbox.repo_dir, backup_dir) #---------------------------------------------------------------------- @@ -549,9 +558,17 @@ if errput: raise SVNUnexpectedStderr(errput) - # unfortunately, FSFS needs to do more checks than BDB resulting in - # different progress output - if svntest.main.is_fs_type_fsfs(): + # unfortunately, some backends needs to do more checks than other + # resulting in different progress output + if svntest.main.is_fs_type_fsx(): + svntest.verify.compare_and_display_lines( + "Error while running 'svnadmin verify'.", + 'STDERR', ["* Verifying metadata at revision 0 ...\n", + "* Verifying repository metadata ...\n", + "* Verified revision 0.\n", + "* Verified revision 1.\n", + "* Verified revision 2.\n"], output) + elif svntest.main.is_fs_type_fsfs(): svntest.verify.compare_and_display_lines( "Error while running 'svnadmin verify'.", 'STDERR', ["* Verifying repository metadata ...\n",
diff --git a/subversion/tests/cmdline/svntest/main.py b/subversion/tests/cmdline/svntest/main.py index 0b5ecf7..057667b 100644 --- a/subversion/tests/cmdline/svntest/main.py +++ b/subversion/tests/cmdline/svntest/main.py
@@ -1330,6 +1330,9 @@ # This assumes that fsfs is the default fs implementation. return options.fs_type == 'fsfs' or options.fs_type is None +def is_fs_type_fsx(): + return options.fs_type == 'fsx' + def is_fs_type_bdb(): return options.fs_type == 'bdb' @@ -1753,7 +1756,7 @@ parser.add_option('--url', action='store', help='Base url to the repos (e.g. svn://localhost)') parser.add_option('--fs-type', action='store', - help='Subversion file system type (fsfs or bdb)') + help='Subversion file system type (fsfs, bdb or fsx)') parser.add_option('--cleanup', action='store_true', help='Whether to clean up') parser.add_option('--enable-sasl', action='store_true',
diff --git a/subversion/tests/libsvn_fs/fs-test.c b/subversion/tests/libsvn_fs/fs-test.c index a77fe5d..db14caa 100644 --- a/subversion/tests/libsvn_fs/fs-test.c +++ b/subversion/tests/libsvn_fs/fs-test.c
@@ -5049,15 +5049,27 @@ int fs_format; svn_version_t *supports_version; svn_version_t v1_5_0 = {1, 5, 0, ""}; + svn_version_t v1_9_0 = {1, 9, 0, ""}; svn_test_opts_t opts2; + svn_boolean_t is_fsx = strcmp(opts->fs_type, "fsx") == 0; opts2 = *opts; - opts2.server_minor_version = 5; + opts2.server_minor_version = is_fsx ? 9 : 5; SVN_ERR(svn_test__create_fs(&fs, "test-fs-format-info", &opts2, pool)); SVN_ERR(svn_fs_info_format(&fs_format, &supports_version, fs, pool, pool)); - SVN_TEST_ASSERT(fs_format == 3); /* happens to be the same for FSFS and BDB */ - SVN_TEST_ASSERT(svn_ver_equal(supports_version, &v1_5_0)); + + if (is_fsx) + { + SVN_TEST_ASSERT(fs_format == 1); + SVN_TEST_ASSERT(svn_ver_equal(supports_version, &v1_9_0)); + } + else + { + /* happens to be the same for FSFS and BDB */ + SVN_TEST_ASSERT(fs_format == 3); + SVN_TEST_ASSERT(svn_ver_equal(supports_version, &v1_5_0)); + } return SVN_NO_ERROR; }
diff --git a/subversion/tests/libsvn_fs_fs/fs-pack-test.c b/subversion/tests/libsvn_fs_fs/fs-fs-pack-test.c similarity index 99% rename from subversion/tests/libsvn_fs_fs/fs-pack-test.c rename to subversion/tests/libsvn_fs_fs/fs-fs-pack-test.c index 781ca9b..20a3a92 100644 --- a/subversion/tests/libsvn_fs_fs/fs-pack-test.c +++ b/subversion/tests/libsvn_fs_fs/fs-fs-pack-test.c
@@ -795,6 +795,11 @@ const svn_fs_fsfs_info_t *fsfs_info; const svn_fs_info_placeholder_t *info; + /* Bail (with success) on known-untestable scenarios */ + if ((strcmp(opts->fs_type, "fsfs") != 0) + || (opts->server_minor_version && (opts->server_minor_version < 8))) + return SVN_NO_ERROR; + SVN_ERR(create_packed_filesystem(REPO_NAME, opts, MAX_REV, SHARD_SIZE, pool));
diff --git a/subversion/tests/libsvn_fs_fs/fs-pack-test.c b/subversion/tests/libsvn_fs_x/fs-x-pack-test.c similarity index 85% copy from subversion/tests/libsvn_fs_fs/fs-pack-test.c copy to subversion/tests/libsvn_fs_x/fs-x-pack-test.c index 781ca9b..5c300c9 100644 --- a/subversion/tests/libsvn_fs_fs/fs-pack-test.c +++ b/subversion/tests/libsvn_fs_x/fs-x-pack-test.c
@@ -25,7 +25,8 @@ #include <apr_pools.h> #include "../svn_test.h" -#include "../../libsvn_fs_fs/fs.h" +#include "../../libsvn_fs_x/fs.h" +#include "../../libsvn_fs_x/reps.h" #include "svn_pools.h" #include "svn_props.h" @@ -52,24 +53,12 @@ const char *contents; path = svn_dirent_join(path, "format", pool); + SVN_TEST_ASSERT(max_files_per_dir > 0); - if (format >= SVN_FS_FS__MIN_LAYOUT_FORMAT_OPTION_FORMAT) - { - if (max_files_per_dir) - contents = apr_psprintf(pool, - "%d\n" - "layout sharded %d\n", - format, max_files_per_dir); - else - contents = apr_psprintf(pool, - "%d\n" - "layout linear", - format); - } - else - { - contents = apr_psprintf(pool, "%d\n", format); - } + contents = apr_psprintf(pool, + "%d\n" + "layout sharded %d\n", + format, max_files_per_dir); SVN_ERR(svn_io_write_atomic(path, contents, strlen(contents), NULL /* copy perms */, pool)); @@ -271,7 +260,7 @@ /*** Tests ***/ /* ------------------------------------------------------------------------ */ -#define REPO_NAME "test-repo-fsfs-pack" +#define REPO_NAME "test-repo-fsx-pack" #define SHARD_SIZE 7 #define MAX_REV 53 static svn_error_t * @@ -286,7 +275,7 @@ apr_size_t len; /* Bail (with success) on known-untestable scenarios */ - if ((strcmp(opts->fs_type, "fsfs") != 0) + if ((strcmp(opts->fs_type, "fsx") != 0) || (opts->server_minor_version && (opts->server_minor_version < 6))) return SVN_NO_ERROR; @@ -307,14 +296,37 @@ return svn_error_createf(SVN_ERR_FS_GENERAL, NULL, "Expected pack file '%s' not found", path); - path = svn_dirent_join_many(pool, REPO_NAME, "revs", - apr_psprintf(pool, "%d.pack", i / SHARD_SIZE), - "manifest", NULL); - SVN_ERR(svn_io_check_path(path, &kind, pool)); - if (kind != svn_node_file) - return svn_error_createf(SVN_ERR_FS_GENERAL, NULL, - "Expected manifest file '%s' not found", - path); + if (opts->server_minor_version && (opts->server_minor_version < 6)) + { + path = svn_dirent_join_many(pool, REPO_NAME, "revs", + apr_psprintf(pool, "%d.pack", i / SHARD_SIZE), + "manifest", NULL); + SVN_ERR(svn_io_check_path(path, &kind, pool)); + if (kind != svn_node_file) + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL, + "Expected manifest file '%s' not found", + path); + } + else + { + path = svn_dirent_join_many(pool, REPO_NAME, "revs", + apr_psprintf(pool, "%d.pack", i / SHARD_SIZE), + "pack.l2p", NULL); + SVN_ERR(svn_io_check_path(path, &kind, pool)); + if (kind != svn_node_file) + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL, + "Expected log-to-phys index file '%s' not found", + path); + + path = svn_dirent_join_many(pool, REPO_NAME, "revs", + apr_psprintf(pool, "%d.pack", i / SHARD_SIZE), + "pack.p2l", NULL); + SVN_ERR(svn_io_check_path(path, &kind, pool)); + if (kind != svn_node_file) + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL, + "Expected phys-to-log index file '%s' not found", + path); + } /* This directory should not exist. */ path = svn_dirent_join_many(pool, REPO_NAME, "revs", @@ -355,7 +367,7 @@ #undef MAX_REV /* ------------------------------------------------------------------------ */ -#define REPO_NAME "test-repo-fsfs-pack-even" +#define REPO_NAME "test-repo-fsx-pack-even" #define SHARD_SIZE 4 #define MAX_REV 11 static svn_error_t * @@ -366,7 +378,7 @@ const char *path; /* Bail (with success) on known-untestable scenarios */ - if ((strcmp(opts->fs_type, "fsfs") != 0) + if ((strcmp(opts->fs_type, "fsx") != 0) || (opts->server_minor_version && (opts->server_minor_version < 6))) return SVN_NO_ERROR; @@ -399,7 +411,7 @@ svn_revnum_t i; /* Bail (with success) on known-untestable scenarios */ - if ((strcmp(opts->fs_type, "fsfs") != 0) + if ((strcmp(opts->fs_type, "fsx") != 0) || (opts->server_minor_version && (opts->server_minor_version < 6))) return SVN_NO_ERROR; @@ -446,7 +458,7 @@ svn_revnum_t after_rev; /* Bail (with success) on known-untestable scenarios */ - if ((strcmp(opts->fs_type, "fsfs") != 0) + if ((strcmp(opts->fs_type, "fsx") != 0) || (opts->server_minor_version && (opts->server_minor_version < 6))) return SVN_NO_ERROR; @@ -481,7 +493,7 @@ svn_string_t *prop_value; /* Bail (with success) on known-untestable scenarios */ - if ((strcmp(opts->fs_type, "fsfs") != 0) + if ((strcmp(opts->fs_type, "fsx") != 0) || (opts->server_minor_version && (opts->server_minor_version < 7))) return SVN_NO_ERROR; @@ -538,7 +550,7 @@ svn_revnum_t rev; /* Bail (with success) on known-untestable scenarios */ - if ((strcmp(opts->fs_type, "fsfs") != 0) + if ((strcmp(opts->fs_type, "fsx") != 0) || (opts->server_minor_version && (opts->server_minor_version < 7))) return SVN_NO_ERROR; @@ -611,7 +623,7 @@ svn_revnum_t rev; /* Bail (with success) on known-untestable scenarios */ - if ((strcmp(opts->fs_type, "fsfs") != 0) + if ((strcmp(opts->fs_type, "fsx") != 0) || (opts->server_minor_version && (opts->server_minor_version < 7))) return SVN_NO_ERROR; @@ -684,7 +696,7 @@ svn_error_t *err; /* Bail (with success) on known-untestable scenarios */ - if ((strcmp(opts->fs_type, "fsfs") != 0) + if ((strcmp(opts->fs_type, "fsx") != 0) || (opts->server_minor_version && (opts->server_minor_version < 7))) return SVN_NO_ERROR; @@ -747,7 +759,7 @@ svn_error_t *err = SVN_NO_ERROR; /* Bail (with success) on known-untestable scenarios */ - if ((strcmp(opts->fs_type, "fsfs") != 0) + if ((strcmp(opts->fs_type, "fsx") != 0) || (opts->server_minor_version && (opts->server_minor_version < 8))) return SVN_NO_ERROR; @@ -784,7 +796,7 @@ #undef SHARD_SIZE /* ------------------------------------------------------------------------ */ -#define REPO_NAME "test-repo-fsfs-info" +#define REPO_NAME "test-repo-fsx-info" #define SHARD_SIZE 3 #define MAX_REV 5 static svn_error_t * @@ -795,6 +807,11 @@ const svn_fs_fsfs_info_t *fsfs_info; const svn_fs_info_placeholder_t *info; + /* Bail (with success) on known-untestable scenarios */ + if ((strcmp(opts->fs_type, "fsx") != 0) + || (opts->server_minor_version && (opts->server_minor_version < 8))) + return SVN_NO_ERROR; + SVN_ERR(create_packed_filesystem(REPO_NAME, opts, MAX_REV, SHARD_SIZE, pool)); @@ -805,7 +822,7 @@ SVN_TEST_STRING_ASSERT(opts->fs_type, info->fs_type); /* Bail (with success) on known-untestable scenarios */ - if (strcmp(opts->fs_type, "fsfs") != 0) + if (strcmp(opts->fs_type, "fsx") != 0) return SVN_NO_ERROR; fsfs_info = (const void *)info; @@ -828,7 +845,62 @@ #undef MAX_REV /* ------------------------------------------------------------------------ */ -#define REPO_NAME "test-repo-fsfs-pack-shard-size-one" +#define REPO_NAME "test-repo-fsx-rev-container" +#define SHARD_SIZE 3 +#define MAX_REV 5 +static svn_error_t * +test_reps(const svn_test_opts_t *opts, + apr_pool_t *pool) +{ + svn_fs_t *fs = NULL; + svn_fs_x__reps_builder_t *builder; + svn_fs_x__reps_t *container; + svn_stringbuf_t *serialized; + svn_stream_t *stream; + svn_stringbuf_t *contents = svn_stringbuf_create_ensure(10000, pool); + + int i; + for (i = 0; i < 10000; ++i) + { + int v, s = 0; + for (v = i; v > 0; v /= 10) + s += v % 10; + + svn_stringbuf_appendbyte(contents, (char)(s + ' ')); + } + + SVN_ERR(create_packed_filesystem(REPO_NAME, opts, MAX_REV, SHARD_SIZE, + pool)); + + SVN_ERR(svn_fs_open(&fs, REPO_NAME, NULL, pool)); + + builder = svn_fs_x__reps_builder_create(fs, pool); + for (i = 10000; i > 10; --i) + { + svn_string_t string; + string.data = contents->data; + string.len = i; + + svn_fs_x__reps_add(builder, &string); + } + + serialized = svn_stringbuf_create_empty(pool); + stream = svn_stream_from_stringbuf(serialized, pool); + SVN_ERR(svn_fs_x__write_reps_container(stream, builder, pool)); + + SVN_ERR(svn_stream_reset(stream)); + SVN_ERR(svn_fs_x__read_reps_container(&container, stream, pool, pool)); + SVN_ERR(svn_stream_close(stream)); + + return SVN_NO_ERROR; +} + +#undef REPO_NAME +#undef SHARD_SIZE +#undef MAX_REV + +/* ------------------------------------------------------------------------ */ +#define REPO_NAME "test-repo-fsx-pack-shard-size-one" #define SHARD_SIZE 1 #define MAX_REV 4 static svn_error_t * @@ -839,7 +911,7 @@ svn_fs_t *fs; /* Bail (with success) on known-untestable scenarios */ - if ((strcmp(opts->fs_type, "fsfs") != 0) + if ((strcmp(opts->fs_type, "fsx") != 0) || (opts->server_minor_version && (opts->server_minor_version < 6))) return SVN_NO_ERROR; @@ -864,25 +936,27 @@ { SVN_TEST_NULL, SVN_TEST_OPTS_PASS(pack_filesystem, - "pack a FSFS filesystem"), + "pack a FSX filesystem"), SVN_TEST_OPTS_PASS(pack_even_filesystem, - "pack FSFS where revs % shard = 0"), + "pack FSX where revs % shard = 0"), SVN_TEST_OPTS_PASS(read_packed_fs, - "read from a packed FSFS filesystem"), + "read from a packed FSX filesystem"), SVN_TEST_OPTS_PASS(commit_packed_fs, - "commit to a packed FSFS filesystem"), + "commit to a packed FSX filesystem"), SVN_TEST_OPTS_PASS(get_set_revprop_packed_fs, - "get/set revprop while packing FSFS filesystem"), + "get/set revprop while packing FSX filesystem"), SVN_TEST_OPTS_PASS(get_set_large_revprop_packed_fs, - "get/set large packed revprops in FSFS"), + "get/set large packed revprops in FSX"), SVN_TEST_OPTS_PASS(get_set_huge_revprop_packed_fs, - "get/set huge packed revprops in FSFS"), + "get/set huge packed revprops in FSX"), SVN_TEST_OPTS_PASS(recover_fully_packed, "recover a fully packed filesystem"), SVN_TEST_OPTS_PASS(file_hint_at_shard_boundary, "test file hint at shard boundary"), SVN_TEST_OPTS_PASS(test_info, "test svn_fs_info"), + SVN_TEST_OPTS_PASS(test_reps, + "test representations container"), SVN_TEST_OPTS_PASS(pack_shard_size_one, "test packing with shard size = 1"), SVN_TEST_NULL
diff --git a/subversion/tests/libsvn_fs_x/string-table-test.c b/subversion/tests/libsvn_fs_x/string-table-test.c new file mode 100644 index 0000000..a9aff17 --- /dev/null +++ b/subversion/tests/libsvn_fs_x/string-table-test.c
@@ -0,0 +1,311 @@ +/* string-table-test.c --- tests for string tables + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "../svn_test.h" +#include "../../libsvn_fs_x/string_table.h" +#include "svn_pools.h" +#include "svn_sorts.h" + +/* Some tests use this list of strings as is. They are all "short strings" + * in the terminology of string tables. We use them also as an input to + * generate strings of arbitrary length. + */ +enum { STRING_COUNT = 12 }; +const char *basic_strings[STRING_COUNT] = + { + "some string", + "this is another string", + "this is a duplicate", + "some longer string", + "this is a very long string", + "and here is another", + "this is a duplicate", + "/some/path/to/a/dir", + "/some/path/to/a/file", + "/some/other/dir", + "/some/other/file", + "" + }; + +/* Generate a string of exactly LEN chars (plus terminating NUL). KEY is + * an arbitrary integer that will be transformed into a character sequence + * using entries of BASIC_STRINGS. The result will be allocated in POOL. + */ +static svn_stringbuf_t * +generate_string(apr_uint64_t key, apr_size_t len, apr_pool_t *pool) +{ + svn_stringbuf_t *result = svn_stringbuf_create_ensure(len, pool); + apr_uint64_t temp = key; + apr_uint64_t run = 0; + + while (len) + { + apr_size_t idx; + apr_size_t add_len; + + if (temp == 0) + { + temp = key; + run++; + } + + idx = (temp + run) % STRING_COUNT; + temp /= STRING_COUNT; + + add_len = strlen(basic_strings[idx]); + add_len = MIN(len, add_len); + + svn_stringbuf_appendbytes(result, basic_strings[idx], add_len); + len -= add_len; + } + + return result; +} + +static svn_error_t * +store_and_load_table(string_table_t **table, apr_pool_t *pool) +{ + svn_stringbuf_t *stream_buffer = svn_stringbuf_create_empty(pool); + svn_stream_t *stream; + + stream = svn_stream_from_stringbuf(stream_buffer, pool); + SVN_ERR(svn_fs_x__write_string_table(stream, *table, pool)); + SVN_ERR(svn_stream_close(stream)); + + *table = NULL; + + stream = svn_stream_from_stringbuf(stream_buffer, pool); + SVN_ERR(svn_fs_x__read_string_table(table, stream, pool, pool)); + SVN_ERR(svn_stream_close(stream)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +create_empty_table_body(svn_boolean_t do_load_store, + apr_pool_t *pool) +{ + string_table_builder_t *builder + = svn_fs_x__string_table_builder_create(pool); + string_table_t *table + = svn_fs_x__string_table_create(builder, pool); + + SVN_TEST_STRING_ASSERT(svn_fs_x__string_table_get(table, 0, NULL, pool), ""); + + if (do_load_store) + SVN_ERR(store_and_load_table(&table, pool)); + + SVN_TEST_STRING_ASSERT(svn_fs_x__string_table_get(table, 0, NULL, pool), ""); + + return SVN_NO_ERROR; +} + +static svn_error_t * +short_string_table_body(svn_boolean_t do_load_store, + apr_pool_t *pool) +{ + apr_size_t indexes[STRING_COUNT] = { 0 }; + + string_table_builder_t *builder; + string_table_t *table; + int i; + + builder = svn_fs_x__string_table_builder_create(pool); + for (i = 0; i < STRING_COUNT; ++i) + indexes[i] = svn_fs_x__string_table_builder_add(builder, basic_strings[i], 0); + + table = svn_fs_x__string_table_create(builder, pool); + if (do_load_store) + SVN_ERR(store_and_load_table(&table, pool)); + + SVN_TEST_ASSERT(indexes[2] == indexes[6]); + for (i = 0; i < STRING_COUNT; ++i) + { + apr_size_t len; + const char *string + = svn_fs_x__string_table_get(table, indexes[i], &len, pool); + + SVN_TEST_STRING_ASSERT(string, basic_strings[i]); + SVN_TEST_ASSERT(len == strlen(string)); + SVN_TEST_ASSERT(len == strlen(basic_strings[i])); + } + + SVN_TEST_STRING_ASSERT(svn_fs_x__string_table_get(table, STRING_COUNT, + NULL, pool), ""); + + return SVN_NO_ERROR; +} + +static svn_error_t * +large_string_table_body(svn_boolean_t do_load_store, + apr_pool_t *pool) +{ + enum { COUNT = 10 }; + + svn_stringbuf_t *strings[COUNT] = { 0 }; + apr_size_t indexes[COUNT] = { 0 }; + + string_table_builder_t *builder; + string_table_t *table; + int i; + + builder = svn_fs_x__string_table_builder_create(pool); + for (i = 0; i < COUNT; ++i) + { + strings[i] = generate_string(0x1234567876543210ull * (i + 1), 73000 + 1000 * i, pool); + indexes[i] = svn_fs_x__string_table_builder_add(builder, strings[i]->data, strings[i]->len); + } + + table = svn_fs_x__string_table_create(builder, pool); + if (do_load_store) + SVN_ERR(store_and_load_table(&table, pool)); + + for (i = 0; i < COUNT; ++i) + { + apr_size_t len; + const char *string + = svn_fs_x__string_table_get(table, indexes[i], &len, pool); + + SVN_TEST_STRING_ASSERT(string, strings[i]->data); + SVN_TEST_ASSERT(len == strlen(string)); + SVN_TEST_ASSERT(len == strings[i]->len); + } + + return SVN_NO_ERROR; +} + +static svn_error_t * +many_strings_table_body(svn_boolean_t do_load_store, + apr_pool_t *pool) +{ + /* cause multiple sub-tables to be created */ + enum { COUNT = 1000 }; + + svn_stringbuf_t *strings[COUNT] = { 0 }; + apr_size_t indexes[COUNT] = { 0 }; + + string_table_builder_t *builder; + string_table_t *table; + int i; + + builder = svn_fs_x__string_table_builder_create(pool); + for (i = 0; i < COUNT; ++i) + { + strings[i] = generate_string(0x1234567876543210ull * (i + 1), + (i * i) % 23000, pool); + indexes[i] = svn_fs_x__string_table_builder_add(builder, + strings[i]->data, + strings[i]->len); + } + + table = svn_fs_x__string_table_create(builder, pool); + if (do_load_store) + SVN_ERR(store_and_load_table(&table, pool)); + + for (i = 0; i < COUNT; ++i) + { + apr_size_t len; + const char *string + = svn_fs_x__string_table_get(table, indexes[i], &len, pool); + + SVN_TEST_STRING_ASSERT(string, strings[i]->data); + SVN_TEST_ASSERT(len == strlen(string)); + SVN_TEST_ASSERT(len == strings[i]->len); + } + + return SVN_NO_ERROR; +} + +static svn_error_t * +create_empty_table(apr_pool_t *pool) +{ + return svn_error_trace(create_empty_table_body(FALSE, pool)); +} + +static svn_error_t * +short_string_table(apr_pool_t *pool) +{ + return svn_error_trace(short_string_table_body(FALSE, pool)); +} + +static svn_error_t * +large_string_table(apr_pool_t *pool) +{ + return svn_error_trace(large_string_table_body(FALSE, pool)); +} + +static svn_error_t * +many_strings_table(apr_pool_t *pool) +{ + return svn_error_trace(many_strings_table_body(FALSE, pool)); +} + +static svn_error_t * +store_load_short_string_table(apr_pool_t *pool) +{ + return svn_error_trace(short_string_table_body(TRUE, pool)); +} + +static svn_error_t * +store_load_large_string_table(apr_pool_t *pool) +{ + return svn_error_trace(large_string_table_body(TRUE, pool)); +} + +static svn_error_t * +store_load_empty_table(apr_pool_t *pool) +{ + return svn_error_trace(create_empty_table_body(TRUE, pool)); +} + +static svn_error_t * +store_load_many_strings_table(apr_pool_t *pool) +{ + return svn_error_trace(many_strings_table_body(TRUE, pool)); +} + + +/* ------------------------------------------------------------------------ */ + +/* The test table. */ + +struct svn_test_descriptor_t test_funcs[] = + { + SVN_TEST_NULL, + SVN_TEST_PASS2(create_empty_table, + "create an empty string table"), + SVN_TEST_PASS2(short_string_table, + "string table with short strings only"), + SVN_TEST_PASS2(large_string_table, + "string table with large strings only"), + SVN_TEST_PASS2(many_strings_table, + "string table with many strings"), + SVN_TEST_PASS2(store_load_empty_table, + "store and load an empty string table"), + SVN_TEST_PASS2(store_load_short_string_table, + "store and load table with short strings only"), + SVN_TEST_PASS2(store_load_large_string_table, + "store and load table with large strings only"), + SVN_TEST_PASS2(store_load_many_strings_table, + "store and load string table with many strings"), + SVN_TEST_NULL + };
diff --git a/subversion/tests/libsvn_repos/repos-test.c b/subversion/tests/libsvn_repos/repos-test.c index f9513f9..4614b0a 100644 --- a/subversion/tests/libsvn_repos/repos-test.c +++ b/subversion/tests/libsvn_repos/repos-test.c
@@ -3271,21 +3271,26 @@ svn_version_t v1_0_0 = {1, 0, 0, ""}; svn_version_t v1_4_0 = {1, 4, 0, ""}; int repos_format; + svn_boolean_t is_fsx = strcmp(opts->fs_type, "fsx") == 0; opts2 = *opts; - opts2.server_minor_version = 3; - SVN_ERR(svn_test__create_repos(&repos, "test-repo-info-3", - &opts2, pool)); - SVN_ERR(svn_repos_capabilities(&capabilities, repos, pool, pool)); - SVN_TEST_ASSERT(apr_hash_count(capabilities) == 0); - SVN_ERR(svn_repos_info_format(&repos_format, &supports_version, repos, - pool, pool)); - SVN_TEST_ASSERT(repos_format == 3); - SVN_TEST_ASSERT(svn_ver_equal(supports_version, &v1_0_0)); + /* for repo types that have been around before 1.4 */ + if (!is_fsx) + { + opts2.server_minor_version = 3; + SVN_ERR(svn_test__create_repos(&repos, "test-repo-info-3", + &opts2, pool)); + SVN_ERR(svn_repos_capabilities(&capabilities, repos, pool, pool)); + SVN_TEST_ASSERT(apr_hash_count(capabilities) == 0); + SVN_ERR(svn_repos_info_format(&repos_format, &supports_version, repos, + pool, pool)); + SVN_TEST_ASSERT(repos_format == 3); + SVN_TEST_ASSERT(svn_ver_equal(supports_version, &v1_0_0)); + } - opts2.server_minor_version = 8; - SVN_ERR(svn_test__create_repos(&repos, "test-repo-info-8", + opts2.server_minor_version = 9; + SVN_ERR(svn_test__create_repos(&repos, "test-repo-info-9", &opts2, pool)); SVN_ERR(svn_repos_capabilities(&capabilities, repos, pool, pool)); SVN_TEST_ASSERT(apr_hash_count(capabilities) == 1);
diff --git a/subversion/tests/libsvn_subr/packed-data-test.c b/subversion/tests/libsvn_subr/packed-data-test.c new file mode 100644 index 0000000..f7be474 --- /dev/null +++ b/subversion/tests/libsvn_subr/packed-data-test.c
@@ -0,0 +1,567 @@ +/* + * packed-data-test.c: a collection of svn_packed__* tests + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +/* ==================================================================== + To add tests, look toward the bottom of this file. + +*/ + + + +#include <stdio.h> +#include <string.h> +#include <apr_pools.h> +#include <apr_poll.h> + +#include "../svn_test.h" + +#include "svn_error.h" +#include "svn_string.h" /* This includes <apr_*.h> */ +#include "private/svn_packed_data.h" + +/* Take the WRITE_ROOT, serialize its contents, parse it again into a new + * data root and return it in *READ_ROOT. Allocate it in POOL. + */ +static svn_error_t* +get_read_root(svn_packed__data_root_t **read_root, + svn_packed__data_root_t *write_root, + apr_pool_t *pool) +{ + svn_stringbuf_t *stream_buffer = svn_stringbuf_create_empty(pool); + svn_stream_t *stream; + + stream = svn_stream_from_stringbuf(stream_buffer, pool); + SVN_ERR(svn_packed__data_write(stream, write_root, pool)); + SVN_ERR(svn_stream_close(stream)); + + stream = svn_stream_from_stringbuf(stream_buffer, pool); + SVN_ERR(svn_packed__data_read(read_root, stream, pool, pool)); + SVN_ERR(svn_stream_close(stream)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +test_empty_container(apr_pool_t *pool) +{ + /* create an empty, readable container */ + svn_packed__data_root_t *root = svn_packed__data_create_root(pool); + SVN_ERR(get_read_root(&root, root, pool)); + + /* there should be no sub-streams */ + SVN_TEST_ASSERT(svn_packed__first_int_stream(root) == NULL); + SVN_TEST_ASSERT(svn_packed__first_byte_stream(root) == NULL); + + return SVN_NO_ERROR; +} + +/* Check that COUNT numbers from VALUES can be written as uints to a + * packed data stream and can be read from that stream again. Deltify + * data in the stream if DIFF is set. Use POOL for allocations. + */ +static svn_error_t * +verify_uint_stream(const apr_uint64_t *values, + apr_size_t count, + svn_boolean_t diff, + apr_pool_t *pool) +{ + svn_packed__data_root_t *root = svn_packed__data_create_root(pool); + svn_packed__int_stream_t *stream + = svn_packed__create_int_stream(root, diff, FALSE); + + apr_size_t i; + for (i = 0; i < count; ++i) + svn_packed__add_uint(stream, values[i]); + + SVN_ERR(get_read_root(&root, root, pool)); + + /* the container should contain exactly one int stream */ + stream = svn_packed__first_int_stream(root); + SVN_TEST_ASSERT(stream); + SVN_TEST_ASSERT(!svn_packed__next_int_stream(stream)); + SVN_TEST_ASSERT(!svn_packed__first_byte_stream(root)); + + /* the stream shall contain exactly the items we put into it */ + SVN_TEST_ASSERT(svn_packed__int_count(stream) == count); + for (i = 0; i < count; ++i) + SVN_TEST_ASSERT(svn_packed__get_uint(stream) == values[i]); + + /* reading beyond eos should return 0 values */ + SVN_TEST_ASSERT(svn_packed__get_uint(stream) == 0); + + return SVN_NO_ERROR; +} + +static svn_error_t * +test_uint_stream(apr_pool_t *pool) +{ + enum { COUNT = 8 }; + const apr_uint64_t values[COUNT] = + { + 0xffffffffffffffffull, + 0, + 0xffffffffffffffffull, + 0x8000000000000000ull, + 0, + 0x7fffffffffffffffull, + 0x1234567890abcdefull, + 0x0fedcba987654321ull, + }; + + SVN_ERR(verify_uint_stream(values, COUNT, FALSE, pool)); + SVN_ERR(verify_uint_stream(values, COUNT, TRUE, pool)); + + return SVN_NO_ERROR; +} + +/* Check that COUNT numbers from VALUES can be written as signed ints to a + * packed data stream and can be read from that stream again. Deltify + * data in the stream if DIFF is set. Use POOL for allocations. + */ +static svn_error_t * +verify_int_stream(const apr_int64_t *values, + apr_size_t count, + svn_boolean_t diff, + apr_pool_t *pool) +{ + svn_packed__data_root_t *root = svn_packed__data_create_root(pool); + svn_packed__int_stream_t *stream + = svn_packed__create_int_stream(root, diff, TRUE); + + apr_size_t i; + for (i = 0; i < count; ++i) + svn_packed__add_int(stream, values[i]); + + SVN_ERR(get_read_root(&root, root, pool)); + + /* the container should contain exactly one int stream */ + stream = svn_packed__first_int_stream(root); + SVN_TEST_ASSERT(stream); + SVN_TEST_ASSERT(!svn_packed__next_int_stream(stream)); + SVN_TEST_ASSERT(!svn_packed__first_byte_stream(root)); + + /* the stream shall contain exactly the items we put into it */ + SVN_TEST_ASSERT(svn_packed__int_count(stream) == count); + for (i = 0; i < count; ++i) + SVN_TEST_ASSERT(svn_packed__get_int(stream) == values[i]); + + /* reading beyond eos should return 0 values */ + SVN_TEST_ASSERT(svn_packed__get_int(stream) == 0); + + return SVN_NO_ERROR; +} + +static svn_error_t * +test_int_stream(apr_pool_t *pool) +{ + enum { COUNT = 7 }; + const apr_int64_t values[COUNT] = + { + 0x7fffffffffffffffll, + -0x8000000000000000ll, + 0, + 0x7fffffffffffffffll, + -0x7fffffffffffffffll, + 0x1234567890abcdefll, + -0x0fedcba987654321ll, + }; + + SVN_ERR(verify_int_stream(values, COUNT, FALSE, pool)); + SVN_ERR(verify_int_stream(values, COUNT, TRUE, pool)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +test_byte_stream(apr_pool_t *pool) +{ + enum { COUNT = 6 }; + const svn_string_t values[COUNT] = + { + { "", 0 }, + { "\0", 1 }, + { "\0", 1 }, + { "some text", 9 }, + { "", 0 }, + { "some more", 9 } + }; + + svn_packed__data_root_t *root = svn_packed__data_create_root(pool); + svn_packed__byte_stream_t *stream + = svn_packed__create_bytes_stream(root); + + apr_size_t i; + for (i = 0; i < COUNT; ++i) + svn_packed__add_bytes(stream, values[i].data, values[i].len); + + SVN_ERR(get_read_root(&root, root, pool)); + + /* the container should contain exactly one byte stream */ + stream = svn_packed__first_byte_stream(root); + SVN_TEST_ASSERT(stream); + SVN_TEST_ASSERT(!svn_packed__next_byte_stream(stream)); + + /* the stream shall contain exactly the items we put into it */ + SVN_TEST_ASSERT(svn_packed__byte_count(stream) == 20); + for (i = 0; i < COUNT; ++i) + { + svn_string_t string; + string.data = svn_packed__get_bytes(stream, &string.len); + + SVN_TEST_ASSERT(string.len == values[i].len); + SVN_TEST_ASSERT(!memcmp(string.data, values[i].data, string.len)); + } + + /* reading beyond eos should return 0 values */ + SVN_TEST_ASSERT(svn_packed__byte_count(stream) == 0); + + return SVN_NO_ERROR; +} + +/* Some simple structure that we use as sub-structure to BASE_RECORD_T. + * Have it contain numbers and strings. + */ +typedef struct sub_record_t +{ + int sub_counter; + svn_string_t text; +} sub_record_t; + +/* signed / unsigned, 64 bits and shorter, diff-able and not, multiple + * strings, multiple sub-records. */ +typedef struct base_record_t +{ + int counter; + svn_string_t description; + apr_uint64_t large_unsigned1; + apr_uint64_t large_unsigned2; + const sub_record_t *left_subs; + apr_int64_t large_signed1; + apr_int64_t large_signed2; + unsigned prime; + const sub_record_t *right_subs; + svn_string_t binary; +} base_record_t; + +/* our test data */ +enum {SUB_RECORD_COUNT = 7}; +enum {BASE_RECORD_COUNT = 4}; + +const sub_record_t sub_records[SUB_RECORD_COUNT] = +{ + { 6, { "this is quite a longish piece of text", 37} }, + { 5, { "x", 1} }, + { 4, { "not empty", 9} }, + { 3, { "another bit of text", 19} }, + { 2, { "", 0} }, + { 1, { "first sub-record", 16} }, + { 0 } +}; + +const base_record_t test_data[BASE_RECORD_COUNT] = +{ + { 1, { "maximum", 7}, + 0xffffffffffffffffull, 0xffffffffffffffffull, sub_records, + 0x7fffffffffffffffll, 0x7fffffffffffffffll, 9967, sub_records + 1, + { "\0\1\2\3\4\5\6\7\x8\x9\xa", 11} }, + { 2, { "minimum", 7}, + 0, 0, sub_records + 6, + -0x8000000000000000ll, -0x8000000000000000ll, 6029, sub_records + 5, + { "X\0\0Y", 4} }, + { 3, { "mean", 4}, + 0x8000000000000000ull, 0x8000000000000000ull, sub_records + 2, + 0, 0, 653, sub_records + 3, + { "\xff\0\1\2\3\4\5\6\7\x8\x9\xa", 12} }, + { 4, { "random", 6}, + 0x1234567890abcdefull, 0xfedcba987654321ull, sub_records + 4, + 0x1234567890abcdll, -0xedcba987654321ll, 7309, sub_records + 1, + { "\x80\x7f\0\1\6", 5} } +}; + +/* Serialize RECORDS into INT_STREAM and TEXT_STREAM. Stop when the + * current record's SUB_COUNTER is 0. + */ +static unsigned +pack_subs(svn_packed__int_stream_t *int_stream, + svn_packed__byte_stream_t *text_stream, + const sub_record_t *records) +{ + unsigned count; + for (count = 0; records[count].sub_counter; ++count) + { + svn_packed__add_int(int_stream, records[count].sub_counter); + svn_packed__add_bytes(text_stream, + records[count].text.data, + records[count].text.len); + } + + return count; +} + +/* Serialize COUNT records starting from DATA into a packed data container + * allocated in POOL and return the container root. + */ +static svn_packed__data_root_t * +pack(const base_record_t *data, + apr_size_t count, + apr_pool_t *pool) +{ + apr_size_t i; + svn_packed__data_root_t *root = svn_packed__data_create_root(pool); + svn_packed__int_stream_t *base_stream + = svn_packed__create_int_stream(root, FALSE, FALSE); + svn_packed__int_stream_t *sub_count_stream + = svn_packed__create_int_stream(root, TRUE, FALSE); + + svn_packed__int_stream_t *left_sub_stream + = svn_packed__create_int_stream(root, FALSE, TRUE); + svn_packed__int_stream_t *right_sub_stream + = svn_packed__create_int_stream(root, FALSE, TRUE); + + svn_packed__byte_stream_t *base_description_stream + = svn_packed__create_bytes_stream(root); + svn_packed__byte_stream_t *base_binary_stream + = svn_packed__create_bytes_stream(root); + svn_packed__byte_stream_t *sub_text_stream + = svn_packed__create_bytes_stream(root); + + svn_packed__create_int_substream(base_stream, TRUE, TRUE); /* counter */ + svn_packed__create_int_substream(base_stream, TRUE, FALSE); /* large_unsigned1 */ + svn_packed__create_int_substream(base_stream, FALSE, FALSE); /* large_unsigned2 */ + svn_packed__create_int_substream(base_stream, TRUE, TRUE); /* large_signed1 */ + svn_packed__create_int_substream(base_stream, FALSE, TRUE); /* large_signed2 */ + svn_packed__create_int_substream(base_stream, TRUE, FALSE); /* prime */ + + for (i = 0; i < count; ++i) + { + svn_packed__add_int(base_stream, data[i].counter); + svn_packed__add_bytes(base_description_stream, + data[i].description.data, + data[i].description.len); + svn_packed__add_uint(base_stream, data[i].large_unsigned1); + svn_packed__add_uint(base_stream, data[i].large_unsigned2); + svn_packed__add_uint(sub_count_stream, + pack_subs(left_sub_stream, sub_text_stream, + data[i].left_subs)); + + svn_packed__add_int(base_stream, data[i].large_signed1); + svn_packed__add_int(base_stream, data[i].large_signed2); + svn_packed__add_uint(base_stream, data[i].prime); + svn_packed__add_uint(sub_count_stream, + pack_subs(right_sub_stream, sub_text_stream, + data[i].right_subs)); + + svn_packed__add_bytes(base_binary_stream, + data[i].binary.data, + data[i].binary.len); + } + + return root; +} + +/* Deserialize COUNT records from INT_STREAM and TEXT_STREAM and return + * the result allocated in POOL. + */ +static sub_record_t * +unpack_subs(svn_packed__int_stream_t *int_stream, + svn_packed__byte_stream_t *text_stream, + apr_size_t count, + apr_pool_t *pool) +{ + sub_record_t *records = apr_pcalloc(pool, (count + 1) * sizeof(*records)); + + apr_size_t i; + for (i = 0; i < count; ++i) + { + records[i].sub_counter = svn_packed__get_int(int_stream); + records[i].text.data = svn_packed__get_bytes(text_stream, + &records[i].text.len); + } + + return records; +} + +/* Deserialize all records from the packed data container ROOT, allocate + * them in POOL and return them. Set *COUNT to the number of records read. + */ +static base_record_t * +unpack(apr_size_t *count, + svn_packed__data_root_t *root, + apr_pool_t *pool) +{ + svn_packed__int_stream_t *base_stream + = svn_packed__first_int_stream(root); + svn_packed__int_stream_t *sub_count_stream + = svn_packed__next_int_stream(base_stream); + svn_packed__byte_stream_t *base_description_stream + = svn_packed__first_byte_stream(root); + svn_packed__byte_stream_t *base_binary_stream + = svn_packed__next_byte_stream(base_description_stream); + svn_packed__byte_stream_t *sub_text_stream + = svn_packed__next_byte_stream(base_binary_stream); + + svn_packed__int_stream_t *left_sub_stream + = svn_packed__next_int_stream(sub_count_stream); + svn_packed__int_stream_t *right_sub_stream + = svn_packed__next_int_stream(left_sub_stream); + + apr_size_t i; + base_record_t *data; + *count = svn_packed__int_count(sub_count_stream) / 2; + data = apr_pcalloc(pool, *count * sizeof(*data)); + + for (i = 0; i < *count; ++i) + { + data[i].counter = svn_packed__get_int(base_stream); + data[i].description.data + = svn_packed__get_bytes(base_description_stream, + &data[i].description.len); + data[i].large_unsigned1 = svn_packed__get_uint(base_stream); + data[i].large_unsigned2 = svn_packed__get_uint(base_stream); + data[i].left_subs = unpack_subs(left_sub_stream, sub_text_stream, + (apr_size_t)svn_packed__get_uint(sub_count_stream), + pool); + + data[i].large_signed1 = svn_packed__get_int(base_stream); + data[i].large_signed2 = svn_packed__get_int(base_stream); + data[i].prime = svn_packed__get_uint(base_stream); + data[i].right_subs = unpack_subs(right_sub_stream, sub_text_stream, + (apr_size_t)svn_packed__get_uint(sub_count_stream), + pool); + + data[i].binary.data + = svn_packed__get_bytes(base_binary_stream, + &data[i].binary.len); + } + + return data; +} + +/* Assert that LHS and RHS contain the same binary data (i.e. don't test + * for a terminating NUL). + */ +static svn_error_t * +compare_binary(const svn_string_t *lhs, + const svn_string_t *rhs) +{ + SVN_TEST_ASSERT(lhs->len == rhs->len); + SVN_TEST_ASSERT(!memcmp(lhs->data, rhs->data, rhs->len)); + + return SVN_NO_ERROR; +} + +/* Assert that LHS and RHS contain the same number of records with the + * same contents. + */ +static svn_error_t * +compare_subs(const sub_record_t *lhs, + const sub_record_t *rhs) +{ + for (; lhs->sub_counter; ++lhs, ++rhs) + { + SVN_TEST_ASSERT(lhs->sub_counter == rhs->sub_counter); + SVN_ERR(compare_binary(&lhs->text, &rhs->text)); + } + + SVN_TEST_ASSERT(lhs->sub_counter == rhs->sub_counter); + return SVN_NO_ERROR; +} + +/* Assert that the first COUNT records in LHS and RHS have the same contents. + */ +static svn_error_t * +compare(const base_record_t *lhs, + const base_record_t *rhs, + apr_size_t count) +{ + apr_size_t i; + for (i = 0; i < count; ++i) + { + SVN_TEST_ASSERT(lhs[i].counter == rhs[i].counter); + SVN_ERR(compare_binary(&lhs[i].description, &rhs[i].description)); + SVN_TEST_ASSERT(lhs[i].large_unsigned1 == rhs[i].large_unsigned1); + SVN_TEST_ASSERT(lhs[i].large_unsigned2 == rhs[i].large_unsigned2); + SVN_ERR(compare_subs(lhs[i].left_subs, rhs[i].left_subs)); + SVN_TEST_ASSERT(lhs[i].counter == rhs[i].counter); + SVN_TEST_ASSERT(lhs[i].large_signed1 == rhs[i].large_signed1); + SVN_TEST_ASSERT(lhs[i].large_signed2 == rhs[i].large_signed2); + SVN_TEST_ASSERT(lhs[i].prime == rhs[i].prime); + SVN_ERR(compare_subs(lhs[i].right_subs, rhs[i].right_subs)); + SVN_ERR(compare_binary(&lhs[i].binary, &rhs[i].binary)); + } + + return SVN_NO_ERROR; +} + +static svn_error_t * +test_empty_structure(apr_pool_t *pool) +{ + base_record_t *unpacked; + apr_size_t count; + + /* create an empty, readable container */ + svn_packed__data_root_t *root = pack(test_data, 0, pool); + + SVN_ERR(get_read_root(&root, root, pool)); + unpacked = unpack(&count, root, pool); + SVN_TEST_ASSERT(count == 0); + SVN_ERR(compare(unpacked, test_data, count)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +test_full_structure(apr_pool_t *pool) +{ + base_record_t *unpacked; + apr_size_t count; + + /* create an empty, readable container */ + svn_packed__data_root_t *root = pack(test_data, BASE_RECORD_COUNT, pool); + + SVN_ERR(get_read_root(&root, root, pool)); + unpacked = unpack(&count, root, pool); + SVN_TEST_ASSERT(count == BASE_RECORD_COUNT); + SVN_ERR(compare(unpacked, test_data, count)); + + return SVN_NO_ERROR; +} + +/* An array of all test functions */ +struct svn_test_descriptor_t test_funcs[] = + { + SVN_TEST_NULL, + SVN_TEST_PASS2(test_empty_container, + "test empty container"), + SVN_TEST_PASS2(test_uint_stream, + "test a single uint stream"), + SVN_TEST_PASS2(test_int_stream, + "test a single int stream"), + SVN_TEST_PASS2(test_byte_stream, + "test a single bytes stream"), + SVN_TEST_PASS2(test_empty_structure, + "test empty, nested structure"), + SVN_TEST_PASS2(test_full_structure, + "test nested structure"), + SVN_TEST_NULL + };
diff --git a/subversion/tests/libsvn_subr/prefix-string-test.c b/subversion/tests/libsvn_subr/prefix-string-test.c new file mode 100644 index 0000000..5ccf1c3 --- /dev/null +++ b/subversion/tests/libsvn_subr/prefix-string-test.c
@@ -0,0 +1,149 @@ +/* + * prefix-string-test.c: a collection of svn_prefix_string__* tests + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +/* ==================================================================== + To add tests, look toward the bottom of this file. + +*/ + + + +#include <stdio.h> +#include <string.h> +#include <apr_pools.h> + +#include "../svn_test.h" + +#include "svn_error.h" +#include "svn_string.h" /* This includes <apr_*.h> */ +#include "private/svn_string_private.h" + +static svn_error_t * +test_empty_string(apr_pool_t *pool) +{ + svn_prefix_tree__t *tree = svn_prefix_tree__create(pool); + svn_prefix_string__t *empty = svn_prefix_string__create(tree, ""); + + /* same instance for all strings of the same value */ + SVN_TEST_ASSERT(empty == svn_prefix_string__create(tree, "")); + + /* does it actually have the right contents? */ + SVN_TEST_ASSERT(svn_prefix_string__expand(empty, pool)->len == 0); + SVN_TEST_STRING_ASSERT(svn_prefix_string__expand(empty, pool)->data, ""); + + /* strings shall be equal to themselves */ + SVN_TEST_ASSERT(0 == svn_prefix_string__compare(empty, empty)); + + return SVN_NO_ERROR; +} + +enum {TEST_CASE_COUNT = 9}; + +const char *test_cases[TEST_CASE_COUNT] = +{ + "a longish string of sorts, longer than 7 anyway", + "some other string", + "more stuff on root", + "some shorter string", + "some short string", + "some short str", + "some short str2", + "a longish string of sorts, longer than ?! anyway", + "a" +}; + +static svn_error_t * +test_string_creation(apr_pool_t *pool) +{ + svn_prefix_tree__t *tree = svn_prefix_tree__create(pool); + svn_prefix_string__t *strings[TEST_CASE_COUNT]; + int i; + + /* create strings and remember their initial references */ + for (i = 0; i < TEST_CASE_COUNT; ++i) + strings[i] = svn_prefix_string__create(tree, test_cases[i]); + + /* doing this again must yield the same pointers */ + for (i = 0; i < TEST_CASE_COUNT; ++i) + SVN_TEST_ASSERT(strings[i] + == svn_prefix_string__create(tree, test_cases[i])); + + /* converting them back to strings must be the initial values */ + for (i = 0; i < TEST_CASE_COUNT; ++i) + { + svn_string_t *expanded = svn_prefix_string__expand(strings[i], pool); + + SVN_TEST_ASSERT(expanded->len == strlen(test_cases[i])); + SVN_TEST_STRING_ASSERT(expanded->data, test_cases[i]); + + } + + return SVN_NO_ERROR; +} + +static svn_error_t * +test_string_comparison(apr_pool_t *pool) +{ + svn_prefix_tree__t *tree = svn_prefix_tree__create(pool); + svn_prefix_string__t *strings[TEST_CASE_COUNT]; + int i, k; + + /* create strings */ + for (i = 0; i < TEST_CASE_COUNT; ++i) + strings[i] = svn_prefix_string__create(tree, test_cases[i]); + + /* comparing them with themselves */ + for (i = 0; i < TEST_CASE_COUNT; ++i) + SVN_TEST_ASSERT(! svn_prefix_string__compare(strings[i], strings[i])); + + /* compare with all other strings */ + for (i = 0; i < TEST_CASE_COUNT; ++i) + { + svn_string_t *lhs = svn_prefix_string__expand(strings[i], pool); + for (k = 0; k < TEST_CASE_COUNT; ++k) + { + svn_string_t *rhs = svn_prefix_string__expand(strings[k], pool); + int expected_diff = strcmp(lhs->data, rhs->data); + int actual_diff = svn_prefix_string__compare(strings[i], strings[k]); + + SVN_TEST_ASSERT((actual_diff < 0) == (expected_diff < 0)); + SVN_TEST_ASSERT((actual_diff > 0) == (expected_diff > 0)); + SVN_TEST_ASSERT(!actual_diff == !expected_diff); + } + } + + return SVN_NO_ERROR; +} + +/* An array of all test functions */ +struct svn_test_descriptor_t test_funcs[] = + { + SVN_TEST_NULL, + SVN_TEST_PASS2(test_empty_string, + "check empty strings"), + SVN_TEST_PASS2(test_string_creation, + "create many strings"), + SVN_TEST_PASS2(test_string_comparison, + "compare strings"), + SVN_TEST_NULL + };
diff --git a/subversion/tests/libsvn_subr/priority-queue-test.c b/subversion/tests/libsvn_subr/priority-queue-test.c new file mode 100644 index 0000000..9863e9a --- /dev/null +++ b/subversion/tests/libsvn_subr/priority-queue-test.c
@@ -0,0 +1,235 @@ +/* + * priority-queue-test.c: a collection of svn_priority_queue__* tests + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +/* ==================================================================== + To add tests, look toward the bottom of this file. + +*/ + + + +#include <stdio.h> +#include <string.h> +#include <apr_pools.h> + +#include "../svn_test.h" + +#include "svn_error.h" +#include "svn_sorts.h" + +/* priority queue test: + * items in the queue are simple integers, in ascending order */ + +/* number of items to put into the queue */ +enum {NUMBER_COUNT = 11}; + +/* the actual values in the order we add them to the queue */ +const int numbers[NUMBER_COUNT] + = { 8395, 0, -1, 3885, 1, -435, 99993, 10, 0, 1, 8395 }; + +/* test_update will modify in-queue data and expects the queue to return + the values in the following order: */ +const int expected_modified[NUMBER_COUNT] + = { -431, 0, 1, 3, 5, 10, 16, 3889, 8395, 8403, 99997 }; + +/* standard compare function for integers */ +static int +compare_func(const void *lhs, const void *rhs) +{ + return *(const int *)lhs - *(const int *)rhs; +} + +/* Check that QUEUE is empty and the usual operations still work */ +static svn_error_t * +verify_empty_queue(svn_priority_queue__t *queue) +{ + /* it's an empty queue */ + SVN_TEST_ASSERT(svn_priority_queue__size(queue) == 0); + SVN_TEST_ASSERT(svn_priority_queue__peek(queue) == NULL); + + /* these should be no-ops */ + svn_priority_queue__update(queue); + svn_priority_queue__pop(queue); + + return SVN_NO_ERROR; +} + +/* check that the tip of QUEUE equals EXPECTED and remove the first element */ +static svn_error_t * +extract_expected(svn_priority_queue__t *queue, int expected) +{ + int value = *(int *)svn_priority_queue__peek(queue); + SVN_TEST_ASSERT(value == expected); + svn_priority_queue__pop(queue); + + return SVN_NO_ERROR; +} + +/* Verify that QUEUE returns all elements in the proper order. + Also check that data can be added & removed without disturbing the order. + */ +static svn_error_t * +verify_queue_order(svn_priority_queue__t *queue) +{ + int sorted[NUMBER_COUNT]; + int i; + + /* reference order */ + memcpy(sorted, numbers, sizeof(numbers)); + qsort(sorted, NUMBER_COUNT, sizeof(sorted[0]), compare_func); + + /* verify that the queue returns the data in the same order */ + for (i = 0; i < NUMBER_COUNT; ++i) + { + int item = *(int *)svn_priority_queue__peek(queue); + int to_insert; + + /* is this the value we expected? */ + SVN_TEST_ASSERT(item == sorted[i]); + + /* add two items at the tip of the queue */ + to_insert = item - 1; + svn_priority_queue__push(queue, &to_insert); + svn_priority_queue__push(queue, &item); + + /* check queue length */ + SVN_TEST_ASSERT(svn_priority_queue__size(queue) == NUMBER_COUNT-i+2); + + /* now, lets extract all 3 of them */ + SVN_ERR(extract_expected(queue, item-1)); + SVN_ERR(extract_expected(queue, item)); + SVN_ERR(extract_expected(queue, item)); + + /* check queue length */ + SVN_TEST_ASSERT(svn_priority_queue__size(queue) == NUMBER_COUNT-i-1); + } + + /* the queue should now be empty */ + verify_empty_queue(queue); + + return SVN_NO_ERROR; +} + +/* return a queue allocated in POOL containing all items of NUMBERS */ +static svn_priority_queue__t * +create_standard_queue(apr_pool_t *pool) +{ + apr_array_header_t *elements + = apr_array_make(pool, 11, sizeof(numbers[0])); + + /* build queue */ + int i; + for (i = 0; i < NUMBER_COUNT; ++i) + APR_ARRAY_PUSH(elements, int) = numbers[i]; + + return svn_priority_queue__create(elements, compare_func); +} + + +static svn_error_t * +test_empty_queue(apr_pool_t *pool) +{ + apr_array_header_t *elements + = apr_array_make(pool, 0, sizeof(int)); + svn_priority_queue__t *queue + = svn_priority_queue__create(elements, compare_func); + + verify_empty_queue(queue); + + return SVN_NO_ERROR; +} + +static svn_error_t * +test_sort_queue(apr_pool_t *pool) +{ + svn_priority_queue__t *queue = create_standard_queue(pool); + + /* data should come out of the queue in sorted order */ + SVN_ERR(verify_queue_order(queue)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +test_push(apr_pool_t *pool) +{ + apr_array_header_t *elements + = apr_array_make(pool, 3, sizeof(int)); + svn_priority_queue__t *queue + = svn_priority_queue__create(elements, compare_func); + + /* build queue */ + int i; + for (i = 0; i < NUMBER_COUNT; ++i) + svn_priority_queue__push(queue, &numbers[i]); + + /* data should come out of the queue in sorted order */ + SVN_ERR(verify_queue_order(queue)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +test_update(apr_pool_t *pool) +{ + svn_priority_queue__t *queue = create_standard_queue(pool); + + /* modify all items in the queue */ + int i; + for (i = 0; i < NUMBER_COUNT; ++i) + { + int *tip = svn_priority_queue__peek(queue); + *tip += 4; + svn_priority_queue__update(queue); + + /* extract and verify tip */ + SVN_TEST_ASSERT(*(int *)svn_priority_queue__peek(queue) + == expected_modified[i]); + svn_priority_queue__pop(queue); + + /* this should be a no-op now */ + svn_priority_queue__update(queue); + + SVN_TEST_ASSERT(svn_priority_queue__size(queue) == NUMBER_COUNT-i-1); + } + + /* the queue should now be empty */ + verify_empty_queue(queue); + + return SVN_NO_ERROR; +} + +/* An array of all test functions */ +struct svn_test_descriptor_t test_funcs[] = + { + SVN_TEST_NULL, + SVN_TEST_PASS2(test_empty_queue, + "test empty queue"), + SVN_TEST_PASS2(test_sort_queue, + "data returned by a priority queue shall be ordered"), + SVN_TEST_PASS2(test_push, + "priority queues can be built up incrementally"), + SVN_TEST_PASS2(test_update, + "updating the head of the queue"), + SVN_TEST_NULL + };
diff --git a/subversion/tests/libsvn_subr/spillbuf-test.c b/subversion/tests/libsvn_subr/spillbuf-test.c index 3b08143..29e6a2c 100644 --- a/subversion/tests/libsvn_subr/spillbuf-test.c +++ b/subversion/tests/libsvn_subr/spillbuf-test.c
@@ -336,9 +336,10 @@ static svn_error_t * test_spillbuf_stream(apr_pool_t *pool) { - svn_stream_t *stream = svn_stream__from_spillbuf(8 /* blocksize */, - 15 /* maxsize */, - pool); + svn_spillbuf_t *buf = svn_spillbuf__create(4 /* blocksize */, + 100 /* maxsize */, + pool); + svn_stream_t *stream = svn_stream__from_spillbuf(buf, pool); char readbuf[256]; apr_size_t readlen; apr_size_t writelen;
diff --git a/subversion/tests/svn_test_fs.c b/subversion/tests/svn_test_fs.c index 48d6eec..58a763a 100644 --- a/subversion/tests/svn_test_fs.c +++ b/subversion/tests/svn_test_fs.c
@@ -145,21 +145,36 @@ * copy that file into the filesystem FS and set *MUST_REOPEN to TRUE, else * set *MUST_REOPEN to FALSE. */ static svn_error_t * -maybe_install_fsfs_conf(svn_fs_t *fs, - const svn_test_opts_t *opts, - svn_boolean_t *must_reopen, - apr_pool_t *pool) +maybe_install_fs_conf(svn_fs_t *fs, + const svn_test_opts_t *opts, + svn_boolean_t *must_reopen, + apr_pool_t *pool) { *must_reopen = FALSE; - if (strcmp(opts->fs_type, "fsfs") != 0 || ! opts->config_file) + if (! opts->config_file) return SVN_NO_ERROR; - *must_reopen = TRUE; - return svn_io_copy_file(opts->config_file, - svn_path_join(svn_fs_path(fs, pool), - "fsfs.conf", pool), - FALSE /* copy_perms */, - pool); + if (strcmp(opts->fs_type, "fsfs") == 0) + { + *must_reopen = TRUE; + return svn_io_copy_file(opts->config_file, + svn_path_join(svn_fs_path(fs, pool), + "fsfs.conf", pool), + FALSE /* copy_perms */, + pool); + } + + if (strcmp(opts->fs_type, "fsx") == 0) + { + *must_reopen = TRUE; + return svn_io_copy_file(opts->config_file, + svn_path_join(svn_fs_path(fs, pool), + "fsx.conf", pool), + FALSE /* copy_perms */, + pool); + } + + return SVN_NO_ERROR; } @@ -184,7 +199,7 @@ SVN_ERR(create_fs(fs_p, name, opts->fs_type, opts->server_minor_version, pool)); - SVN_ERR(maybe_install_fsfs_conf(*fs_p, opts, &must_reopen, pool)); + SVN_ERR(maybe_install_fs_conf(*fs_p, opts, &must_reopen, pool)); if (must_reopen) { SVN_ERR(svn_fs_open(fs_p, name, NULL, pool)); @@ -230,8 +245,8 @@ /* Register this repo for cleanup. */ svn_test_add_dir_cleanup(name); - SVN_ERR(maybe_install_fsfs_conf(svn_repos_fs(repos), opts, &must_reopen, - pool)); + SVN_ERR(maybe_install_fs_conf(svn_repos_fs(repos), opts, &must_reopen, + pool)); if (must_reopen) { SVN_ERR(svn_repos_open2(&repos, name, NULL, pool));