Return-Path: X-Original-To: apmail-subversion-commits-archive@minotaur.apache.org Delivered-To: apmail-subversion-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 04E5E90B7 for ; Mon, 9 Apr 2012 21:46:57 +0000 (UTC) Received: (qmail 72510 invoked by uid 500); 9 Apr 2012 21:46:56 -0000 Delivered-To: apmail-subversion-commits-archive@subversion.apache.org Received: (qmail 72478 invoked by uid 500); 9 Apr 2012 21:46:56 -0000 Mailing-List: contact commits-help@subversion.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@subversion.apache.org Delivered-To: mailing list commits@subversion.apache.org Received: (qmail 72469 invoked by uid 99); 9 Apr 2012 21:46:56 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 09 Apr 2012 21:46:56 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 09 Apr 2012 21:46:54 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 9BE5223889E1; Mon, 9 Apr 2012 21:46:34 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1311476 - in /subversion/trunk/subversion/libsvn_fs_fs: fs.h fs_fs.c Date: Mon, 09 Apr 2012 21:46:34 -0000 To: commits@subversion.apache.org From: stefan2@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120409214634.9BE5223889E1@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: stefan2 Date: Mon Apr 9 21:46:34 2012 New Revision: 1311476 URL: http://svn.apache.org/viewvc?rev=1311476&view=rev Log: Turn hard-coded deltification parameters into config parameters for format 4 and later (they all keep compatibility with 1.6 and 1.7). * subversion/libsvn_fs_fs/fs.h (CONFIG_SECTION_DELTIFICATION, CONFIG_OPTION_ENABLE_DIR_DELTIFICATION, CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION, CONFIG_OPTION_MAX_DELTIFICATION_WALK, CONFIG_OPTION_MAX_LINEAR_DELTIFICATION, SVN_FS_FS__MIN_DELTIFICATION_FORMAT): define new constants (fs_fs_data_t): add new config options as members * subversion/libsvn_fs_fs/fs_fs.c (read_config): read / init the new config options (write_config): extend the template for new config files (choose_delta_base, write_final_rev): use config parameter instead of #defines (write_hash_delta_rep): compile unconditionally Modified: subversion/trunk/subversion/libsvn_fs_fs/fs.h subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c Modified: subversion/trunk/subversion/libsvn_fs_fs/fs.h URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/fs.h?rev=1311476&r1=1311475&r2=1311476&view=diff ============================================================================== --- subversion/trunk/subversion/libsvn_fs_fs/fs.h (original) +++ subversion/trunk/subversion/libsvn_fs_fs/fs.h Mon Apr 9 21:46:34 2012 @@ -82,6 +82,11 @@ extern "C" { #define CONFIG_OPTION_FAIL_STOP "fail-stop" #define CONFIG_SECTION_REP_SHARING "rep-sharing" #define CONFIG_OPTION_ENABLE_REP_SHARING "enable-rep-sharing" +#define CONFIG_SECTION_DELTIFICATION "deltification" +#define CONFIG_OPTION_ENABLE_DIR_DELTIFICATION "enable-dir-deltification" +#define CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION "enable-props-deltification" +#define CONFIG_OPTION_MAX_DELTIFICATION_WALK "max-deltification-walk" +#define CONFIG_OPTION_MAX_LINEAR_DELTIFICATION "max-linear-deltification" /* The format number of this filesystem. This is independent of the repository format number, and @@ -118,6 +123,10 @@ extern "C" { /* The minimum format number that stores node kinds in changed-paths lists. */ #define SVN_FS_FS__MIN_KIND_IN_CHANGED_FORMAT 4 +/* 1.8 deltification options should work with any FSFS repo but to avoid + * issues with very old servers, restrict those options to the 1.6+ format*/ +#define SVN_FS_FS__MIN_DELTIFICATION_FORMAT 4 + /* The 1.7-dev format, never released, that packed revprops into SQLite revprops.db . */ #define SVN_FS_FS__PACKED_REVPROP_SQLITE_DEV_FORMAT 5 @@ -275,6 +284,19 @@ typedef struct fs_fs_data_t /* Whether rep-sharing is supported by the filesystem * and allowed by the configuration. */ svn_boolean_t rep_sharing_allowed; + + /* Whether directory nodes shall be deltified just like file nodes. */ + svn_boolean_t deltify_directories; + + /* Whether nodes properties shall be deltified. */ + svn_boolean_t deltify_properties; + + /* Restart deltification histories after each multiple of this value */ + apr_int64_t max_deltification_walk; + + /* Maximum number of length of the linear part at the top of the + * deltification history after which skip deltas will be used. */ + apr_int64_t max_linear_deltification; } fs_fs_data_t; Modified: subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c?rev=1311476&r1=1311475&r2=1311476&view=diff ============================================================================== --- subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c (original) +++ subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c Mon Apr 9 21:46:34 2012 @@ -1082,6 +1082,34 @@ read_config(svn_fs_t *fs, else ffd->rep_sharing_allowed = FALSE; + /* Initialize ffd->deltify_directories. */ + if (ffd->format >= SVN_FS_FS__MIN_DELTIFICATION_FORMAT) + { + SVN_ERR(svn_config_get_bool(ffd->config, &ffd->deltify_directories, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_ENABLE_DIR_DELTIFICATION, + FALSE)); + SVN_ERR(svn_config_get_bool(ffd->config, &ffd->deltify_properties, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION, + FALSE)); + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->max_deltification_walk, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_MAX_DELTIFICATION_WALK, + SVN_FS_FS_MAX_DELTIFICATION_WALK)); + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->max_linear_deltification, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_MAX_LINEAR_DELTIFICATION, + SVN_FS_FS_MAX_LINEAR_DELTIFICATION)); + } + else + { + ffd->deltify_directories = FALSE; + ffd->deltify_properties = FALSE; + ffd->max_deltification_walk = SVN_FS_FS_MAX_DELTIFICATION_WALK; + ffd->max_linear_deltification = SVN_FS_FS_MAX_LINEAR_DELTIFICATION; + } + return SVN_NO_ERROR; } @@ -1130,7 +1158,67 @@ write_config(svn_fs_t *fs, "### 'svnadmin verify' will check the rep-cache regardless of this setting." NL "### rep-sharing is enabled by default." NL "# " CONFIG_OPTION_ENABLE_REP_SHARING " = true" NL - +"" NL +"[" CONFIG_SECTION_DELTIFICATION "]" NL +"### To conserve space, the filesystem stores data as differences against" NL +"### existing representations. This comes at a slight cost in performance," NL +"### as calculating differences can increase commit times. Reading data" NL +"### will also create higher CPU load and the data will be fragmented." NL +"### Since deltification tends to save significant amounts of disk space," NL +"### the overall I/O load can actually be lower." NL +"###" NL +"### The options in this section allow for tuning the deltification" NL +"### strategy. Their effects on data size and server performance may vary" NL +"### from one repository to another. Versions prior to 1.8 will ignore" NL +"### this section." NL +"###" NL +"### The following parameter enables deltification for directories. It can" NL +"### be switched on and off at will, but for best space-saving results" NL +"### should be enabled consistently over the life of the repository." NL +"### Repositories containing large directories will benefit greatly." NL +"### In rarely read repositories, the I/O overhead may be significant as" NL +"### cache hit rates will most likely be low" NL +"### directory deltification is disabled by default." NL +"# " CONFIG_OPTION_ENABLE_DIR_DELTIFICATION " = true" NL +"###" NL +"### The following parameter enables deltification for properties on files" NL +"### and directories. Overall, this is a minor tuning option but can save" NL +"### some disk space if frequently merge or if you frequently change node" NL +"### properties. You should not activate this if rep-sharing has been" NL +"### disabled." NL +"### property deltification is disabled by default." NL +"# " CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION " = true" NL +"###" NL +"### During commit, the server may need to walk the whole change history of" NL +"### of a given node to find a suitable deltification base. This linear" NL +"### process can impact commit times, svnadmin load and similar operations." NL +"### This setting limits the depth of the deltification history. If the" NL +"### threshold has been reached, the node will be stored as fulltext and a" NL +"### new deltification history begins." NL +"### Note, this is unrelated to svn log." NL +"### Very large values rarely provide significant additional savings but" NL +"### can impact performance greatly - in particular if directory" NL +"### deltification has been activated. Very small values may be useful in" NL +"### repositories that are dominated by large, changing binaries." NL +"### Should be a power of two minus 1. A value of 0 will effectively" NL +"### disable deltification." NL +"### For 1.8, the default value is 1023; earlier versions have no limit." NL +"# " CONFIG_OPTION_MAX_DELTIFICATION_WALK " = 1023" NL +"###" NL +"### The skip-delta scheme used by FSFS tends to repeatably store redundant" NL +"### delta information where a simple delta against the latest version is" NL +"### often smaller. By default, 1.8+ will therefore use skip deltas only" NL +"### after the linear chain of deltas has grown beyond the threshold" NL +"### specified by this setting." NL +"### Values up to 64 can result in some reduction in repository size for" NL +"### the cost of quickly increasing I/O and CPU costs. Similarly, smaller" NL +"### numbers can reduce those costs at the cost of more disk space. For" NL +"### rarely read repositories or those containing larger binaries, this may" NL +"### present a better trade-off." NL +"### Should be a power of two. A value of 1 or smaller will cause the" NL +"### exclusive use of skip-deltas (as in pre-1.8)." NL +"### For 1.8, the default value is 16; earlier versions use 1." NL +"# " CONFIG_OPTION_MAX_LINEAR_DELTIFICATION " = 16" NL ; #undef NL return svn_io_file_create(svn_dirent_join(fs->path, PATH_CONFIG, pool), @@ -5265,6 +5353,7 @@ choose_delta_base(representation_t **rep int count; int walk; node_revision_t *base; + fs_fs_data_t *ffd = fs->fsap_data; /* If we have no predecessors, then use the empty stream as a base. */ @@ -5285,14 +5374,14 @@ choose_delta_base(representation_t **rep along very long node histories. Close to HEAD however, we create a linear history to minimize delta size. */ walk = noderev->predecessor_count - count; - if (walk < SVN_FS_FS_MAX_LINEAR_DELTIFICATION) + if (walk < (int)ffd->max_linear_deltification) count = noderev->predecessor_count - 1; /* Finding the delta base over a very long distance can become extremely expensive for very deep histories, possibly causing client timeouts etc. OTOH, this is a rare operation and its gains are minimal. Lets simply start deltification anew close every other 1000 changes or so. */ - if (walk > SVN_FS_FS_MAX_DELTIFICATION_WALK) + if (walk > (int)ffd->max_deltification_walk) { *rep = NULL; return SVN_NO_ERROR; @@ -5772,7 +5861,6 @@ write_hash_rep(representation_t *rep, earlier reps with the same content. When such existing reps can be found, we will truncate the one just written from the file and return the existing rep. Perform temporary allocations in POOL. */ -#ifdef SVN_FS_FS_DELTIFY_DIRECTORIES static svn_error_t * write_hash_delta_rep(representation_t *rep, apr_file_t *file, @@ -5873,7 +5961,6 @@ write_hash_delta_rep(representation_t *r return SVN_NO_ERROR; } -#endif /* Sanity check ROOT_NODEREV, a candidate for being the root node-revision of (not yet committed) revision REV in FS. Use POOL for temporary @@ -6026,14 +6113,13 @@ write_final_rev(const svn_fs_id_t **new_ noderev->data_rep->txn_id = NULL; noderev->data_rep->revision = rev; -#ifdef SVN_FS_FS_DELTIFY_DIRECTORIES - SVN_ERR(write_hash_delta_rep(noderev->data_rep, file, - str_entries, fs, noderev, NULL, - pool)); -#else - SVN_ERR(write_hash_rep(noderev->data_rep, file, str_entries, - fs, NULL, pool)); -#endif + if (ffd->deltify_directories) + SVN_ERR(write_hash_delta_rep(noderev->data_rep, file, + str_entries, fs, noderev, NULL, + pool)); + else + SVN_ERR(write_hash_rep(noderev->data_rep, file, str_entries, + fs, NULL, pool)); } } else @@ -6066,14 +6152,13 @@ write_final_rev(const svn_fs_id_t **new_ noderev->prop_rep->txn_id = NULL; noderev->prop_rep->revision = rev; -#ifdef SVN_FS_FS_DELTIFY_PROPS - SVN_ERR(write_hash_delta_rep(noderev->prop_rep, file, - proplist, fs, noderev, reps_hash, - pool)); -#else - SVN_ERR(write_hash_rep(noderev->prop_rep, file, proplist, - fs, reps_hash, pool)); -#endif + if (ffd->deltify_properties) + SVN_ERR(write_hash_delta_rep(noderev->prop_rep, file, + proplist, fs, noderev, reps_hash, + pool)); + else + SVN_ERR(write_hash_rep(noderev->prop_rep, file, proplist, + fs, reps_hash, pool)); }