From commits-return-49779-archive-asf-public=cust-asf.ponee.io@subversion.apache.org Wed Nov 7 13:30:26 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id AB03C1807AC for ; Wed, 7 Nov 2018 13:30:21 +0100 (CET) Received: (qmail 19918 invoked by uid 500); 7 Nov 2018 12:30:20 -0000 Mailing-List: contact commits-help@subversion.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@subversion.apache.org Delivered-To: mailing list commits@subversion.apache.org Received: (qmail 19136 invoked by uid 99); 7 Nov 2018 12:30:19 -0000 Received: from Unknown (HELO svn01-us-west.apache.org) (209.188.14.144) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 07 Nov 2018 12:30:19 +0000 Received: from svn01-us-west.apache.org (localhost [127.0.0.1]) by svn01-us-west.apache.org (ASF Mail Server at svn01-us-west.apache.org) with ESMTP id 8CE603A2691 for ; Wed, 7 Nov 2018 12:30:18 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1846002 [11/44] - in /subversion/branches/ra-git: ./ build/ build/ac-macros/ build/generator/ build/generator/swig/ build/generator/templates/ build/generator/util/ build/win32/ contrib/client-side/ contrib/client-side/svn_load_dirs/ contr... Date: Wed, 07 Nov 2018 12:30:11 -0000 To: commits@subversion.apache.org From: rhuijben@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20181107123018.8CE603A2691@svn01-us-west.apache.org> Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/fs_fs.c URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/fs_fs.c?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/fs_fs.c (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/fs_fs.c Wed Nov 7 12:30:06 2018 @@ -683,6 +683,60 @@ verify_block_size(apr_int64_t block_size return SVN_NO_ERROR; } +static svn_error_t * +parse_compression_option(compression_type_t *compression_type_p, + int *compression_level_p, + const char *value) +{ + compression_type_t type; + int level; + svn_boolean_t is_valid = TRUE; + + /* compression = none | lz4 | zlib | zlib-1 ... zlib-9 */ + if (strcmp(value, "none") == 0) + { + type = compression_type_none; + level = SVN_DELTA_COMPRESSION_LEVEL_NONE; + } + else if (strcmp(value, "lz4") == 0) + { + type = compression_type_lz4; + level = SVN_DELTA_COMPRESSION_LEVEL_DEFAULT; + } + else if (strncmp(value, "zlib", 4) == 0) + { + const char *p = value + 4; + + type = compression_type_zlib; + if (*p == 0) + { + level = SVN_DELTA_COMPRESSION_LEVEL_DEFAULT; + } + else if (*p == '-') + { + p++; + SVN_ERR(svn_cstring_atoi(&level, p)); + if (level < 1 || level > 9) + is_valid = FALSE; + } + else + is_valid = FALSE; + } + else + { + is_valid = FALSE; + } + + if (!is_valid) + return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE, NULL, + _("Invalid 'compression' value '%s' in the config"), + value); + + *compression_type_p = type; + *compression_level_p = level; + return SVN_NO_ERROR; +} + /* Read the configuration information of the file system at FS_PATH * and set the respective values in FFD. Use pools as usual. */ @@ -709,8 +763,6 @@ read_config(fs_fs_data_t *ffd, /* Initialize deltification settings in ffd. */ if (ffd->format >= SVN_FS_FS__MIN_DELTIFICATION_FORMAT) { - apr_int64_t compression_level; - SVN_ERR(svn_config_get_bool(config, &ffd->deltify_directories, CONFIG_SECTION_DELTIFICATION, CONFIG_OPTION_ENABLE_DIR_DELTIFICATION, @@ -727,14 +779,6 @@ read_config(fs_fs_data_t *ffd, CONFIG_SECTION_DELTIFICATION, CONFIG_OPTION_MAX_LINEAR_DELTIFICATION, SVN_FS_FS_MAX_LINEAR_DELTIFICATION)); - - SVN_ERR(svn_config_get_int64(config, &compression_level, - CONFIG_SECTION_DELTIFICATION, - CONFIG_OPTION_COMPRESSION_LEVEL, - SVN_DELTA_COMPRESSION_LEVEL_DEFAULT)); - ffd->delta_compression_level - = (int)MIN(MAX(SVN_DELTA_COMPRESSION_LEVEL_NONE, compression_level), - SVN_DELTA_COMPRESSION_LEVEL_MAX); } else { @@ -742,7 +786,6 @@ read_config(fs_fs_data_t *ffd, ffd->deltify_properties = FALSE; ffd->max_deltification_walk = SVN_FS_FS_MAX_DELTIFICATION_WALK; ffd->max_linear_deltification = SVN_FS_FS_MAX_LINEAR_DELTIFICATION; - ffd->delta_compression_level = SVN_DELTA_COMPRESSION_LEVEL_DEFAULT; } /* Initialize revprop packing settings in ffd. */ @@ -817,6 +860,83 @@ read_config(fs_fs_data_t *ffd, ffd->pack_after_commit = FALSE; } + /* Initialize compression settings in ffd. */ + if (ffd->format >= SVN_FS_FS__MIN_DELTIFICATION_FORMAT) + { + const char *compression_val; + const char *compression_level_val; + + svn_config_get(config, &compression_val, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_COMPRESSION, NULL); + svn_config_get(config, &compression_level_val, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_COMPRESSION_LEVEL, NULL); + if (compression_val && compression_level_val) + { + return svn_error_create(SVN_ERR_BAD_CONFIG_VALUE, NULL, + _("The 'compression' and 'compression-level' " + "config options are mutually exclusive")); + } + else if (compression_val) + { + SVN_ERR(parse_compression_option(&ffd->delta_compression_type, + &ffd->delta_compression_level, + compression_val)); + if (ffd->delta_compression_type == compression_type_lz4 && + ffd->format < SVN_FS_FS__MIN_SVNDIFF2_FORMAT) + { + return svn_error_create(SVN_ERR_BAD_CONFIG_VALUE, NULL, + _("Compression type 'lz4' requires " + "filesystem format 8 or higher")); + } + } + else if (compression_level_val) + { + /* Handle the deprecated 'compression-level' option. */ + ffd->delta_compression_type = compression_type_zlib; + SVN_ERR(svn_cstring_atoi(&ffd->delta_compression_level, + compression_level_val)); + ffd->delta_compression_level = + MIN(MAX(SVN_DELTA_COMPRESSION_LEVEL_NONE, + ffd->delta_compression_level), + SVN_DELTA_COMPRESSION_LEVEL_MAX); + } + else + { + /* Nothing specified explicitly, use the default settings: + * LZ4 compression for formats supporting it and zlib otherwise. */ + if (ffd->format >= SVN_FS_FS__MIN_SVNDIFF2_FORMAT) + ffd->delta_compression_type = compression_type_lz4; + else + ffd->delta_compression_type = compression_type_zlib; + + ffd->delta_compression_level = SVN_DELTA_COMPRESSION_LEVEL_DEFAULT; + } + } + else if (ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT) + { + ffd->delta_compression_type = compression_type_zlib; + ffd->delta_compression_level = SVN_DELTA_COMPRESSION_LEVEL_DEFAULT; + } + else + { + ffd->delta_compression_type = compression_type_none; + ffd->delta_compression_level = SVN_DELTA_COMPRESSION_LEVEL_NONE; + } + +#ifdef SVN_DEBUG + SVN_ERR(svn_config_get_bool(config, &ffd->verify_before_commit, + CONFIG_SECTION_DEBUG, + CONFIG_OPTION_VERIFY_BEFORE_COMMIT, + TRUE)); +#else + SVN_ERR(svn_config_get_bool(config, &ffd->verify_before_commit, + CONFIG_SECTION_DEBUG, + CONFIG_OPTION_VERIFY_BEFORE_COMMIT, + FALSE)); +#endif + /* memcached configuration */ SVN_ERR(svn_cache__make_memcache_from_config(&ffd->memcache, config, result_pool, scratch_pool)); @@ -935,23 +1055,30 @@ write_config(svn_fs_t *fs, "### For 1.8, the default value is 16; earlier versions use 1." NL "# " CONFIG_OPTION_MAX_LINEAR_DELTIFICATION " = 16" NL "###" NL -"### After deltification, we compress the data through zlib to minimize on-" NL -"### disk size. That can be an expensive and ineffective process. This" NL -"### setting controls the usage of zlib in future revisions." NL -"### Revisions with highly compressible data in them may shrink in size" NL -"### if the setting is increased but may take much longer to commit. The" NL -"### time taken to uncompress that data again is widely independent of the" NL -"### compression level." NL -"### Compression will be ineffective if the incoming content is already" NL -"### highly compressed. In that case, disabling the compression entirely" NL -"### will speed up commits as well as reading the data. Repositories with" NL -"### many small compressible files (source code) but also a high percentage" NL -"### of large incompressible ones (artwork) may benefit from compression" NL -"### levels lowered to e.g. 1." NL -"### Valid values are 0 to 9 with 9 providing the highest compression ratio" NL -"### and 0 disabling it altogether." NL -"### The default value is 5." NL -"# " CONFIG_OPTION_COMPRESSION_LEVEL " = 5" NL +"### After deltification, we compress the data to minimize on-disk size." NL +"### This setting controls the compression algorithm, which will be used in" NL +"### future revisions. It can be used to either disable compression or to" NL +"### select between available algorithms (zlib, lz4). zlib is a general-" NL +"### purpose compression algorithm. lz4 is a fast compression algorithm" NL +"### which should be preferred for repositories with large and, possibly," NL +"### incompressible files. Note that the compression ratio of lz4 is" NL +"### usually lower than the one provided by zlib, but using it can" NL +"### significantly speed up commits as well as reading the data." NL +"### lz4 compression algorithm is supported, starting from format 8" NL +"### repositories, available in Subversion 1.10 and higher." NL +"### The syntax of this option is:" NL +"### " CONFIG_OPTION_COMPRESSION " = none | lz4 | zlib | zlib-1 ... zlib-9" NL +"### Versions prior to Subversion 1.10 will ignore this option." NL +"### The default value is 'lz4' if supported by the repository format and" NL +"### 'zlib' otherwise. 'zlib' is currently equivalent to 'zlib-5'." NL +"# " CONFIG_OPTION_COMPRESSION " = lz4" NL +"###" NL +"### DEPRECATED: The new '" CONFIG_OPTION_COMPRESSION "' option deprecates previously used" NL +"### '" CONFIG_OPTION_COMPRESSION_LEVEL "' option, which was used to configure zlib compression." NL +"### For compatibility with previous versions of Subversion, this option can"NL +"### still be used (and it will result in zlib compression with the" NL +"### corresponding compression level)." NL +"### " CONFIG_OPTION_COMPRESSION_LEVEL " = 0 ... 9 (default is 5)" NL "" NL "[" CONFIG_SECTION_PACKED_REVPROPS "]" NL "### This parameter controls the size (in kBytes) of packed revprop files." NL @@ -1020,6 +1147,13 @@ write_config(svn_fs_t *fs, "### Must be a power of 2." NL "### p2l-page-size is given in kBytes and with a default of 1024 kBytes." NL "# " CONFIG_OPTION_P2L_PAGE_SIZE " = 1024" NL +"" NL +"[" CONFIG_SECTION_DEBUG "]" NL +"###" NL +"### Whether to verify each new revision immediately before finalizing" NL +"### the commit. This is disabled by default except in maintainer-mode" NL +"### builds." NL +"# " CONFIG_OPTION_VERIFY_BEFORE_COMMIT " = false" NL ; #undef NL return svn_io_file_create(svn_dirent_join(fs->path, PATH_CONFIG, pool), @@ -1336,7 +1470,10 @@ svn_fs_fs__min_unpacked_rev(svn_revnum_t { fs_fs_data_t *ffd = fs->fsap_data; - SVN_ERR(svn_fs_fs__update_min_unpacked_rev(fs, pool)); + /* Calling this for pre-v4 repos is illegal. */ + if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) + SVN_ERR(svn_fs_fs__update_min_unpacked_rev(fs, pool)); + *min_unpacked = ffd->min_unpacked_rev; return SVN_NO_ERROR; @@ -1811,6 +1948,8 @@ svn_fs_fs__create(svn_fs_t *fs, case 8: format = 6; break; + case 9: format = 7; + break; default:format = SVN_FS_FS__FORMAT_NUMBER; } @@ -2180,8 +2319,11 @@ svn_fs_fs__info_format(int *fs_format, case 7: (*supports_version)->minor = 9; break; + case 8: + (*supports_version)->minor = 10; + break; #ifdef SVN_DEBUG -# if SVN_FS_FS__FORMAT_NUMBER != 7 +# if SVN_FS_FS__FORMAT_NUMBER != 8 # error "Need to add a 'case' statement here" # endif #endif Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/id.c URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/id.c?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/id.c (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/id.c Wed Nov 7 12:30:06 2018 @@ -591,7 +591,8 @@ svn_fs_fs__id_parse(const svn_fs_id_t ** svn_fs_id_t *id = id_parse(data, pool); if (id == NULL) return svn_error_createf(SVN_ERR_FS_MALFORMED_NODEREV_ID, NULL, - "Malformed node revision ID string"); + "Malformed node revision ID string '%s'", + data); *id_p = id; Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/index.c URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/index.c?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/index.c (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/index.c Wed Nov 7 12:30:06 2018 @@ -3191,9 +3191,9 @@ compare_p2l_entry_revision(const void *l const void *rhs) { const svn_fs_fs__p2l_entry_t *lhs_entry - =*(const svn_fs_fs__p2l_entry_t **)lhs; + =*(const svn_fs_fs__p2l_entry_t *const *)lhs; const svn_fs_fs__p2l_entry_t *rhs_entry - =*(const svn_fs_fs__p2l_entry_t **)rhs; + =*(const svn_fs_fs__p2l_entry_t *const *)rhs; if (lhs_entry->item.revision < rhs_entry->item.revision) return -1; Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/load-index.c URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/load-index.c?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/load-index.c (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/load-index.c Wed Nov 7 12:30:06 2018 @@ -83,9 +83,9 @@ compare_p2l_entry_revision(const void *l const void *rhs) { const svn_fs_fs__p2l_entry_t *lhs_entry - =*(const svn_fs_fs__p2l_entry_t **)lhs; + =*(const svn_fs_fs__p2l_entry_t *const *)lhs; const svn_fs_fs__p2l_entry_t *rhs_entry - =*(const svn_fs_fs__p2l_entry_t **)rhs; + =*(const svn_fs_fs__p2l_entry_t *const *)rhs; if (lhs_entry->offset < rhs_entry->offset) return -1; Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/low_level.c URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/low_level.c?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/low_level.c (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/low_level.c Wed Nov 7 12:30:06 2018 @@ -670,7 +670,7 @@ svn_fs_fs__write_changes(svn_stream_t *s } if (terminate_list) - svn_stream_puts(stream, "\n"); + SVN_ERR(svn_stream_puts(stream, "\n")); svn_pool_destroy(iterpool); @@ -741,6 +741,9 @@ read_header_block(apr_hash_t **headers, return SVN_NO_ERROR; } +/* ### Ouch! The implementation of this function currently modifies + ### the input string when tokenizing it (so the input cannot be + ### used after that). */ svn_error_t * svn_fs_fs__parse_representation(representation_t **rep_p, svn_stringbuf_t *text, @@ -811,13 +814,21 @@ svn_fs_fs__parse_representation(represen if (str == NULL) return SVN_NO_ERROR; - /* Read the SHA1 hash. */ - if (strlen(str) != (APR_SHA1_DIGESTSIZE * 2)) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("Malformed text representation offset line in node-rev")); + /* Is the SHA1 hash present? */ + if (str[0] == '-' && str[1] == 0) + { + checksum = NULL; + } + else + { + /* Read the SHA1 hash. */ + if (strlen(str) != (APR_SHA1_DIGESTSIZE * 2)) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed text representation offset line in node-rev")); - SVN_ERR(svn_checksum_parse_hex(&checksum, svn_checksum_sha1, str, - scratch_pool)); + SVN_ERR(svn_checksum_parse_hex(&checksum, svn_checksum_sha1, str, + scratch_pool)); + } /* We do have a valid SHA1 but it might be all 0. We cannot be sure where that came from (Alas! legacy), so let's not @@ -829,21 +840,36 @@ svn_fs_fs__parse_representation(represen if (checksum) memcpy(rep->sha1_digest, checksum->digest, sizeof(rep->sha1_digest)); - /* Read the uniquifier. */ - str = svn_cstring_tokenize("/", &string); + str = svn_cstring_tokenize(" ", &string); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Malformed text representation offset line in node-rev")); - SVN_ERR(svn_fs_fs__id_txn_parse(&rep->uniquifier.noderev_txn_id, str)); + /* Is the uniquifier present? */ + if (str[0] == '-' && str[1] == 0) + { + end = string; + } + else + { + char *substring = str; - str = svn_cstring_tokenize(" ", &string); - if (str == NULL || *str != '_') - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("Malformed text representation offset line in node-rev")); + /* Read the uniquifier. */ + str = svn_cstring_tokenize("/", &substring); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed text representation offset line in node-rev")); + + SVN_ERR(svn_fs_fs__id_txn_parse(&rep->uniquifier.noderev_txn_id, str)); + + str = svn_cstring_tokenize(" ", &substring); + if (str == NULL || *str != '_') + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Malformed text representation offset line in node-rev")); - ++str; - rep->uniquifier.number = svn__base36toui64(&end, str); + ++str; + rep->uniquifier.number = svn__base36toui64(&end, str); + } if (*end) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, @@ -1034,25 +1060,37 @@ svn_fs_fs__read_noderev(node_revision_t } /* Return a textual representation of the DIGEST of given KIND. - * If IS_NULL is TRUE, no digest is available. * Allocate the result in RESULT_POOL. */ static const char * format_digest(const unsigned char *digest, svn_checksum_kind_t kind, - svn_boolean_t is_null, apr_pool_t *result_pool) { svn_checksum_t checksum; checksum.digest = digest; checksum.kind = kind; - if (is_null) - return "(null)"; - return svn_checksum_to_cstring_display(&checksum, result_pool); } +/* Return a textual representation of the uniquifier represented + * by NODEREV_TXN_ID and NUMBER. Use POOL for the allocations. + */ +static const char * +format_uniquifier(const svn_fs_fs__id_part_t *noderev_txn_id, + apr_uint64_t number, + apr_pool_t *pool) +{ + char buf[SVN_INT64_BUFFER_SIZE]; + const char *txn_id_str; + + txn_id_str = svn_fs_fs__id_txn_unparse(noderev_txn_id, pool); + svn__ui64tobase36(buf, number); + + return apr_psprintf(pool, "%s/_%s", txn_id_str, buf); +} + svn_stringbuf_t * svn_fs_fs__unparse_representation(representation_t *rep, int format, @@ -1060,32 +1098,80 @@ svn_fs_fs__unparse_representation(repres apr_pool_t *result_pool, apr_pool_t *scratch_pool) { - char buffer[SVN_INT64_BUFFER_SIZE]; + svn_stringbuf_t *str; + const char *sha1_str; + const char *uniquifier_str; + if (svn_fs_fs__id_txn_used(&rep->txn_id) && mutable_rep_truncated) return svn_stringbuf_ncreate("-1", 2, result_pool); - if (format < SVN_FS_FS__MIN_REP_SHARING_FORMAT || !rep->has_sha1) - return svn_stringbuf_createf - (result_pool, "%ld %" APR_UINT64_T_FMT " %" SVN_FILESIZE_T_FMT - " %" SVN_FILESIZE_T_FMT " %s", - rep->revision, rep->item_index, rep->size, - rep->expanded_size, - format_digest(rep->md5_digest, svn_checksum_md5, FALSE, - scratch_pool)); - - svn__ui64tobase36(buffer, rep->uniquifier.number); - return svn_stringbuf_createf - (result_pool, "%ld %" APR_UINT64_T_FMT " %" SVN_FILESIZE_T_FMT - " %" SVN_FILESIZE_T_FMT " %s %s %s/_%s", - rep->revision, rep->item_index, rep->size, - rep->expanded_size, - format_digest(rep->md5_digest, svn_checksum_md5, - FALSE, scratch_pool), - format_digest(rep->sha1_digest, svn_checksum_sha1, - !rep->has_sha1, scratch_pool), - svn_fs_fs__id_txn_unparse(&rep->uniquifier.noderev_txn_id, - scratch_pool), - buffer); + /* Format of the string: + [] [] + */ + str = svn_stringbuf_createf( + result_pool, + "%ld" + " %" APR_UINT64_T_FMT + " %" SVN_FILESIZE_T_FMT + " %" SVN_FILESIZE_T_FMT + " %s", + rep->revision, + rep->item_index, + rep->size, + rep->expanded_size, + format_digest(rep->md5_digest, svn_checksum_md5, scratch_pool)); + + /* Compatibility: these formats don't understand and . */ + if (format < SVN_FS_FS__MIN_REP_SHARING_FORMAT) + return str; + + if (format < SVN_FS_FS__MIN_REP_STRING_OPTIONAL_VALUES_FORMAT) + { + /* Compatibility: these formats can only have and + present simultaneously, or don't have them at all. */ + if (rep->has_sha1) + { + sha1_str = format_digest(rep->sha1_digest, svn_checksum_sha1, + scratch_pool); + uniquifier_str = format_uniquifier(&rep->uniquifier.noderev_txn_id, + rep->uniquifier.number, + scratch_pool); + svn_stringbuf_appendbyte(str, ' '); + svn_stringbuf_appendcstr(str, sha1_str); + svn_stringbuf_appendbyte(str, ' '); + svn_stringbuf_appendcstr(str, uniquifier_str); + } + return str; + } + + /* The most recent formats support optional and values. */ + if (rep->has_sha1) + { + sha1_str = format_digest(rep->sha1_digest, svn_checksum_sha1, + scratch_pool); + } + else + sha1_str = "-"; + + if (rep->uniquifier.number == 0 && + rep->uniquifier.noderev_txn_id.number == 0 && + rep->uniquifier.noderev_txn_id.revision == 0) + { + uniquifier_str = "-"; + } + else + { + uniquifier_str = format_uniquifier(&rep->uniquifier.noderev_txn_id, + rep->uniquifier.number, + scratch_pool); + } + + svn_stringbuf_appendbyte(str, ' '); + svn_stringbuf_appendcstr(str, sha1_str); + svn_stringbuf_appendbyte(str, ' '); + svn_stringbuf_appendcstr(str, uniquifier_str); + + return str; } Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/pack.c URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/pack.c?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/pack.c (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/pack.c Wed Nov 7 12:30:06 2018 @@ -277,6 +277,12 @@ initialize_pack_context(pack_context_t * context->end_rev = shard_rev; context->shard_end_rev = shard_rev + ffd->max_files_per_dir; + /* the pool used for temp structures */ + context->info_pool = svn_pool_create(pool); + context->paths = svn_prefix_tree__create(context->info_pool); + + context->flush_to_disk = flush_to_disk; + /* Create the new directory and pack file. */ context->shard_dir = shard_dir; context->pack_file_dir = pack_file_dir; @@ -304,15 +310,18 @@ initialize_pack_context(pack_context_t * context->changes = apr_array_make(pool, max_items, sizeof(svn_fs_fs__p2l_entry_t *)); SVN_ERR(svn_io_open_unique_file3(&context->changes_file, NULL, temp_dir, - svn_io_file_del_on_close, pool, pool)); + svn_io_file_del_on_close, + context->info_pool, pool)); context->file_props = apr_array_make(pool, max_items, sizeof(svn_fs_fs__p2l_entry_t *)); SVN_ERR(svn_io_open_unique_file3(&context->file_props_file, NULL, temp_dir, - svn_io_file_del_on_close, pool, pool)); + svn_io_file_del_on_close, + context->info_pool, pool)); context->dir_props = apr_array_make(pool, max_items, sizeof(svn_fs_fs__p2l_entry_t *)); SVN_ERR(svn_io_open_unique_file3(&context->dir_props_file, NULL, temp_dir, - svn_io_file_del_on_close, pool, pool)); + svn_io_file_del_on_close, + context->info_pool, pool)); /* noderev and representation item bucket */ context->rev_offsets = apr_array_make(pool, max_revs, sizeof(int)); @@ -325,12 +334,6 @@ initialize_pack_context(pack_context_t * SVN_ERR(svn_io_open_unique_file3(&context->reps_file, NULL, temp_dir, svn_io_file_del_on_close, pool, pool)); - /* the pool used for temp structures */ - context->info_pool = svn_pool_create(pool); - context->paths = svn_prefix_tree__create(context->info_pool); - - context->flush_to_disk = flush_to_disk; - return SVN_NO_ERROR; } @@ -2064,9 +2067,9 @@ pack_body(void *baton, if (fully_packed) { if (pb->notify_func) - (*pb->notify_func)(pb->notify_baton, - ffd->min_unpacked_rev / ffd->max_files_per_dir, - svn_fs_pack_notify_noop, pool); + SVN_ERR(pb->notify_func(pb->notify_baton, + ffd->min_unpacked_rev / ffd->max_files_per_dir, + svn_fs_pack_notify_noop, pool)); return SVN_NO_ERROR; } @@ -2119,7 +2122,7 @@ svn_fs_fs__pack(svn_fs_t *fs, if (!ffd->max_files_per_dir) { if (notify_func) - (*notify_func)(notify_baton, -1, svn_fs_pack_notify_noop, pool); + SVN_ERR(notify_func(notify_baton, -1, svn_fs_pack_notify_noop, pool)); return SVN_NO_ERROR; } @@ -2129,9 +2132,9 @@ svn_fs_fs__pack(svn_fs_t *fs, if (fully_packed) { if (notify_func) - (*notify_func)(notify_baton, - ffd->min_unpacked_rev / ffd->max_files_per_dir, - svn_fs_pack_notify_noop, pool); + SVN_ERR(notify_func(notify_baton, + ffd->min_unpacked_rev / ffd->max_files_per_dir, + svn_fs_pack_notify_noop, pool)); return SVN_NO_ERROR; } Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/recovery.c URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/recovery.c?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/recovery.c (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/recovery.c Wed Nov 7 12:30:06 2018 @@ -471,9 +471,15 @@ recover_body(void *baton, apr_pool_t *po } /* Prune younger-than-(newfound-youngest) revisions from the rep - cache if sharing is enabled taking care not to create the cache - if it does not exist. */ - if (ffd->rep_sharing_allowed) + cache, taking care not to create the cache if it does not exist. + + We do this whenever rep-cache.db exists, whether it's currently enabled + or not, to prevent a data loss that could result from having revisions + created after this 'recover' operation referring to rep-cache.db rows + that were created before the recover and that point to revisions younger- + than-(newfound-youngest). + */ + if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) { svn_boolean_t rep_cache_exists; Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/rep-cache-db.sql URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/rep-cache-db.sql?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/rep-cache-db.sql (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/rep-cache-db.sql Wed Nov 7 12:30:06 2018 @@ -21,7 +21,7 @@ * ==================================================================== */ --- STMT_CREATE_SCHEMA +-- STMT_CREATE_SCHEMA_V1 /* A table mapping representation hashes to locations in a rev file. */ CREATE TABLE rep_cache ( hash TEXT NOT NULL PRIMARY KEY, @@ -33,36 +33,63 @@ CREATE TABLE rep_cache ( PRAGMA USER_VERSION = 1; +-- STMT_CREATE_SCHEMA_V2 +/* A table mapping representation hashes to locations in a rev file. + Same as in V1 schema, except that it uses the `WITHOUT ROWID` optimization: + https://sqlite.org/withoutrowid.html + + Note that this optimization is only supported starting from SQLite version + 3.8.2 (2013-12-06). To keep compatibility with existing binaries, it is + only used for newer filesystem formats that were released together with + bumping the minimum required SQLite version. + */ +CREATE TABLE rep_cache ( + hash TEXT NOT NULL PRIMARY KEY, + revision INTEGER NOT NULL, + offset INTEGER NOT NULL, + size INTEGER NOT NULL, + expanded_size INTEGER NOT NULL + ) WITHOUT ROWID; + +PRAGMA USER_VERSION = 2; -- STMT_GET_REP +/* Works for both V1 and V2 schemas. */ SELECT revision, offset, size, expanded_size FROM rep_cache WHERE hash = ?1 -- STMT_SET_REP +/* Works for both V1 and V2 schemas. */ INSERT OR FAIL INTO rep_cache (hash, revision, offset, size, expanded_size) VALUES (?1, ?2, ?3, ?4, ?5) -- STMT_GET_REPS_FOR_RANGE +/* Works for both V1 and V2 schemas. */ SELECT hash, revision, offset, size, expanded_size FROM rep_cache WHERE revision >= ?1 AND revision <= ?2 -- STMT_GET_MAX_REV +/* Works for both V1 and V2 schemas. */ SELECT MAX(revision) FROM rep_cache -- STMT_DEL_REPS_YOUNGER_THAN_REV +/* Works for both V1 and V2 schemas. */ DELETE FROM rep_cache WHERE revision > ?1 /* An INSERT takes an SQLite reserved lock that prevents other writes but doesn't block reads. The incomplete transaction means that no permanent change is made to the database and the transaction is - removed when the database is closed. */ + removed when the database is closed. + + Works for both V1 and V2 schemas. */ -- STMT_LOCK_REP BEGIN TRANSACTION; INSERT INTO rep_cache VALUES ('dummy', 0, 0, 0, 0) -- STMT_UNLOCK_REP +/* Works for both V1 and V2 schemas. */ ROLLBACK TRANSACTION; Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/rep-cache.c URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/rep-cache.c?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/rep-cache.c (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/rep-cache.c Wed Nov 7 12:30:06 2018 @@ -36,9 +36,6 @@ #include "rep-cache-db.h" -/* A few magic values */ -#define REP_CACHE_SCHEMA_FORMAT 1 - REP_CACHE_DB_SQL_DECLARE_STATEMENTS(statements); @@ -102,13 +99,17 @@ open_rep_cache(void *baton, SVN_SQLITE__ERR_CLOSE(svn_sqlite__read_schema_version(&version, sdb, pool), sdb); - if (version < REP_CACHE_SCHEMA_FORMAT) + /* If we have an uninitialized database, go ahead and create the schema. */ + if (version <= 0) { - /* Must be 0 -- an uninitialized (no schema) database. Create - the schema. Results in schema version of 1. */ - SVN_SQLITE__ERR_CLOSE(svn_sqlite__exec_statements(sdb, - STMT_CREATE_SCHEMA), - sdb); + int stmt; + + if (ffd->format >= SVN_FS_FS__MIN_REP_CACHE_SCHEMA_V2_FORMAT) + stmt = STMT_CREATE_SCHEMA_V2; + else + stmt = STMT_CREATE_SCHEMA_V1; + + SVN_SQLITE__ERR_CLOSE(svn_sqlite__exec_statements(sdb, stmt), sdb); } /* This is used as a flag that the database is available so don't @@ -125,7 +126,10 @@ svn_fs_fs__open_rep_cache(svn_fs_t *fs, fs_fs_data_t *ffd = fs->fsap_data; svn_error_t *err = svn_atomic__init_once(&ffd->rep_cache_db_opened, open_rep_cache, fs, pool); - return svn_error_quick_wrap(err, _("Couldn't open rep-cache database")); + return svn_error_quick_wrapf(err, + _("Couldn't open rep-cache database '%s'"), + svn_dirent_local_style( + path_rep_cache_db(fs->path, pool), pool)); } svn_error_t * Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/revprops.c URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/revprops.c?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/revprops.c (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/revprops.c Wed Nov 7 12:30:06 2018 @@ -229,9 +229,12 @@ prepare_revprop_cache(svn_fs_t *fs, } /* Store the unparsed revprop hash CONTENT for REVISION in FS's revprop - * cache. Use SCRATCH_POOL for temporary allocations. */ + * cache. If CACHED is not NULL, set *CACHED if there already is such + * an entry and skip the cache write in that case. Use SCRATCH_POOL for + * temporary allocations. */ static svn_error_t * -cache_revprops(svn_fs_t *fs, +cache_revprops(svn_boolean_t *is_cached, + svn_fs_t *fs, svn_revnum_t revision, svn_string_t *content, apr_pool_t *scratch_pool) @@ -244,6 +247,14 @@ cache_revprops(svn_fs_t *fs, key.revision = revision; key.second = ffd->revprop_prefix; + if (is_cached) + { + SVN_ERR(svn_cache__has_key(is_cached, ffd->revprop_cache, &key, + scratch_pool)); + if (*is_cached) + return SVN_NO_ERROR; + } + SVN_ERR(svn_cache__set(ffd->revprop_cache, &key, content, scratch_pool)); return SVN_NO_ERROR; @@ -287,7 +298,7 @@ read_non_packed_revprop(apr_hash_t **pro SVN_ERR(parse_revprop(properties, fs, rev, as_string, pool, iterpool)); if (populate_cache) - SVN_ERR(cache_revprops(fs, rev, as_string, iterpool)); + SVN_ERR(cache_revprops(NULL, fs, rev, as_string, iterpool)); } svn_pool_clear(iterpool); @@ -453,12 +464,15 @@ parse_packed_revprops(svn_fs_t *fs, const char *header_end; apr_pool_t *iterpool = svn_pool_create(scratch_pool); + /* Initial value for the "Leaking bucket" pattern. */ + int bucket = 4; + /* decompress (even if the data is only "stored", there is still a * length header to remove) */ svn_stringbuf_t *compressed = revprops->packed_revprops; svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(result_pool); - SVN_ERR(svn__decompress(compressed->data, compressed->len, - uncompressed, APR_SIZE_MAX)); + SVN_ERR(svn__decompress_zlib(compressed->data, compressed->len, + uncompressed, APR_SIZE_MAX)); /* read first revision number and number of revisions in the pack */ stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); @@ -546,7 +560,30 @@ parse_packed_revprops(svn_fs_t *fs, } if (populate_cache) - SVN_ERR(cache_revprops(fs, revision, &serialized, iterpool)); + { + /* Adding all those revprops is expensive, in particular in a + * multi-threaded environment. There are situations where hit + * rates are low and revprops get evicted before re-using them. + * + * We try to detect thosse cases here. + * Only keep going while most (at least 2/3) aren't cached, yet. */ + svn_boolean_t already_cached; + SVN_ERR(cache_revprops(&already_cached, fs, revision, &serialized, + iterpool)); + + /* Stop populating the cache once we encountered too many entries + * already present relative to the numbers being added. */ + if (!already_cached) + { + ++bucket; + } + else + { + bucket -= 2; + if (bucket < 0) + populate_cache = FALSE; + } + } if (read_all) { @@ -900,11 +937,11 @@ repack_revprops(svn_fs_t *fs, SVN_ERR(svn_stream_close(stream)); /* compress / store the data */ - SVN_ERR(svn__compress(uncompressed->data, uncompressed->len, - compressed, - ffd->compress_packed_revprops - ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT - : SVN_DELTA_COMPRESSION_LEVEL_NONE)); + SVN_ERR(svn__compress_zlib(uncompressed->data, uncompressed->len, + compressed, + ffd->compress_packed_revprops + ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT + : SVN_DELTA_COMPRESSION_LEVEL_NONE)); /* finally, write the content to the target file, flush and close it */ SVN_ERR(svn_io_file_write_full(file, compressed->data, compressed->len, @@ -1308,8 +1345,8 @@ svn_fs_fs__copy_revprops(const char *pac SVN_ERR(svn_stream_close(pack_stream)); /* compress the content (or just store it for COMPRESSION_LEVEL 0) */ - SVN_ERR(svn__compress(uncompressed->data, uncompressed->len, - compressed, compression_level)); + SVN_ERR(svn__compress_zlib(uncompressed->data, uncompressed->len, + compressed, compression_level)); /* write the pack file content to disk */ SVN_ERR(svn_io_file_write_full(pack_file, compressed->data, compressed->len, Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/stats.c URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/stats.c?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/stats.c (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/stats.c Wed Nov 7 12:30:06 2018 @@ -617,6 +617,10 @@ read_noderev(query_t *query, svn_stream_t *stream = svn_stream_from_stringbuf(noderev_str, scratch_pool); SVN_ERR(svn_fs_fs__read_noderev(&noderev, stream, scratch_pool, scratch_pool)); + SVN_ERR(svn_fs_fs__fixup_expanded_size(query->fs, noderev->data_rep, + scratch_pool)); + SVN_ERR(svn_fs_fs__fixup_expanded_size(query->fs, noderev->prop_rep, + scratch_pool)); if (noderev->data_rep) { Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/structure URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/structure?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/structure (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/structure Wed Nov 7 12:30:06 2018 @@ -150,12 +150,14 @@ The formats are: Format 5, understood by Subversion 1.7-dev, never released Format 6, understood by Subversion 1.8 Format 7, understood by Subversion 1.9 + Format 8, understood by Subversion 1.10 The differences between the formats are: Delta representation in revision files - Format 1: svndiff0 only - Formats 2+: svndiff0 or svndiff1 + Format 1: svndiff0 only + Formats 2-7: svndiff0 or svndiff1 + Formats 8: svndiff0, svndiff1 or svndiff2 Format options Formats 1-2: none permitted @@ -566,6 +568,9 @@ defined: ### in formats >=4, also present: gives hex SHA1 digest of expanded rep see representation_t->uniquifier in fs.h + ### Starting from format 8, a special notation "-" + can be used for optional values that are not present + ( and ). cpath FS pathname node was created at copyfrom " " of copyfrom data copyroot " " of the root of this copy Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/temp_serializer.h URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/temp_serializer.h?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/temp_serializer.h (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/temp_serializer.h Wed Nov 7 12:30:06 2018 @@ -53,13 +53,16 @@ svn_fs_fs__noderev_deserialize(void *buf /** * Adds position information to the raw window data in WINDOW. */ -typedef struct +typedef struct svn_fs_fs__raw_cached_window_t { /* the (unprocessed) txdelta window byte sequence cached / to be cached */ svn_string_t window; /* the offset within the representation right after reading the window */ apr_off_t end_offset; + + /* svndiff version */ + int ver; } svn_fs_fs__raw_cached_window_t; /** @@ -86,7 +89,7 @@ svn_fs_fs__deserialize_raw_window(void * * #svn_txdelta_window_t is not sufficient for caching the data it * represents because data read process needs auxiliary information. */ -typedef struct +typedef struct svn_fs_fs__txdelta_cached_window_t { /* the txdelta window information cached / to be cached */ svn_txdelta_window_t *window; @@ -374,7 +377,7 @@ typedef struct svn_fs_fs__changes_list_t of elements in the list is a multiple of our block / range size. */ svn_boolean_t eol; - /* Array of #svn_fs_x__change_t * representing a consecutive sub-range of + /* Array of #svn_fs_fs__change_t * representing a consecutive sub-range of elements in a changed paths list. */ /* number of entries in the array */ Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/transaction.c URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/transaction.c?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/transaction.c (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/transaction.c Wed Nov 7 12:30:06 2018 @@ -442,7 +442,7 @@ get_writable_proto_rev(apr_file_t **file /* Now open the prototype revision file and seek to the end. */ err = svn_io_file_open(file, svn_fs_fs__path_txn_proto_rev(fs, txn_id, pool), - APR_WRITE | APR_BUFFERED, APR_OS_DEFAULT, + APR_READ | APR_WRITE | APR_BUFFERED, APR_OS_DEFAULT, pool); /* You might expect that we could dispense with the following seek @@ -2159,6 +2159,35 @@ rep_write_cleanup(void *data) return APR_SUCCESS; } +static void +txdelta_to_svndiff(svn_txdelta_window_handler_t *handler, + void **handler_baton, + svn_stream_t *output, + svn_fs_t *fs, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + int svndiff_version; + + if (ffd->delta_compression_type == compression_type_lz4) + { + SVN_ERR_ASSERT_NO_RETURN(ffd->format >= SVN_FS_FS__MIN_SVNDIFF2_FORMAT); + svndiff_version = 2; + } + else if (ffd->delta_compression_type == compression_type_zlib) + { + SVN_ERR_ASSERT_NO_RETURN(ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT); + svndiff_version = 1; + } + else + { + svndiff_version = 0; + } + + svn_txdelta_to_svndiff3(handler, handler_baton, output, svndiff_version, + ffd->delta_compression_level, pool); +} + /* Get a rep_write_baton and store it in *WB_P for the representation indicated by NODEREV in filesystem FS. Perform allocations in POOL. Only appropriate for file contents, not for props or @@ -2175,8 +2204,6 @@ rep_write_get_baton(struct rep_write_bat svn_stream_t *source; svn_txdelta_window_handler_t wh; void *whb; - fs_fs_data_t *ffd = fs->fsap_data; - int diff_version = ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT ? 1 : 0; svn_fs_fs__rep_header_t header = { 0 }; b = apr_pcalloc(pool, sizeof(*b)); @@ -2232,12 +2259,7 @@ rep_write_get_baton(struct rep_write_bat apr_pool_cleanup_null); /* Prepare to write the svndiff data. */ - svn_txdelta_to_svndiff3(&wh, - &whb, - b->rep_stream, - diff_version, - ffd->delta_compression_level, - pool); + txdelta_to_svndiff(&wh, &whb, b->rep_stream, fs, pool); b->delta_stream = svn_txdelta_target_push(wh, whb, source, b->scratch_pool); @@ -2253,6 +2275,11 @@ rep_write_get_baton(struct rep_write_bat there may be new duplicate representations within the same uncommitted revision, those can be passed in REPS_HASH (maps a sha1 digest onto representation_t*), otherwise pass in NULL for REPS_HASH. + + The content of both representations will be compared, taking REP's content + from FILE at OFFSET. Only if they actually match, will *OLD_REP not be + NULL. + Use RESULT_POOL for *OLD_REP allocations and SCRATCH_POOL for temporaries. The lifetime of *OLD_REP is limited by both, RESULT_POOL and REP lifetime. */ @@ -2260,6 +2287,8 @@ static svn_error_t * get_shared_rep(representation_t **old_rep, svn_fs_t *fs, representation_t *rep, + apr_file_t *file, + apr_off_t offset, apr_hash_t *reps_hash, apr_pool_t *result_pool, apr_pool_t *scratch_pool) @@ -2267,6 +2296,10 @@ get_shared_rep(representation_t **old_re svn_error_t *err; fs_fs_data_t *ffd = fs->fsap_data; + svn_checksum_t checksum; + checksum.digest = rep->sha1_digest; + checksum.kind = svn_checksum_sha1; + /* Return NULL, if rep sharing has been disabled. */ *old_rep = NULL; if (!ffd->rep_sharing_allowed) @@ -2287,9 +2320,6 @@ get_shared_rep(representation_t **old_re /* If we haven't found anything yet, try harder and consult our DB. */ if (*old_rep == NULL) { - svn_checksum_t checksum; - checksum.digest = rep->sha1_digest; - checksum.kind = svn_checksum_sha1; err = svn_fs_fs__get_rep_reference(old_rep, fs, &checksum, result_pool); /* ### Other error codes that we shouldn't mask out? */ if (err == SVN_NO_ERROR) @@ -2355,10 +2385,6 @@ get_shared_rep(representation_t **old_re Because not sharing reps is always a safe option, terminating the request would be inappropriate. */ - svn_checksum_t checksum; - checksum.digest = rep->sha1_digest; - checksum.kind = svn_checksum_sha1; - err = svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, "Rep size %s mismatches rep-cache.db value %s " "for SHA1 %s.\n" @@ -2388,6 +2414,72 @@ get_shared_rep(representation_t **old_re (*old_rep)->uniquifier = rep->uniquifier; } + /* If we (very likely) found a matching representation, compare the actual + * contents such that we can be sure that no rep-cache.db corruption or + * hash collision produced a false positive. */ + if (*old_rep) + { + apr_off_t old_position; + svn_stream_t *contents; + svn_stream_t *old_contents; + svn_boolean_t same; + + /* The existing representation may itself be part of the current + * transaction. In that case, it may be in different stages of + * the commit finalization process. + * + * OLD_REP_NORM is the same as that OLD_REP but it is assigned + * explicitly to REP's transaction if OLD_REP does not point + * to an already committed revision. This then prevents the + * revision lookup and the txn data will be accessed. + */ + representation_t old_rep_norm = **old_rep; + if ( !SVN_IS_VALID_REVNUM(old_rep_norm.revision) + || old_rep_norm.revision > ffd->youngest_rev_cache) + old_rep_norm.txn_id = rep->txn_id; + + /* Make sure we can later restore FILE's current position. */ + SVN_ERR(svn_io_file_get_offset(&old_position, file, scratch_pool)); + + /* Compare the two representations. + * Note that the stream comparison might also produce MD5 checksum + * errors or other failures in case of SHA1 collisions. */ + SVN_ERR(svn_fs_fs__get_contents_from_file(&contents, fs, rep, file, + offset, scratch_pool)); + SVN_ERR(svn_fs_fs__get_contents(&old_contents, fs, &old_rep_norm, + FALSE, scratch_pool)); + err = svn_stream_contents_same2(&same, contents, old_contents, + scratch_pool); + + /* A mismatch should be extremely rare. + * If it does happen, reject the commit. */ + if (!same || err) + { + /* SHA1 collision or worse. */ + svn_stringbuf_t *old_rep_str + = svn_fs_fs__unparse_representation(*old_rep, + ffd->format, FALSE, + scratch_pool, + scratch_pool); + svn_stringbuf_t *rep_str + = svn_fs_fs__unparse_representation(rep, + ffd->format, FALSE, + scratch_pool, + scratch_pool); + const char *checksum__str + = svn_checksum_to_cstring_display(&checksum, scratch_pool); + + return svn_error_createf(SVN_ERR_FS_AMBIGUOUS_CHECKSUM_REP, + err, "SHA1 of reps '%s' and '%s' " + "matches (%s) but contents differ", + old_rep_str->data, rep_str->data, + checksum__str); + } + + /* Restore FILE's read / write position. */ + SVN_ERR(svn_io_file_seek(file, APR_SET, &old_position, scratch_pool)); + } + return SVN_NO_ERROR; } @@ -2449,8 +2541,8 @@ rep_write_contents_close(void *baton) /* Check and see if we already have a representation somewhere that's identical to the one we just wrote out. */ - SVN_ERR(get_shared_rep(&old_rep, b->fs, rep, NULL, b->result_pool, - b->scratch_pool)); + SVN_ERR(get_shared_rep(&old_rep, b->fs, rep, b->file, b->rep_offset, NULL, + b->result_pool, b->scratch_pool)); if (old_rep) { @@ -2605,6 +2697,8 @@ svn_fs_fs__set_proplist(svn_fs_t *fs, { noderev->prop_rep = apr_pcalloc(pool, sizeof(*noderev->prop_rep)); noderev->prop_rep->txn_id = *svn_fs_fs__id_txn_id(noderev->id); + SVN_ERR(set_uniquifier(fs, noderev->prop_rep, pool)); + noderev->prop_rep->revision = SVN_INVALID_REVNUM; SVN_ERR(svn_fs_fs__put_node_revision(fs, noderev->id, noderev, FALSE, pool)); } @@ -2678,11 +2772,16 @@ write_directory_to_stream(svn_stream_t * /* Write out the COLLECTION as a text representation to file FILE using WRITER. In the process, record position, the total size of the dump and MD5 as well as SHA1 in REP. Add the representation of type ITEM_TYPE to - the indexes if necessary. If rep sharing has been enabled and REPS_HASH - is not NULL, it will be used in addition to the on-disk cache to find - earlier reps with the same content. When such existing reps can be - found, we will truncate the one just written from the file and return - the existing rep. Perform temporary allocations in SCRATCH_POOL. */ + the indexes if necessary. + + If ALLOW_REP_SHARING is FALSE, rep-sharing will not be used, regardless + of any other option and rep-sharing settings. If rep sharing has been + enabled and REPS_HASH is not NULL, it will be used in addition to the + on-disk cache to find earlier reps with the same content. If such + existing reps can be found, we will truncate the one just written from + the file and return the existing rep. + + Perform temporary allocations in SCRATCH_POOL. */ static svn_error_t * write_container_rep(representation_t *rep, apr_file_t *file, @@ -2690,13 +2789,13 @@ write_container_rep(representation_t *re collection_writer_t writer, svn_fs_t *fs, apr_hash_t *reps_hash, + svn_boolean_t allow_rep_sharing, apr_uint32_t item_type, apr_pool_t *scratch_pool) { svn_stream_t *stream; struct write_container_baton *whb; svn_checksum_ctx_t *fnv1a_checksum_ctx; - representation_t *old_rep; apr_off_t offset = 0; SVN_ERR(svn_io_file_get_offset(&offset, file, scratch_pool)); @@ -2724,20 +2823,27 @@ write_container_rep(representation_t *re /* Store the results. */ SVN_ERR(digests_final(rep, whb->md5_ctx, whb->sha1_ctx, scratch_pool)); - /* Check and see if we already have a representation somewhere that's - identical to the one we just wrote out. */ + /* Update size info. */ rep->expanded_size = whb->size; - SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, scratch_pool, - scratch_pool)); + rep->size = whb->size; - if (old_rep) + /* Check and see if we already have a representation somewhere that's + identical to the one we just wrote out. */ + if (allow_rep_sharing) { - /* We need to erase from the protorev the data we just wrote. */ - SVN_ERR(svn_io_file_trunc(file, offset, scratch_pool)); + representation_t *old_rep; + SVN_ERR(get_shared_rep(&old_rep, fs, rep, file, offset, reps_hash, + scratch_pool, scratch_pool)); - /* Use the old rep for this content. */ - memcpy(rep, old_rep, sizeof (*rep)); - return SVN_NO_ERROR; + if (old_rep) + { + /* We need to erase from the protorev the data we just wrote. */ + SVN_ERR(svn_io_file_trunc(file, offset, scratch_pool)); + + /* Use the old rep for this content. */ + memcpy(rep, old_rep, sizeof (*rep)); + return SVN_NO_ERROR; + } } /* Write out our cosmetic end marker. */ @@ -2763,20 +2869,20 @@ write_container_rep(representation_t *re SVN_ERR(store_p2l_index_entry(fs, &rep->txn_id, &entry, scratch_pool)); } - /* update the representation */ - rep->size = whb->size; - return SVN_NO_ERROR; } /* Write out the COLLECTION pertaining to the NODEREV in FS as a deltified text representation to file FILE using WRITER. In the process, record the total size and the md5 digest in REP and add the representation of type - ITEM_TYPE to the indexes if necessary. If rep sharing has been enabled and - REPS_HASH is not NULL, it will be used in addition to the on-disk cache to - find earlier reps with the same content. When such existing reps can be - found, we will truncate the one just written from the file and return the - existing rep. + ITEM_TYPE to the indexes if necessary. + + If ALLOW_REP_SHARING is FALSE, rep-sharing will not be used, regardless + of any other option and rep-sharing settings. If rep sharing has been + enabled and REPS_HASH is not NULL, it will be used in addition to the + on-disk cache to find earlier reps with the same content. If such + existing reps can be found, we will truncate the one just written from + the file and return the existing rep. If ITEM_TYPE is IS_PROPS equals SVN_FS_FS__ITEM_TYPE_*_PROPS, assume that we want to a props representation as the base for our delta. @@ -2790,6 +2896,7 @@ write_container_delta_rep(representation svn_fs_t *fs, node_revision_t *noderev, apr_hash_t *reps_hash, + svn_boolean_t allow_rep_sharing, apr_uint32_t item_type, apr_pool_t *scratch_pool) { @@ -2799,7 +2906,6 @@ write_container_delta_rep(representation svn_stream_t *file_stream; svn_stream_t *stream; representation_t *base_rep; - representation_t *old_rep; svn_checksum_ctx_t *fnv1a_checksum_ctx; svn_stream_t *source; svn_fs_fs__rep_header_t header = { 0 }; @@ -2809,8 +2915,6 @@ write_container_delta_rep(representation apr_off_t offset = 0; struct write_container_baton *whb; - fs_fs_data_t *ffd = fs->fsap_data; - int diff_version = ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT ? 1 : 0; svn_boolean_t is_props = (item_type == SVN_FS_FS__ITEM_TYPE_FILE_PROPS) || (item_type == SVN_FS_FS__ITEM_TYPE_DIR_PROPS); @@ -2843,12 +2947,7 @@ write_container_delta_rep(representation SVN_ERR(svn_io_file_get_offset(&delta_start, file, scratch_pool)); /* Prepare to write the svndiff data. */ - svn_txdelta_to_svndiff3(&diff_wh, - &diff_whb, - file_stream, - diff_version, - ffd->delta_compression_level, - scratch_pool); + txdelta_to_svndiff(&diff_wh, &diff_whb, file_stream, fs, scratch_pool); whb = apr_pcalloc(scratch_pool, sizeof(*whb)); whb->stream = svn_txdelta_target_push(diff_wh, diff_whb, source, @@ -2868,24 +2967,31 @@ write_container_delta_rep(representation /* Store the results. */ SVN_ERR(digests_final(rep, whb->md5_ctx, whb->sha1_ctx, scratch_pool)); - /* Check and see if we already have a representation somewhere that's - identical to the one we just wrote out. */ + /* Update size info. */ + SVN_ERR(svn_io_file_get_offset(&rep_end, file, scratch_pool)); + rep->size = rep_end - delta_start; rep->expanded_size = whb->size; - SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, scratch_pool, - scratch_pool)); - if (old_rep) + /* Check and see if we already have a representation somewhere that's + identical to the one we just wrote out. */ + if (allow_rep_sharing) { - /* We need to erase from the protorev the data we just wrote. */ - SVN_ERR(svn_io_file_trunc(file, offset, scratch_pool)); + representation_t *old_rep; + SVN_ERR(get_shared_rep(&old_rep, fs, rep, file, offset, reps_hash, + scratch_pool, scratch_pool)); - /* Use the old rep for this content. */ - memcpy(rep, old_rep, sizeof (*rep)); - return SVN_NO_ERROR; + if (old_rep) + { + /* We need to erase from the protorev the data we just wrote. */ + SVN_ERR(svn_io_file_trunc(file, offset, scratch_pool)); + + /* Use the old rep for this content. */ + memcpy(rep, old_rep, sizeof (*rep)); + return SVN_NO_ERROR; + } } /* Write out our cosmetic end marker. */ - SVN_ERR(svn_io_file_get_offset(&rep_end, file, scratch_pool)); SVN_ERR(svn_stream_puts(file_stream, "ENDREP\n")); SVN_ERR(allocate_item_index(&rep->item_index, fs, &rep->txn_id, @@ -2908,9 +3014,6 @@ write_container_delta_rep(representation SVN_ERR(store_p2l_index_entry(fs, &rep->txn_id, &entry, scratch_pool)); } - /* update the representation */ - rep->size = rep_end - delta_start; - return SVN_NO_ERROR; } @@ -3095,13 +3198,14 @@ write_final_rev(const svn_fs_id_t **new_ SVN_ERR(write_container_delta_rep(noderev->data_rep, file, entries, write_directory_to_stream, - fs, noderev, NULL, + fs, noderev, NULL, FALSE, SVN_FS_FS__ITEM_TYPE_DIR_REP, pool)); else SVN_ERR(write_container_rep(noderev->data_rep, file, entries, write_directory_to_stream, fs, NULL, - SVN_FS_FS__ITEM_TYPE_DIR_REP, pool)); + FALSE, SVN_FS_FS__ITEM_TYPE_DIR_REP, + pool)); reset_txn_in_rep(noderev->data_rep); @@ -3157,17 +3261,18 @@ write_final_rev(const svn_fs_id_t **new_ ? SVN_FS_FS__ITEM_TYPE_DIR_PROPS : SVN_FS_FS__ITEM_TYPE_FILE_PROPS; SVN_ERR(svn_fs_fs__get_proplist(&proplist, fs, noderev, pool)); - + noderev->prop_rep->txn_id = *txn_id; + SVN_ERR(set_uniquifier(fs, noderev->prop_rep, pool)); noderev->prop_rep->revision = rev; if (ffd->deltify_properties) SVN_ERR(write_container_delta_rep(noderev->prop_rep, file, proplist, write_hash_to_stream, fs, noderev, - reps_hash, item_type, pool)); + reps_hash, TRUE, item_type, pool)); else SVN_ERR(write_container_rep(noderev->prop_rep, file, proplist, write_hash_to_stream, fs, reps_hash, - item_type, pool)); + TRUE, item_type, pool)); reset_txn_in_rep(noderev->prop_rep); } @@ -3226,13 +3331,28 @@ write_final_rev(const svn_fs_id_t **new_ } } - /* don't serialize SHA1 for dirs to disk (waste of space) */ + /* don't serialize SHA1 for dir content to disk (waste of space) */ + /* ### Could clients record bogus last-changed-revisions (issue #4700)? */ if (noderev->data_rep && noderev->kind == svn_node_dir) noderev->data_rep->has_sha1 = FALSE; - /* don't serialize SHA1 for props to disk (waste of space) */ - if (noderev->prop_rep) - noderev->prop_rep->has_sha1 = FALSE; + /* Compatibility: while we don't need to serialize SHA1 for props (it is + not used), older formats can only have representation strings that either + have both the SHA1 value *and* the uniquifier, or don't have them at all. + For such formats, both values get written to the disk only if the SHA1 + is present. + + We cannot omit the uniquifier, as doing so breaks svn_fs_props_changed() + for properties with shared representations, see issues #4623 and #4700. + Therefore, we skip writing SHA1, but only for the newer formats where + this dependency is untied and we can write the uniquifier to the disk + without the SHA1. + */ + if (ffd->format >= SVN_FS_FS__MIN_REP_STRING_OPTIONAL_VALUES_FORMAT && + noderev->prop_rep) + { + noderev->prop_rep->has_sha1 = FALSE; + } /* Workaround issue #4031: is-fresh-txn-root in revision files. */ noderev->is_fresh_txn_root = FALSE; @@ -3334,11 +3454,10 @@ write_final_changed_path_info(apr_off_t Intended to be called as the very last step in a commit before 'current' is bumped. This implies that we are holding the write lock. */ static svn_error_t * -verify_as_revision_before_current_plus_plus(svn_fs_t *fs, - svn_revnum_t new_rev, - apr_pool_t *pool) +verify_before_commit(svn_fs_t *fs, + svn_revnum_t new_rev, + apr_pool_t *pool) { -#ifdef SVN_DEBUG fs_fs_data_t *ffd = fs->fsap_data; svn_fs_t *ft; /* fs++ == ft */ svn_fs_root_t *root; @@ -3368,7 +3487,6 @@ verify_as_revision_before_current_plus_p SVN_ERR_ASSERT(root->is_txn_root == FALSE && root->rev == new_rev); SVN_ERR_ASSERT(ft_ffd->youngest_rev_cache == new_rev); SVN_ERR(svn_fs_fs__verify_root(root, pool)); -#endif /* SVN_DEBUG */ return SVN_NO_ERROR; } @@ -3788,8 +3906,13 @@ commit_body(void *baton, apr_pool_t *poo SVN_ERR(write_final_revprop(revprop_filename, old_rev_filename, cb->txn, ffd->flush_to_disk, pool)); + /* Run paranoia checks. */ + if (ffd->verify_before_commit) + { + SVN_ERR(verify_before_commit(cb->fs, new_rev, pool)); + } + /* Update the 'current' file. */ - SVN_ERR(verify_as_revision_before_current_plus_plus(cb->fs, new_rev, pool)); SVN_ERR(write_final_current(cb->fs, txn_id, new_rev, start_node_id, start_copy_id, pool)); Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/tree.c URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/tree.c?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/tree.c (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/tree.c Wed Nov 7 12:30:06 2018 @@ -1143,8 +1143,28 @@ open_path(parent_path_t **parent_path_p, /* The path isn't finished yet; we'd better be in a directory. */ if (svn_fs_fs__dag_node_kind(child) != svn_node_dir) - SVN_ERR_W(SVN_FS__ERR_NOT_DIRECTORY(fs, path_so_far->data), - apr_psprintf(iterpool, _("Failure opening '%s'"), path)); + { + const char *msg; + + /* Since this is not a directory and we are looking for some + sub-path, that sub-path will not exist. That will be o.k., + if we are just here to check for the path's existence. */ + if (flags & open_path_allow_null) + { + parent_path = NULL; + break; + } + + /* It's really a problem ... */ + msg = root->is_txn_root + ? apr_psprintf(iterpool, + _("Failure opening '%s' in transaction '%s'"), + path, root->txn) + : apr_psprintf(iterpool, + _("Failure opening '%s' in revision %ld"), + path, root->rev); + SVN_ERR_W(SVN_FS__ERR_NOT_DIRECTORY(fs, path_so_far->data), msg); + } rest = next; here = child; @@ -4122,21 +4142,19 @@ assemble_history(svn_fs_t *fs, /* DIR_DAG is a directory DAG node which has mergeinfo in its descendants. This function iterates over its children. For each - child with immediate mergeinfo, it adds its mergeinfo to - RESULT_CATALOG. appropriate arguments. For each child with - descendants with mergeinfo, it recurses. Note that it does *not* - call the action on the path for DIR_DAG itself. - - POOL is used for temporary allocations, including the mergeinfo - hashes passed to actions; RESULT_POOL is used for the mergeinfo added - to RESULT_CATALOG. + child with immediate mergeinfo, call RECEIVER with it and BATON. + For each child with descendants with mergeinfo, it recurses. Note + that it does *not* call the action on the path for DIR_DAG itself. + + SCRATCH_POOL is used for temporary allocations, including the mergeinfo + hashes passed to actions. */ static svn_error_t * crawl_directory_dag_for_mergeinfo(svn_fs_root_t *root, const char *this_path, dag_node_t *dir_dag, - svn_mergeinfo_catalog_t result_catalog, - apr_pool_t *result_pool, + svn_fs_mergeinfo_receiver_t receiver, + void *baton, apr_pool_t *scratch_pool) { apr_array_header_t *entries; @@ -4183,7 +4201,7 @@ crawl_directory_dag_for_mergeinfo(svn_fs error. */ err = svn_mergeinfo_parse(&kid_mergeinfo, mergeinfo_string->data, - result_pool); + iterpool); if (err) { if (err->apr_err == SVN_ERR_MERGEINFO_PARSE_ERROR) @@ -4193,8 +4211,7 @@ crawl_directory_dag_for_mergeinfo(svn_fs } else { - svn_hash_sets(result_catalog, apr_pstrdup(result_pool, kid_path), - kid_mergeinfo); + SVN_ERR(receiver(kid_path, kid_mergeinfo, baton, iterpool)); } } @@ -4202,8 +4219,8 @@ crawl_directory_dag_for_mergeinfo(svn_fs SVN_ERR(crawl_directory_dag_for_mergeinfo(root, kid_path, kid_dag, - result_catalog, - result_pool, + receiver, + baton, iterpool)); } @@ -4387,14 +4404,13 @@ get_mergeinfo_for_path(svn_mergeinfo_t * return SVN_NO_ERROR; } -/* Adds mergeinfo for each descendant of PATH (but not PATH itself) - under ROOT to RESULT_CATALOG. Returned values are allocated in - RESULT_POOL; temporary values in POOL. */ +/* Invoke RECEIVER with BATON for each mergeinfo found on descendants of + PATH (but not PATH itself). Use SCRATCH_POOL for temporary values. */ static svn_error_t * -add_descendant_mergeinfo(svn_mergeinfo_catalog_t result_catalog, - svn_fs_root_t *root, +add_descendant_mergeinfo(svn_fs_root_t *root, const char *path, - apr_pool_t *result_pool, + svn_fs_mergeinfo_receiver_t receiver, + void *baton, apr_pool_t *scratch_pool) { dag_node_t *this_dag; @@ -4407,27 +4423,28 @@ add_descendant_mergeinfo(svn_mergeinfo_c SVN_ERR(crawl_directory_dag_for_mergeinfo(root, path, this_dag, - result_catalog, - result_pool, + receiver, + baton, scratch_pool)); return SVN_NO_ERROR; } -/* Get the mergeinfo for a set of paths, returned in - *MERGEINFO_CATALOG. Returned values are allocated in - POOL, while temporary values are allocated in a sub-pool. */ +/* Find all the mergeinfo for a set of PATHS under ROOT and report it + through RECEIVER with BATON. INHERITED, INCLUDE_DESCENDANTS and + ADJUST_INHERITED_MERGEINFO are the same as in the FS API. + + Allocate temporary values are allocated in SCRATCH_POOL. */ static svn_error_t * get_mergeinfos_for_paths(svn_fs_root_t *root, - svn_mergeinfo_catalog_t *mergeinfo_catalog, const apr_array_header_t *paths, svn_mergeinfo_inheritance_t inherit, svn_boolean_t include_descendants, svn_boolean_t adjust_inherited_mergeinfo, - apr_pool_t *result_pool, + svn_fs_mergeinfo_receiver_t receiver, + void *baton, apr_pool_t *scratch_pool) { - svn_mergeinfo_catalog_t result_catalog = svn_hash__make(result_pool); apr_pool_t *iterpool = svn_pool_create(scratch_pool); int i; @@ -4441,7 +4458,7 @@ get_mergeinfos_for_paths(svn_fs_root_t * err = get_mergeinfo_for_path(&path_mergeinfo, root, path, inherit, adjust_inherited_mergeinfo, - result_pool, iterpool); + iterpool, iterpool); if (err) { if (err->apr_err == SVN_ERR_MERGEINFO_PARSE_ERROR) @@ -4457,27 +4474,26 @@ get_mergeinfos_for_paths(svn_fs_root_t * } if (path_mergeinfo) - svn_hash_sets(result_catalog, path, path_mergeinfo); + SVN_ERR(receiver(path, path_mergeinfo, baton, iterpool)); if (include_descendants) - SVN_ERR(add_descendant_mergeinfo(result_catalog, root, path, - result_pool, scratch_pool)); + SVN_ERR(add_descendant_mergeinfo(root, path, receiver, baton, + iterpool)); } svn_pool_destroy(iterpool); - *mergeinfo_catalog = result_catalog; return SVN_NO_ERROR; } /* Implements svn_fs_get_mergeinfo. */ static svn_error_t * -fs_get_mergeinfo(svn_mergeinfo_catalog_t *catalog, - svn_fs_root_t *root, +fs_get_mergeinfo(svn_fs_root_t *root, const apr_array_header_t *paths, svn_mergeinfo_inheritance_t inherit, svn_boolean_t include_descendants, svn_boolean_t adjust_inherited_mergeinfo, - apr_pool_t *result_pool, + svn_fs_mergeinfo_receiver_t receiver, + void *baton, apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = root->fs->fsap_data; @@ -4495,11 +4511,11 @@ fs_get_mergeinfo(svn_mergeinfo_catalog_t SVN_FS_FS__MIN_MERGEINFO_FORMAT, root->fs->path, ffd->format); /* Retrieve a path -> mergeinfo hash mapping. */ - return get_mergeinfos_for_paths(root, catalog, paths, - inherit, + return get_mergeinfos_for_paths(root, paths, inherit, include_descendants, adjust_inherited_mergeinfo, - result_pool, scratch_pool); + receiver, baton, + scratch_pool); } Modified: subversion/branches/ra-git/subversion/libsvn_fs_fs/verify.c URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/subversion/libsvn_fs_fs/verify.c?rev=1846002&r1=1846001&r2=1846002&view=diff ============================================================================== --- subversion/branches/ra-git/subversion/libsvn_fs_fs/verify.c (original) +++ subversion/branches/ra-git/subversion/libsvn_fs_fs/verify.c Wed Nov 7 12:30:06 2018 @@ -673,15 +673,45 @@ compare_p2l_to_rev(svn_fs_t *fs, apr_off_t_toa(pool, offset), apr_off_t_toa(pool, entry->offset)); - /* empty sections must contain NUL bytes only */ + /* Check type <-> item dependencies. */ + + /* Entry types must be within the valid range. */ + if (entry->type >= SVN_FS_FS__ITEM_TYPE_ANY_REP) + return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, + NULL, + _("p2l index entry for revision r%ld" + " at offset %s contains invalid item" + " type %d"), + start, + apr_off_t_toa(pool, offset), + entry->type); + + /* There can be only one changes entry and that has a fixed type + * and item number. Its presence and parse-ability will be checked + * during later stages of the verification process. */ + if ( (entry->type == SVN_FS_FS__ITEM_TYPE_CHANGES) + != (entry->item.number == SVN_FS_FS__ITEM_INDEX_CHANGES)) + return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, + NULL, + _("p2l index entry for changes in" + " revision r%ld is item %ld of type" + " %d at offset %s"), + entry->item.revision, + entry->item.number, + entry->type, + apr_off_t_toa(pool, offset)); + + /* Check contents. */ if (entry->type == SVN_FS_FS__ITEM_TYPE_UNUSED) { - /* skip filler entry at the end of the p2l index */ + /* Empty sections must contain NUL bytes only. + * Beware of the filler at the end of the p2l index. */ if (entry->offset != max_offset) SVN_ERR(read_all_nul(rev_file->file, entry->size, pool)); } else { + /* Generic contents check against checksum. */ if (entry->size < STREAM_THRESHOLD) SVN_ERR(expected_buffered_checksum(rev_file->file, entry, pool)); Propchange: subversion/branches/ra-git/subversion/libsvn_fs_x/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 7 12:30:06 2018 @@ -3,6 +3,7 @@ /subversion/branches/1.7.x-fs-verify/subversion/libsvn_fs_x:1146708,1161180 /subversion/branches/10Gb/subversion/libsvn_fs_x:1388102,1388163-1388190,1388195,1388202,1388205,1388211,1388276,1388362,1388375,1388394,1388636,1388639-1388640,1388643-1388644,1388654,1388720,1388789,1388795,1388801,1388805,1388807,1388810,1388816,1389044,1389276,1389289,1389662,1389867,1390017,1390209,1390216,1390407,1390409,1390414,1390419,1390955 /subversion/branches/atomic-revprop/subversion/libsvn_fs_x:965046-1000689 +/subversion/branches/authzperf/subversion/libsvn_fs_x:1613053-1776831 /subversion/branches/auto-props-sdc/subversion/libsvn_fs_x:1384106-1401643 /subversion/branches/bdb-reverse-deltas/subversion/libsvn_fs_x:872050-872529 /subversion/branches/cache-server/subversion/libsvn_fs_x:1458643-1476567 @@ -94,5 +95,5 @@ /subversion/branches/verify-at-commit/subversion/libsvn_fs_x:1462039-1462408 /subversion/branches/verify-keep-going/subversion/libsvn_fs_x:1439280-1492639,1546002-1546110 /subversion/branches/wc-collate-path/subversion/libsvn_fs_x:1402685-1480384 -/subversion/trunk/subversion/libsvn_fs_fs:1415133-1596500,1596567,1597414,1597989,1598273,1599140,1600872,1601633,1603485-1603487,1603499,1603605,1604128,1604188,1604413-1604414,1604416-1604417,1604421,1604442,1604700,1604717,1604720,1604726,1604755,1604794,1604802,1604824,1604836,1604844,1604902-1604903,1604911,1604925,1604933,1604947,1605059-1605060,1605064-1605065,1605068,1605071-1605073,1605075,1605123,1605188-1605189,1605191,1605197,1605444,1605633,1606132,1606142,1606144,1606514,1606526,1606528,1606551,1606554,1606564,1606598-1606599,1606656,1606658,1606662,1606744,1606840,1607085,1607572,1612407,1612810,1613339,1613872,1614611,1615348,1615351-1615352,1615356,1616338-1616339,1616613,1617586,1617688,1618138,1618151,1618153,1618226,1618641,1618653,1618662,1619068,1619358,1619413,1619769,1619774,1620602,1620909,1620912,1620928,1620930,1621275,1621635,1622931,1622937,1622942,1622946,1622959-1622960,1622963,1622987,1623007,1623368,1623373,1623377,1623379,1623381,1623398,1623402,162 4011,1624265,1624512,1626246,1626871,1626873,1626886,1627497-1627498,1627502,1627947-1627949,1627966,1628083,1628093,1628158-1628159,1628161,1628392-1628393,1628415,1628427,1628676,1628738,1628762,1628764,1629854-1629855,1629857,1629865,1629873,1629875,1629879,1630067,1630070,1631049-1631051,1631075,1631115,1631171,1631180,1631185-1631186,1631196-1631197,1631239-1631240,1631548,1631550,1631563,1631567,1631588,1631598,1632646,1632776,1632849,1632851-1632853,1632856-1632857,1632868,1632908,1632926,1633232,1633617-1633618,1634872,1634875,1634879-1634880,1634920,1636478,1636483,1636629,1636644,1637184,1637186,1637330,1637358,1637363,1637393,1639319,1639322,1639335,1639348,1639352,1639355,1639358,1639414,1639419,1639426,1639430,1639436,1639440,1639549,1640061-1640062,1640197,1640915,1640966,1641013,1643139,1643233,1645567,1646021,1646712,1646716,1647537,1647540-1647541,1647820,1647905,1648230,1648238,1648241-1648243,1648253,1648272,1648532,1648537-1648539,1648542,1648591,1648612,1649590, 1651567,1652068,1652076,1652441,1652451,1653608,1654932,1654934,1654937,1655635,1655649,1655651,1655664,1656176,1657525,1657972,1657978,1658482,1659212,1659217,1659314,1659509,1662668,1665318,1665854,1665894,1667090,1667101,1667538,1669743,1669746,1669749,1669945,1670139,1670953,1673170,1673197,1673202,1673204,1673445,1673454,1673685,1673689,1673875,1674165,1674341,1674400,1674404,1674631,1674669,1674673,1675396,1676667,1677431,1678149,1678151,1678718,1678725,1679169,1679907,1679920-1679924,1679926,1680347,1680460,1680464,1680476,1680819,1681949,1681966,1681974,1681994,1682008,1682076,1682086,1682093,1682259,1682265,1682739,1682864,1683311,1683330,1683378,1683544,1683553,1684047,1686232,1686542,1686546,1686554,1686557,1687061,1687064,1687070-1687071,1687074,1687078-1687079,1688270,1688425,1692650,1693886,1694489,1694848,1696171,1696185,1696627-1696628,1696630,1696758,1697372,1697381,1697387,1697393,1697403,1697405,1701017,1701053,1702600,1702922,1703069,1703142,1703237,1703240,17052 66,1705638,1705643,1705646,1705724,1705730,1705739,1706612,1706615,1706617,1706619,1706675-1706676,1706679,1706979-1706980,1707308,1707971-1707973,1707986,1707988-1707989,1708004,1709388,1709799,1710017,1710359,1710368,1710370,1711507,1711582,1711672,1712927,1715793,1715947,1716047,1716067,1716784,1716973-1716974,1717332,1717334,1717864,1719269,1719336,1719413,1719730,1720015,1721285,1723715,1723720,1723834,1723839,1725179-1725180,1726004,1726099,1726116,1726897,1726995,1727006-1727007,1727028,1727040,1727707,1727822,1730491,1735916,1736357,1736359,1737355-1737356,1740721-1740722,1741096,1741200,1741206,1741214,1741224,1742540,1745055,1745107,1745852,1746006,1746012,1746026 -/subversion/trunk/subversion/libsvn_fs_x:1414756-1509914,1583624-1773991 +/subversion/trunk/subversion/libsvn_fs_fs:1415133-1596500,1596567,1597414,1597989,1598273,1599140,1600872,1601633,1603485-1603487,1603499,1603605,1604128,1604188,1604413-1604414,1604416-1604417,1604421,1604442,1604700,1604717,1604720,1604726,1604755,1604794,1604802,1604824,1604836,1604844,1604902-1604903,1604911,1604925,1604933,1604947,1605059-1605060,1605064-1605065,1605068,1605071-1605073,1605075,1605123,1605188-1605189,1605191,1605197,1605444,1605633,1606132,1606142,1606144,1606514,1606526,1606528,1606551,1606554,1606564,1606598-1606599,1606656,1606658,1606662,1606744,1606840,1607085,1607572,1612407,1612810,1613339,1613872,1614611,1615348,1615351-1615352,1615356,1616338-1616339,1616613,1617586,1617688,1618138,1618151,1618153,1618226,1618641,1618653,1618662,1619068,1619358,1619413,1619769,1619774,1620602,1620909,1620912,1620928,1620930,1621275,1621635,1622931,1622937,1622942,1622946,1622959-1622960,1622963,1622987,1623007,1623368,1623373,1623377,1623379,1623381,1623398,1623402,162 4011,1624265,1624512,1626246,1626871,1626873,1626886,1627497-1627498,1627502,1627947-1627949,1627966,1628083,1628093,1628158-1628159,1628161,1628392-1628393,1628415,1628427,1628676,1628738,1628762,1628764,1629854-1629855,1629857,1629865,1629873,1629875,1629879,1630067,1630070,1631049-1631051,1631075,1631115,1631171,1631180,1631185-1631186,1631196-1631197,1631239-1631240,1631548,1631550,1631563,1631567,1631588,1631598,1632646,1632776,1632849,1632851-1632853,1632856-1632857,1632868,1632908,1632926,1633232,1633617-1633618,1634872,1634875,1634879-1634880,1634920,1636478,1636483,1636629,1636644,1637184,1637186,1637330,1637358,1637363,1637393,1639319,1639322,1639335,1639348,1639352,1639355,1639358,1639414,1639419,1639426,1639430,1639436,1639440,1639549,1640061-1640062,1640197,1640915,1640966,1641013,1643139,1643233,1645567,1646021,1646712,1646716,1647537,1647540-1647541,1647820,1647905,1648230,1648238,1648241-1648243,1648253,1648272,1648532,1648537-1648539,1648542,1648591,1648612,1649590, 1651567,1652068,1652076,1652441,1652451,1653608,1654932,1654934,1654937,1655635,1655649,1655651,1655664,1656176,1657525,1657972,1657978,1658482,1659212,1659217,1659314,1659509,1662668,1665318,1665854,1665894,1667090,1667101,1667538,1669743,1669746,1669749,1669945,1670139,1670953,1673170,1673197,1673202,1673204,1673445,1673454,1673685,1673689,1673875,1674165,1674341,1674400,1674404,1674631,1674669,1674673,1675396,1676667,1677431,1678149,1678151,1678718,1678725,1679169,1679907,1679920-1679924,1679926,1680347,1680460,1680464,1680476,1680819,1681949,1681966,1681974,1681994,1682008,1682076,1682086,1682093,1682259,1682265,1682739,1682864,1683311,1683330,1683378,1683544,1683553,1684047,1686232,1686542,1686546,1686554,1686557,1687061,1687064,1687070-1687071,1687074,1687078-1687079,1688270,1688425,1692650,1693886,1694489,1694848,1696171,1696185,1696627-1696628,1696630,1696758,1697372,1697381,1697387,1697393,1697403,1697405,1701017,1701053,1702600,1702922,1703069,1703142,1703237,1703240,17052 66,1705638,1705643,1705646,1705724,1705730,1705739,1706612,1706615,1706617,1706619,1706675-1706676,1706679,1706979-1706980,1707308,1707971-1707973,1707986,1707988-1707989,1708004,1709388,1709799,1710017,1710359,1710368,1710370,1711507,1711582,1711672,1712927,1715793,1715947,1716047,1716067,1716784,1716973-1716974,1717332,1717334,1717864,1719269,1719336,1719413,1719730,1720015,1721285,1723715,1723720,1723834,1723839,1725179-1725180,1726004,1726099,1726116,1726897,1726995,1727006-1727007,1727028,1727040,1727707,1727822,1730491,1735916,1736357,1736359,1737355-1737356,1740721-1740722,1741096,1741200,1741206,1741214,1741224,1742540,1745055,1745107,1745852,1746006,1746012,1746026,1756258-1756266,1756364,1756377,1759117,1759122-1759126,1759135,1759404-1759405,1759686,1764340,1764481,1764676,1766352,1780810,1781655,1781694,1785053,1785737-1785738,1785741,1785754,1785904,1786445-1786446,1786515 +/subversion/trunk/subversion/libsvn_fs_x:1414756-1509914,1583624-1846000