subversion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From stef...@apache.org
Subject svn commit: r1741682 [10/26] - in /subversion/branches/authzperf: ./ build/ build/ac-macros/ build/generator/ contrib/server-side/svncutter/ notes/ notes/api-errata/1.9/ notes/move-tracking/ subversion/ subversion/bindings/ctypes-python/ subversion/bin...
Date Fri, 29 Apr 2016 18:38:56 GMT
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/index.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/index.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/index.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/index.c Fri Apr 29 18:38:53 2016
@@ -231,7 +231,7 @@ stream_error_create(svn_fs_x__packed_num
   apr_off_t offset;
   SVN_ERR(svn_io_file_name_get(&file_name, stream->file,
                                stream->pool));
-  SVN_ERR(svn_fs_x__get_file_offset(&offset, stream->file, stream->pool));
+  SVN_ERR(svn_io_file_get_offset(&offset, stream->file, stream->pool));
 
   return svn_error_createf(err, NULL, message, file_name,
                            apr_psprintf(stream->pool,
@@ -963,8 +963,8 @@ svn_fs_x__l2p_index_append(svn_checksum_
               /* 1 page with up to L2P_PAGE_SIZE entries.
                * fsfs.conf settings validation guarantees this to fit into
                * our address space. */
-              apr_size_t last_buffer_size
-                = (apr_size_t)svn_spillbuf__get_size(buffer);
+              apr_uint64_t last_buffer_size
+                = (apr_uint64_t)svn_spillbuf__get_size(buffer);
 
               svn_pool_clear(iterpool);
 
@@ -1932,7 +1932,7 @@ svn_fs_x__l2p_get_max_ids(apr_array_head
       apr_uint64_t item_count;
       apr_size_t first_page_index, last_page_index;
 
-      if (revision >= header->first_revision + header->revision_count)
+      if (revision - header->first_revision >= header->revision_count)
         {
           /* need to read the next index. Clear up memory used for the
            * previous one.  Note that intermittent pack runs do not change
@@ -2182,8 +2182,8 @@ svn_fs_x__p2l_index_append(svn_checksum_
 
   apr_uint64_t last_entry_end = 0;
   apr_uint64_t last_page_end = 0;
-  apr_size_t last_buffer_size = 0;  /* byte offset in the spill buffer at
-                                       the begin of the current revision */
+  apr_uint64_t last_buffer_size = 0;  /* byte offset in the spill buffer at
+                                         the begin of the current revision */
   apr_uint64_t file_size = 0;
 
   /* temporary data structures that collect the data which will be moved
@@ -2650,6 +2650,13 @@ read_entry(svn_fs_x__packed_number_strea
         }
     }
 
+  /* Corrupted SIZE values might cause arithmetic overflow.
+   * The same can happen if you copy a repository from a system with 63 bit
+   * file lengths to one with 31 bit file lengths. */
+  if ((apr_uint64_t)entry.offset + (apr_uint64_t)entry.size > off_t_max)
+    return svn_error_create(SVN_ERR_FS_INDEX_OVERFLOW , NULL,
+                            _("P2L index entry size overflow."));
+
   APR_ARRAY_PUSH(result, svn_fs_x__p2l_entry_t) = entry;
   *item_offset += entry.size;
 

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/lock.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/lock.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/lock.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/lock.c Fri Apr 29 18:38:53 2016
@@ -235,7 +235,7 @@ write_digest_file(apr_hash_t *children,
   if ((err = svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR,
                              scratch_pool)))
     {
-      svn_error_clear(svn_stream_close(stream));
+      err = svn_error_compose_create(err, svn_stream_close(stream));
       return svn_error_createf(err->apr_err,
                                err,
                                _("Cannot write lock/entries hashfile '%s'"),
@@ -287,7 +287,7 @@ read_digest_file(apr_hash_t **children_p
   hash = apr_hash_make(pool);
   if ((err = svn_hash_read2(hash, stream, SVN_HASH_TERMINATOR, pool)))
     {
-      svn_error_clear(svn_stream_close(stream));
+      err = svn_error_compose_create(err, svn_stream_close(stream));
       return svn_error_createf(err->apr_err,
                                err,
                                _("Can't parse lock/entries hashfile '%s'"),
@@ -951,6 +951,7 @@ lock_body(void *baton,
         }
     }
 
+  svn_pool_destroy(iterpool);
   return SVN_NO_ERROR;
 }
 
@@ -1163,7 +1164,7 @@ svn_fs_x__lock(svn_fs_t *fs,
 
   lb.fs = fs;
   lb.targets = sorted_targets;
-  lb.infos = apr_array_make(scratch_pool, sorted_targets->nelts,
+  lb.infos = apr_array_make(result_pool, sorted_targets->nelts,
                             sizeof(struct lock_info_t));
   lb.comment = comment;
   lb.is_dav_comment = is_dav_comment;
@@ -1259,7 +1260,7 @@ svn_fs_x__unlock(svn_fs_t *fs,
 
   ub.fs = fs;
   ub.targets = sorted_targets;
-  ub.infos = apr_array_make(scratch_pool, sorted_targets->nelts,
+  ub.infos = apr_array_make(result_pool, sorted_targets->nelts,
                             sizeof(struct unlock_info_t));
   ub.skip_check = FALSE;
   ub.break_lock = break_lock;

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.c Fri Apr 29 18:38:53 2016
@@ -102,6 +102,19 @@ parse_revnum(svn_revnum_t *rev,
   return SVN_NO_ERROR;
 }
 
+/* If ERR is not NULL, wrap it MESSAGE.  The latter must have an %ld
+ * format parameter that will be filled with REV. */
+static svn_error_t *
+wrap_footer_error(svn_error_t *err,
+                  const char *message,
+                  svn_revnum_t rev)
+{
+  if (err)
+    return svn_error_quick_wrapf(err, message, rev);
+
+  return SVN_NO_ERROR;
+}
+
 svn_error_t *
 svn_fs_x__parse_footer(apr_off_t *l2p_offset,
                        svn_checksum_t **l2p_checksum,
@@ -109,6 +122,7 @@ svn_fs_x__parse_footer(apr_off_t *l2p_of
                        svn_checksum_t **p2l_checksum,
                        svn_stringbuf_t *footer,
                        svn_revnum_t rev,
+                       apr_off_t footer_offset,
                        apr_pool_t *result_pool)
 {
   apr_int64_t val;
@@ -117,17 +131,20 @@ svn_fs_x__parse_footer(apr_off_t *l2p_of
   /* Get the L2P offset. */
   const char *str = svn_cstring_tokenize(" ", &last_str);
   if (str == NULL)
-    return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
-                            _("Invalid revision footer"));
+    return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                             "Invalid r%ld footer", rev);
 
-  SVN_ERR(svn_cstring_atoi64(&val, str));
+  SVN_ERR(wrap_footer_error(svn_cstring_strtoi64(&val, str, 0,
+                                                 footer_offset - 1, 10),
+                            "Invalid L2P offset in r%ld footer",
+                            rev));
   *l2p_offset = (apr_off_t)val;
 
   /* Get the L2P checksum. */
   str = svn_cstring_tokenize(" ", &last_str);
   if (str == NULL)
-    return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
-                            _("Invalid revision footer"));
+    return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                             "Invalid r%ld footer", rev);
 
   SVN_ERR(svn_checksum_parse_hex(l2p_checksum, svn_checksum_md5, str,
                                  result_pool));
@@ -135,17 +152,33 @@ svn_fs_x__parse_footer(apr_off_t *l2p_of
   /* Get the P2L offset. */
   str = svn_cstring_tokenize(" ", &last_str);
   if (str == NULL)
-    return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
-                            _("Invalid revision footer"));
+    return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                             "Invalid r%ld footer", rev);
 
-  SVN_ERR(svn_cstring_atoi64(&val, str));
+  SVN_ERR(wrap_footer_error(svn_cstring_strtoi64(&val, str, 0,
+                                                 footer_offset - 1, 10),
+                            "Invalid P2L offset in r%ld footer",
+                            rev));
   *p2l_offset = (apr_off_t)val;
 
+  /* The P2L indes follows the L2P index */
+  if (*p2l_offset <= *l2p_offset)
+    return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                             "P2L offset %s must be larger than L2P offset %s"
+                             " in r%ld footer",
+                             apr_psprintf(result_pool,
+                                          "%" APR_UINT64_T_HEX_FMT,
+                                          (apr_uint64_t)*p2l_offset),
+                             apr_psprintf(result_pool,
+                                          "%" APR_UINT64_T_HEX_FMT,
+                                          (apr_uint64_t)*l2p_offset),
+                             rev);
+
   /* Get the P2L checksum. */
   str = svn_cstring_tokenize(" ", &last_str);
   if (str == NULL)
-    return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
-                            _("Invalid revision footer"));
+    return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                             "Invalid r%ld footer", rev);
 
   SVN_ERR(svn_checksum_parse_hex(p2l_checksum, svn_checksum_md5, str,
                                  result_pool));
@@ -796,14 +829,6 @@ read_change(svn_fs_x__change_t **change_
   change = apr_pcalloc(result_pool, sizeof(*change));
   last_str = line->data;
 
-  /* Get the node-id of the change. */
-  str = svn_cstring_tokenize(" ", &last_str);
-  if (str == NULL)
-    return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
-                            _("Invalid changes line in rev-file"));
-
-  SVN_ERR(svn_fs_x__id_parse(&change->noderev_id, str));
-
   /* Get the change type. */
   str = svn_cstring_tokenize(" ", &last_str);
   if (str == NULL)
@@ -1011,7 +1036,6 @@ write_change_entry(svn_stream_t *stream,
                    svn_fs_x__change_t *change,
                    apr_pool_t *scratch_pool)
 {
-  const char *idstr;
   const char *change_string = NULL;
   const char *kind_string = "";
   svn_stringbuf_t *buf;
@@ -1037,8 +1061,6 @@ write_change_entry(svn_stream_t *stream,
                                change->change_kind);
     }
 
-  idstr = svn_fs_x__id_unparse(&change->noderev_id, scratch_pool)->data;
-
   SVN_ERR_ASSERT(change->node_kind == svn_node_dir
                  || change->node_kind == svn_node_file);
   kind_string = apr_psprintf(scratch_pool, "-%s",
@@ -1046,8 +1068,8 @@ write_change_entry(svn_stream_t *stream,
                              ? SVN_FS_X__KIND_DIR
                              : SVN_FS_X__KIND_FILE);
 
-  buf = svn_stringbuf_createf(scratch_pool, "%s %s%s %s %s %s %s\n",
-                              idstr, change_string, kind_string,
+  buf = svn_stringbuf_createf(scratch_pool, "%s%s %s %s %s %s\n",
+                              change_string, kind_string,
                               change->text_mod ? FLAG_TRUE : FLAG_FALSE,
                               change->prop_mod ? FLAG_TRUE : FLAG_FALSE,
                               change->mergeinfo_mod == svn_tristate_true
@@ -1113,3 +1135,107 @@ svn_fs_x__write_changes(svn_stream_t *st
   return SVN_NO_ERROR;
 }
 
+svn_error_t *
+svn_fs_x__parse_properties(apr_hash_t **properties,
+                           const svn_string_t *content,
+                           apr_pool_t *result_pool)
+{
+  const apr_byte_t *p = (const apr_byte_t *)content->data;
+  const apr_byte_t *end = p + content->len;
+  apr_uint64_t count;
+
+  *properties = apr_hash_make(result_pool);
+
+  /* Extract the number of properties we are expected to read. */
+  p = svn__decode_uint(&count, p, end);
+
+  /* Read all the properties we find.
+     Because prop-name and prop-value are nicely NUL-terminated
+     sub-strings of CONTENT, we can simply reference them there.
+     I.e. there is no need to copy them around.
+   */
+  while (p < end)
+    {
+      apr_uint64_t value_len;
+      svn_string_t *value;
+
+      const char *key = (const char *)p;
+
+      /* Note that this may never overflow / segfault because
+         CONTENT itself is NUL-terminated. */
+      apr_size_t key_len = strlen(key);
+      p += key_len + 1;
+      if (key[key_len])
+        return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL,
+                                 "Property name not NUL terminated");
+
+      if (p >= end)
+        return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL,
+                                 "Property value missing");
+      p = svn__decode_uint(&value_len, p, end);
+      if (value_len >= (end - p))
+        return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL,
+                                 "Property value too long");
+
+      value = apr_pcalloc(result_pool, sizeof(*value));
+      value->data = (const char *)p;
+      value->len = (apr_size_t)value_len;
+      if (p[value->len])
+        return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL,
+                                 "Property value not NUL terminated");
+
+      p += value->len + 1;
+
+      apr_hash_set(*properties, key, key_len, value);
+    }
+
+  /* Check that we read the expected number of properties. */
+  if ((apr_uint64_t)apr_hash_count(*properties) != count)
+    return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL,
+                             "Property count mismatch");
+
+  return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs_x__write_properties(svn_stream_t *stream,
+                           apr_hash_t *proplist,
+                           apr_pool_t *scratch_pool)
+{
+  apr_byte_t buffer[SVN__MAX_ENCODED_UINT_LEN];
+  apr_size_t len;
+  apr_hash_index_t *hi;
+
+  /* Write the number of properties in this list. */
+  len = svn__encode_uint(buffer, apr_hash_count(proplist)) - buffer;
+  SVN_ERR(svn_stream_write(stream, (const char *)buffer, &len));
+
+  /* Serialize each property as follows:
+     <Prop-name> <NUL>
+     <Value-len> <Prop-value> <NUL>
+   */
+  for (hi = apr_hash_first(scratch_pool, proplist);
+       hi;
+       hi = apr_hash_next(hi))
+    {
+      const char *key;
+      apr_size_t key_len;
+      svn_string_t *value;
+      apr_hash_this(hi, (const void **)&key, (apr_ssize_t *)&key_len,
+                    (void **)&value);
+
+      /* Include the terminating NUL. */
+      ++key_len;
+      SVN_ERR(svn_stream_write(stream, key, &key_len));
+
+      len = svn__encode_uint(buffer, value->len) - buffer;
+      SVN_ERR(svn_stream_write(stream, (const char *)buffer, &len));
+      SVN_ERR(svn_stream_write(stream, value->data, &value->len));
+
+      /* Terminate with NUL. */
+      len = 1;
+      SVN_ERR(svn_stream_write(stream, "", &len));
+    }
+
+  return SVN_NO_ERROR;
+}

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.h
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.h?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.h (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.h Fri Apr 29 18:38:53 2016
@@ -50,6 +50,8 @@ extern "C" {
  * *P2L_OFFSET, respectively.  Also, return the expected checksums in
  * in *L2P_CHECKSUM and *P2L_CHECKSUM.
  *
+ * FOOTER_OFFSET is used for validation.
+ *
  * Note that REV is only used to construct nicer error objects that
  * mention this revision.  Allocate the checksums in RESULT_POOL.
  */
@@ -60,6 +62,7 @@ svn_fs_x__parse_footer(apr_off_t *l2p_of
                        svn_checksum_t **p2l_checksum,
                        svn_stringbuf_t *footer,
                        svn_revnum_t rev,
+                       apr_off_t footer_offset,
                        apr_pool_t *result_pool);
 
 /* Given the offset of the L2P index data in L2P_OFFSET, the content
@@ -207,6 +210,23 @@ svn_fs_x__write_changes(svn_stream_t *st
                         svn_boolean_t terminate_list,
                         apr_pool_t *scratch_pool);
 
+/* Parse the property list serialized in CONTENT and return it in
+   *PROPERTIES, allocated from RESULT_POOL.  CONTENT must remain
+   valid at least until the next cleanup of RESULT_POOL.
+ */
+svn_error_t *
+svn_fs_x__parse_properties(apr_hash_t **properties,
+                           const svn_string_t *content,
+                           apr_pool_t *result_pool);
+
+/* Write the property list PROPLIST to STREAM in serialized format.
+   Use SCRATCH_POOL for temporary allocations.
+ */
+svn_error_t *
+svn_fs_x__write_properties(svn_stream_t *stream,
+                           apr_hash_t *proplist,
+                           apr_pool_t *scratch_pool);
+
 #ifdef __cplusplus
 }
 #endif /* __cplusplus */

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c Fri Apr 29 18:38:53 2016
@@ -106,9 +106,6 @@ typedef struct path_order_t
   /* when this change happened */
   svn_revnum_t revision;
 
-  /* this is a directory node */
-  svn_boolean_t is_dir;
-
   /* length of the expanded representation content */
   apr_int64_t expanded_size;
 
@@ -247,6 +244,7 @@ initialize_pack_context(pack_context_t *
                         const char *shard_dir,
                         svn_revnum_t shard_rev,
                         int max_items,
+                        svn_fs_x__batch_fsync_t *batch,
                         svn_cancel_func_t cancel_func,
                         void *cancel_baton,
                         apr_pool_t *pool)
@@ -275,9 +273,9 @@ initialize_pack_context(pack_context_t *
   context->pack_file_dir = pack_file_dir;
   context->pack_file_path
     = svn_dirent_join(pack_file_dir, PATH_PACKED, pool);
-  SVN_ERR(svn_io_file_open(&context->pack_file, context->pack_file_path,
-                           APR_WRITE | APR_BUFFERED | APR_BINARY | APR_EXCL
-                             | APR_CREATE, APR_OS_DEFAULT, pool));
+
+  SVN_ERR(svn_fs_x__batch_fsync_open_file(&context->pack_file, batch,
+                                          context->pack_file_path, pool));
 
   /* Proto index files */
   SVN_ERR(svn_fs_x__l2p_proto_index_open(
@@ -382,8 +380,6 @@ close_pack_context(pack_context_t *conte
   SVN_ERR(svn_io_remove_file2(proto_l2p_index_path, FALSE, scratch_pool));
   SVN_ERR(svn_io_remove_file2(proto_p2l_index_path, FALSE, scratch_pool));
 
-  SVN_ERR(svn_io_file_close(context->pack_file, scratch_pool));
-
   return SVN_NO_ERROR;
 }
 
@@ -395,7 +391,7 @@ static svn_error_t *
 copy_file_data(pack_context_t *context,
                apr_file_t *dest,
                apr_file_t *source,
-               apr_off_t size,
+               svn_filesize_t size,
                apr_pool_t *scratch_pool)
 {
   /* most non-representation items will be small.  Minimize the buffer
@@ -482,8 +478,8 @@ copy_item_to_temp(pack_context_t *contex
   svn_fs_x__p2l_entry_t *new_entry
     = svn_fs_x__p2l_entry_dup(entry, context->info_pool);
 
-  SVN_ERR(svn_fs_x__get_file_offset(&new_entry->offset, temp_file,
-                                    scratch_pool));
+  SVN_ERR(svn_io_file_get_offset(&new_entry->offset, temp_file,
+                                 scratch_pool));
   APR_ARRAY_PUSH(entries, svn_fs_x__p2l_entry_t *) = new_entry;
 
   SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file));
@@ -576,8 +572,8 @@ copy_rep_to_temp(pack_context_t *context
   /* create a copy of ENTRY, make it point to the copy destination and
    * store it in CONTEXT */
   entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool);
-  SVN_ERR(svn_fs_x__get_file_offset(&entry->offset, context->reps_file,
-                                    scratch_pool));
+  SVN_ERR(svn_io_file_get_offset(&entry->offset, context->reps_file,
+                                 scratch_pool));
   add_item_rep_mapping(context, entry);
 
   /* read & parse the representation header */
@@ -621,9 +617,6 @@ compare_dir_entries(const svn_sort__item
   const svn_fs_dirent_t *lhs = (const svn_fs_dirent_t *) a->value;
   const svn_fs_dirent_t *rhs = (const svn_fs_dirent_t *) b->value;
 
-  if (lhs->kind != rhs->kind)
-    return lhs->kind == svn_node_dir ? -1 : 1;
-
   return strcmp(lhs->name, rhs->name);
 }
 
@@ -705,8 +698,8 @@ copy_node_to_temp(pack_context_t *contex
   /* create a copy of ENTRY, make it point to the copy destination and
    * store it in CONTEXT */
   entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool);
-  SVN_ERR(svn_fs_x__get_file_offset(&entry->offset, context->reps_file,
-                                    scratch_pool));
+  SVN_ERR(svn_io_file_get_offset(&entry->offset, context->reps_file,
+                                 scratch_pool));
   add_item_rep_mapping(context, entry);
 
   /* copy the noderev to our temp file */
@@ -740,7 +733,6 @@ copy_node_to_temp(pack_context_t *contex
   path_order->path = svn_prefix_string__create(context->paths, sort_path);
   path_order->node_id = noderev->node_id;
   path_order->revision = svn_fs_x__get_revnum(noderev->noderev_id.change_set);
-  path_order->is_dir = noderev->kind == svn_node_dir;
   path_order->noderev_id = noderev->noderev_id;
   APR_ARRAY_PUSH(context->path_order, path_order_t *) = path_order;
 
@@ -784,13 +776,8 @@ compare_path_order(const path_order_t *
   const path_order_t * lhs = *lhs_p;
   const path_order_t * rhs = *rhs_p;
 
-  /* cluster all directories */
-  int diff = rhs->is_dir - lhs->is_dir;
-  if (diff)
-    return diff;
-
   /* lexicographic order on path and node (i.e. latest first) */
-  diff = svn_prefix_string__compare(lhs->path, rhs->path);
+  int diff = svn_prefix_string__compare(lhs->path, rhs->path);
   if (diff)
     return diff;
 
@@ -834,7 +821,7 @@ sort_reps(pack_context_t *context)
 /* Return the remaining unused bytes in the current block in CONTEXT's
  * pack file.
  */
-static apr_ssize_t
+static apr_off_t
 get_block_left(pack_context_t *context)
 {
   svn_fs_x__data_t *ffd = context->fs->fsap_data;
@@ -1852,20 +1839,15 @@ append_revision(pack_context_t *context,
   apr_pool_t *iterpool = svn_pool_create(scratch_pool);
   svn_fs_x__revision_file_t *rev_file;
   apr_file_t *file;
-  apr_finfo_t finfo;
-
-  /* Get the size of the file. */
-  const char *path = svn_dirent_join(context->shard_dir,
-                                     apr_psprintf(iterpool, "%ld",
-                                                  context->start_rev),
-                                     scratch_pool);
-  SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, scratch_pool));
+  svn_filesize_t revfile_size;
 
   /* Copy all the bits from the rev file to the end of the pack file. */
   SVN_ERR(svn_fs_x__rev_file_init(&rev_file, context->fs, context->start_rev,
                                   scratch_pool));
   SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file));
-  SVN_ERR(copy_file_data(context, context->pack_file, file, finfo.size,
+
+  SVN_ERR(svn_io_file_size_get(&revfile_size, file, scratch_pool));
+  SVN_ERR(copy_file_data(context, context->pack_file, file, revfile_size,
                          iterpool));
 
   /* mark the start of a new revision */
@@ -1874,7 +1856,7 @@ append_revision(pack_context_t *context,
 
   /* read the phys-to-log index file until we covered the whole rev file.
    * That index contains enough info to build both target indexes from it. */
-  while (offset < finfo.size)
+  while (offset < revfile_size)
     {
       /* read one cluster */
       int i;
@@ -1896,7 +1878,7 @@ append_revision(pack_context_t *context,
 
           /* process entry while inside the rev file */
           offset = entry->offset;
-          if (offset < finfo.size)
+          if (offset < revfile_size)
             {
               /* there should be true containers */
               SVN_ERR_ASSERT(entry->item_count == 1);
@@ -1915,7 +1897,7 @@ append_revision(pack_context_t *context,
     }
 
   svn_pool_destroy(iterpool);
-  context->pack_offset += finfo.size;
+  context->pack_offset += revfile_size;
 
   return SVN_NO_ERROR;
 }
@@ -1926,6 +1908,7 @@ append_revision(pack_context_t *context,
  * SHARD_DIR into the PACK_FILE_DIR, using SCRATCH_POOL for temporary
  * allocations.  Limit the extra memory consumption to MAX_MEM bytes.
  * CANCEL_FUNC and CANCEL_BATON are what you think they are.
+ * Schedule necessary fsync calls in BATCH.
  */
 static svn_error_t *
 pack_log_addressed(svn_fs_t *fs,
@@ -1933,6 +1916,7 @@ pack_log_addressed(svn_fs_t *fs,
                    const char *shard_dir,
                    svn_revnum_t shard_rev,
                    apr_size_t max_mem,
+                   svn_fs_x__batch_fsync_t *batch,
                    svn_cancel_func_t cancel_func,
                    void *cancel_baton,
                    apr_pool_t *scratch_pool)
@@ -1959,7 +1943,7 @@ pack_log_addressed(svn_fs_t *fs,
 
   /* set up a pack context */
   SVN_ERR(initialize_pack_context(&context, fs, pack_file_dir, shard_dir,
-                                  shard_rev, max_items, cancel_func,
+                                  shard_rev, max_items, batch, cancel_func,
                                   cancel_baton, scratch_pool));
 
   /* phase 1: determine the size of the revisions to pack */
@@ -1969,7 +1953,8 @@ pack_log_addressed(svn_fs_t *fs,
 
   /* pack revisions in ranges that don't exceed MAX_MEM */
   for (i = 0; i < max_ids->nelts; ++i)
-    if (APR_ARRAY_IDX(max_ids, i, apr_uint64_t) + item_count <= max_items)
+    if (   APR_ARRAY_IDX(max_ids, i, apr_uint64_t)
+        <= (apr_uint64_t)max_items - item_count)
       {
         context.end_rev++;
       }
@@ -2016,7 +2001,7 @@ pack_log_addressed(svn_fs_t *fs,
  * MAX_FILES_PER_DIR revisions from SHARD_PATH into the PACK_FILE_DIR,
  * using SCRATCH_POOL for temporary allocations.  Try to limit the amount of
  * temporary memory needed to MAX_MEM bytes.  CANCEL_FUNC and CANCEL_BATON
- * are what you think they are.
+ * are what you think they are.  Schedule necessary fsync calls in BATCH.
  *
  * If for some reason we detect a partial packing already performed, we
  * remove the pack file and start again.
@@ -2030,6 +2015,7 @@ pack_rev_shard(svn_fs_t *fs,
                apr_int64_t shard,
                int max_files_per_dir,
                apr_size_t max_mem,
+               svn_fs_x__batch_fsync_t *batch,
                svn_cancel_func_t cancel_func,
                void *cancel_baton,
                apr_pool_t *scratch_pool)
@@ -2046,10 +2032,11 @@ pack_rev_shard(svn_fs_t *fs,
 
   /* Create the new directory and pack file. */
   SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool));
+  SVN_ERR(svn_fs_x__batch_fsync_new_path(batch, pack_file_dir, scratch_pool));
 
   /* Index information files */
   SVN_ERR(pack_log_addressed(fs, pack_file_dir, shard_path, shard_rev,
-                              max_mem, cancel_func, cancel_baton,
+                             max_mem, batch, cancel_func, cancel_baton,
                              scratch_pool));
 
   SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, scratch_pool));
@@ -2082,45 +2069,39 @@ pack_shard(const char *dir,
            apr_pool_t *scratch_pool)
 {
   svn_fs_x__data_t *ffd = fs->fsap_data;
-  const char *rev_shard_path, *rev_pack_file_dir;
-  const char *revprops_shard_path, *revprops_pack_file_dir;
+  const char *shard_path, *pack_file_dir;
+  svn_fs_x__batch_fsync_t *batch;
 
   /* Notify caller we're starting to pack this shard. */
   if (notify_func)
     SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start,
                         scratch_pool));
 
+  /* Perform all fsyncs through this instance. */
+  SVN_ERR(svn_fs_x__batch_fsync_create(&batch, scratch_pool));
+
   /* Some useful paths. */
-  rev_pack_file_dir = svn_dirent_join(dir,
+  pack_file_dir = svn_dirent_join(dir,
                   apr_psprintf(scratch_pool,
                                "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
                                shard),
                   scratch_pool);
-  rev_shard_path = svn_dirent_join(dir,
+  shard_path = svn_dirent_join(dir,
                       apr_psprintf(scratch_pool, "%" APR_INT64_T_FMT, shard),
                       scratch_pool);
 
   /* pack the revision content */
-  SVN_ERR(pack_rev_shard(fs, rev_pack_file_dir, rev_shard_path,
-                         shard, max_files_per_dir, DEFAULT_MAX_MEM,
+  SVN_ERR(pack_rev_shard(fs, pack_file_dir, shard_path,
+                         shard, max_files_per_dir, DEFAULT_MAX_MEM, batch,
                          cancel_func, cancel_baton, scratch_pool));
 
   /* pack the revprops in an equivalent way */
-  revprops_pack_file_dir = svn_dirent_join(dir,
-                apr_psprintf(scratch_pool,
-                            "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
-                            shard),
-                scratch_pool);
-  revprops_shard_path = svn_dirent_join(dir,
-                apr_psprintf(scratch_pool, "%" APR_INT64_T_FMT, shard),
-                scratch_pool);
-
   SVN_ERR(svn_fs_x__pack_revprops_shard(fs,
-                                        revprops_pack_file_dir,
-                                        revprops_shard_path,
+                                        pack_file_dir,
+                                        shard_path,
                                         shard, max_files_per_dir,
                                         (int)(0.9 * max_pack_size),
-                                        compression_level,
+                                        compression_level, batch,
                                         cancel_func, cancel_baton,
                                         scratch_pool));
 
@@ -2130,8 +2111,11 @@ pack_shard(const char *dir,
                           scratch_pool));
   ffd->min_unpacked_rev = (svn_revnum_t)((shard + 1) * max_files_per_dir);
 
+  /* Ensure that packed file is written to disk.*/
+  SVN_ERR(svn_fs_x__batch_fsync_run(batch, scratch_pool));
+
   /* Finally, remove the existing shard directories. */
-  SVN_ERR(svn_io_remove_dir2(rev_shard_path, TRUE,
+  SVN_ERR(svn_io_remove_dir2(shard_path, TRUE,
                              cancel_func, cancel_baton, scratch_pool));
 
   /* Notify caller we're starting to pack this shard. */
@@ -2142,6 +2126,34 @@ pack_shard(const char *dir,
   return SVN_NO_ERROR;
 }
 
+/* Read the youngest rev and the first non-packed rev info for FS from disk.
+   Set *FULLY_PACKED when there is no completed unpacked shard.
+   Use SCRATCH_POOL for temporary allocations.
+ */
+static svn_error_t *
+get_pack_status(svn_boolean_t *fully_packed,
+                svn_fs_t *fs,
+                apr_pool_t *scratch_pool)
+{
+  svn_fs_x__data_t *ffd = fs->fsap_data;
+  apr_int64_t completed_shards;
+  svn_revnum_t youngest;
+
+  SVN_ERR(svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, fs,
+                                          scratch_pool));
+
+  SVN_ERR(svn_fs_x__youngest_rev(&youngest, fs, scratch_pool));
+  completed_shards = (youngest + 1) / ffd->max_files_per_dir;
+
+  /* See if we've already completed all possible shards thus far. */
+  if (ffd->min_unpacked_rev == (completed_shards * ffd->max_files_per_dir))
+    *fully_packed = TRUE;
+  else
+    *fully_packed = FALSE;
+
+  return SVN_NO_ERROR;
+}
+
 typedef struct pack_baton_t
 {
   svn_fs_t *fs;
@@ -2174,21 +2186,24 @@ pack_body(void *baton,
   svn_fs_x__data_t *ffd = pb->fs->fsap_data;
   apr_int64_t completed_shards;
   apr_int64_t i;
-  svn_revnum_t youngest;
   apr_pool_t *iterpool;
   const char *data_path;
+  svn_boolean_t fully_packed;
 
-  /* If we aren't using sharding, we can't do any packing, so quit. */
-  SVN_ERR(svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, pb->fs,
-                                          scratch_pool));
+  /* Since another process might have already packed the repo,
+     we need to re-read the pack status. */
+  SVN_ERR(get_pack_status(&fully_packed, pb->fs, scratch_pool));
+  if (fully_packed)
+    {
+      if (pb->notify_func)
+        (*pb->notify_func)(pb->notify_baton,
+                           ffd->min_unpacked_rev / ffd->max_files_per_dir,
+                           svn_fs_pack_notify_noop, scratch_pool);
 
-  SVN_ERR(svn_fs_x__youngest_rev(&youngest, pb->fs, scratch_pool));
-  completed_shards = (youngest + 1) / ffd->max_files_per_dir;
-
-  /* See if we've already completed all possible shards thus far. */
-  if (ffd->min_unpacked_rev == (completed_shards * ffd->max_files_per_dir))
-    return SVN_NO_ERROR;
+      return SVN_NO_ERROR;
+    }
 
+  completed_shards = (ffd->youngest_rev_cache + 1) / ffd->max_files_per_dir;
   data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, scratch_pool);
 
   iterpool = svn_pool_create(scratch_pool);
@@ -2224,6 +2239,23 @@ svn_fs_x__pack(svn_fs_t *fs,
                apr_pool_t *scratch_pool)
 {
   pack_baton_t pb = { 0 };
+  svn_boolean_t fully_packed;
+
+  /* Is there we even anything to do?. */
+  SVN_ERR(get_pack_status(&fully_packed, fs, scratch_pool));
+  if (fully_packed)
+    {
+      svn_fs_x__data_t *ffd = fs->fsap_data;
+
+      if (notify_func)
+        (*notify_func)(notify_baton,
+                       ffd->min_unpacked_rev / ffd->max_files_per_dir,
+                       svn_fs_pack_notify_noop, scratch_pool);
+
+      return SVN_NO_ERROR;
+    }
+
+  /* Lock the repo and start the pack process. */
   pb.fs = fs;
   pb.notify_func = notify_func;
   pb.notify_baton = notify_baton;

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c Fri Apr 29 18:38:53 2016
@@ -22,6 +22,7 @@
 
 #include "recovery.h"
 
+#include "svn_dirent_uri.h"
 #include "svn_hash.h"
 #include "svn_pools.h"
 #include "private/svn_string_private.h"
@@ -106,6 +107,86 @@ recover_get_largest_revision(svn_fs_t *f
   return SVN_NO_ERROR;
 }
 
+/* Delete all files and sub-directories (recursively) of DIR_PATH but
+   leave DIR_PATH itself in place.  Use SCRATCH_POOL for temporaries. */
+static svn_error_t *
+clear_directory(const char *dir_path,
+                apr_pool_t *scratch_pool)
+{
+  apr_hash_t *dirents;
+  apr_hash_index_t *hi;
+  apr_pool_t *iterpool = svn_pool_create(scratch_pool);
+
+  SVN_ERR(svn_io_get_dirents3(&dirents, dir_path, TRUE, scratch_pool,
+                              scratch_pool));
+
+  for (hi = apr_hash_first(scratch_pool, dirents);
+       hi;
+       hi = apr_hash_next(hi))
+    {
+      const char *path;
+      const char *name;
+      svn_dirent_t *dirent;
+
+      svn_pool_clear(iterpool);
+      apr_hash_this(hi, (const void **)&name, NULL, (void **)&dirent);
+
+      path = svn_dirent_join(dir_path, name, iterpool);
+      if (dirent->kind == svn_node_dir)
+        SVN_ERR(svn_io_remove_dir2(path, TRUE, NULL, NULL, iterpool));
+      else
+        SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
+    }
+
+  svn_pool_destroy(iterpool);
+
+  return SVN_NO_ERROR;
+}
+
+/* Delete all uncommitted transaction data from FS.
+   Use SCRATCH_POOL for temporaries. */
+static svn_error_t *
+discard_transactions(svn_fs_t *fs,
+                     apr_pool_t *scratch_pool)
+{
+  svn_fs_x__data_t *ffd = fs->fsap_data;
+  svn_fs_x__shared_data_t *ffsd = ffd->shared;
+
+  /* In case this FS has been opened more than once in this process,
+     we should purge their shared transaction data as well.  We do the
+     same as abort_txn would, except that we don't expect all txn files
+     to be complete on disk. */
+  while (ffsd->txns)
+    {
+      svn_fs_x__shared_txn_data_t *txn = ffsd->txns;
+      ffsd->txns = txn->next;
+
+      svn_pool_destroy(txn->pool);
+    }
+
+  /* Remove anything from the transaction folders. */
+  SVN_ERR(clear_directory(svn_fs_x__path_txns_dir(fs, scratch_pool),
+                          scratch_pool));
+  SVN_ERR(clear_directory(svn_fs_x__path_txn_proto_revs(fs, scratch_pool),
+                          scratch_pool));
+
+  return SVN_NO_ERROR;
+}
+
+/* Reset txn-current in FS.  Use SCRATCH_POOL for temporaries. */
+static svn_error_t *
+reset_txn_number(svn_fs_t *fs,
+                 apr_pool_t *scratch_pool)
+{
+  const char *initial_txn = "0\n";
+  SVN_ERR(svn_io_write_atomic2(svn_fs_x__path_txn_current(fs, scratch_pool),
+                               initial_txn, strlen(initial_txn),
+                               svn_fs_x__path_uuid(fs, scratch_pool),
+                               FALSE, scratch_pool));
+
+  return SVN_NO_ERROR;
+}
+
 /* Baton used for recover_body below. */
 typedef struct recover_baton_t {
   svn_fs_t *fs;
@@ -134,7 +215,13 @@ recover_body(void *baton,
   /* The admin may have created a plain copy of this repo before attempting
      to recover it (hotcopy may or may not work with corrupted repos).
      Bump the instance ID. */
-  SVN_ERR(svn_fs_x__set_uuid(fs, fs->uuid, NULL, scratch_pool));
+  SVN_ERR(svn_fs_x__set_uuid(fs, fs->uuid, NULL, TRUE, scratch_pool));
+
+  /* Because transactions are not resilient against system crashes,
+     any existing transaction is suspect (and would probably not be
+     reopened anyway).  Get rid of those. */
+  SVN_ERR(discard_transactions(fs, scratch_pool));
+  SVN_ERR(reset_txn_number(fs, scratch_pool));
 
   /* We need to know the largest revision in the filesystem. */
   SVN_ERR(recover_get_largest_revision(fs, &max_rev, scratch_pool));

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c Fri Apr 29 18:38:53 2016
@@ -417,8 +417,8 @@ svn_fs_x__reps_add_base(svn_fs_x__reps_b
   apr_size_t idx;
   SVN_ERR(svn_fs_x__get_contents(&stream, builder->fs, rep, FALSE,
                                  scratch_pool));
-  SVN_ERR(svn_string_from_stream(&contents, stream, scratch_pool,
-                                 scratch_pool));
+  SVN_ERR(svn_string_from_stream2(&contents, stream, SVN__STREAM_CHUNK_SIZE,
+                                  scratch_pool));
   SVN_ERR(svn_fs_x__reps_add(&idx, builder, contents));
 
   base.revision = svn_fs_x__get_revnum(rep->id.change_set);

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/rev_file.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/rev_file.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/rev_file.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/rev_file.c Fri Apr 29 18:38:53 2016
@@ -331,7 +331,7 @@ auto_read_footer(svn_fs_x__revision_file
                                      &file->p2l_info.start,
                                      &file->p2l_info.checksum,
                                      footer, file->file_info.start_revision,
-                                     file->pool));
+                                     filesize - footer_length - 1, file->pool));
       file->l2p_info.end = file->p2l_info.start;
       file->p2l_info.end = filesize - footer_length - 1;
     }
@@ -490,8 +490,8 @@ svn_fs_x__rev_file_offset(apr_off_t *off
                           svn_fs_x__revision_file_t *file)
 {
   SVN_ERR(auto_open(file));
-  return svn_error_trace(svn_fs_x__get_file_offset(offset, file->file,
-                                                   file->pool));
+  return svn_error_trace(svn_io_file_get_offset(offset, file->file,
+                                                file->pool));
 }
 
 svn_error_t *




Mime
View raw message