subversion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From stef...@apache.org
Subject svn commit: r1741682 [9/26] - in /subversion/branches/authzperf: ./ build/ build/ac-macros/ build/generator/ contrib/server-side/svncutter/ notes/ notes/api-errata/1.9/ notes/move-tracking/ subversion/ subversion/bindings/ctypes-python/ subversion/bind...
Date Fri, 29 Apr 2016 18:38:56 GMT
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/cached_data.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/cached_data.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/cached_data.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/cached_data.c Fri Apr 29 18:38:53 2016
@@ -30,6 +30,7 @@
 
 #include "private/svn_io_private.h"
 #include "private/svn_sorts_private.h"
+#include "private/svn_string_private.h"
 #include "private/svn_subr_private.h"
 #include "private/svn_temp_serializer.h"
 
@@ -54,6 +55,7 @@ block_read(void **result,
            svn_fs_t *fs,
            const svn_fs_x__id_t *id,
            svn_fs_x__revision_file_t *revision_file,
+           void *baton,
            apr_pool_t *result_pool,
            apr_pool_t *scratch_pool);
 
@@ -288,14 +290,13 @@ get_node_revision_body(svn_fs_x__noderev
                                                          scratch_pool),
                              APR_READ | APR_BUFFERED, APR_OS_DEFAULT,
                              scratch_pool);
-      if (err)
+      if (err && APR_STATUS_IS_ENOENT(err->apr_err))
+        {
+          svn_error_clear(err);
+          return svn_error_trace(err_dangling_id(fs, id));
+        }
+      else if (err)
         {
-          if (APR_STATUS_IS_ENOENT(err->apr_err))
-            {
-              svn_error_clear(err);
-              return svn_error_trace(err_dangling_id(fs, id));
-            }
-
           return svn_error_trace(err);
         }
 
@@ -352,6 +353,7 @@ get_node_revision_body(svn_fs_x__noderev
       SVN_ERR(block_read((void **)noderev_p, fs,
                          id,
                          revision_file,
+                         NULL,
                          result_pool,
                          scratch_pool));
       SVN_ERR(svn_fs_x__close_revision_file(revision_file));
@@ -698,7 +700,7 @@ create_rep_state_body(rep_state_t **rep_
       /* populate the cache if appropriate */
       if (SVN_IS_VALID_REVNUM(revision))
         {
-          SVN_ERR(block_read(NULL, fs, &rs->rep_id, rs->sfile->rfile,
+          SVN_ERR(block_read(NULL, fs, &rs->rep_id, rs->sfile->rfile, NULL,
                              result_pool, scratch_pool));
           SVN_ERR(svn_cache__set(ffd->rep_header_cache, &key, rh,
                                  scratch_pool));
@@ -1310,7 +1312,7 @@ read_delta_window(svn_txdelta_window_t *
       && svn_fs_x__is_revision(rs->rep_id.change_set)
       && rs->window_cache)
     {
-      SVN_ERR(block_read(NULL, rs->sfile->fs, &rs->rep_id, file,
+      SVN_ERR(block_read(NULL, rs->sfile->fs, &rs->rep_id, file, NULL,
                          result_pool, scratch_pool));
 
       /* reading the whole block probably also provided us with the
@@ -1341,7 +1343,7 @@ read_delta_window(svn_txdelta_window_t *
       SVN_ERR(svn_fs_x__rev_file_get(&apr_file, file));
       SVN_ERR(svn_txdelta_skip_svndiff_window(apr_file, rs->ver, iterpool));
       rs->chunk_index++;
-      SVN_ERR(svn_fs_x__get_file_offset(&start_offset, apr_file, iterpool));
+      SVN_ERR(svn_io_file_get_offset(&start_offset, apr_file, iterpool));
 
       rs->current = start_offset - rs->start;
       if (rs->current >= rs->size)
@@ -1405,7 +1407,8 @@ read_container_window(svn_stringbuf_t **
     {
       SVN_ERR(auto_open_shared_file(rs->sfile));
       SVN_ERR(block_read((void **)&extractor, fs, &rs->rep_id,
-                         rs->sfile->rfile, result_pool, scratch_pool));
+                         rs->sfile->rfile, NULL,
+                         result_pool, scratch_pool));
     }
 
   SVN_ERR(svn_fs_x__extractor_drive(nwin, extractor, rs->current, size,
@@ -1510,7 +1513,7 @@ get_combined_window(svn_stringbuf_t **re
 }
 
 /* Returns whether or not the expanded fulltext of the file is cachable
- * based on its size SIZE.  The decision depends on the cache used by RB.
+ * based on its size SIZE.  The decision depends on the cache used by FFD.
  */
 static svn_boolean_t
 fulltext_size_is_cachable(svn_fs_x__data_t *ffd,
@@ -1748,10 +1751,10 @@ get_contents_from_windows(rep_read_baton
              This is where we need the pseudo rep_state created
              by build_rep_list(). */
           apr_size_t offset = (apr_size_t)rs->current;
-          if (copy_len + offset > rb->base_window->len)
-            copy_len = offset < rb->base_window->len
-                     ? rb->base_window->len - offset
-                     : 0ul;
+          if (offset >= rb->base_window->len)
+            copy_len = 0ul;
+          else if (copy_len > rb->base_window->len - offset)
+            copy_len = rb->base_window->len - offset;
 
           memcpy (cur, rb->base_window->data + offset, copy_len);
         }
@@ -2366,117 +2369,147 @@ compare_dirent_name(const void *a,
   return strcmp(lhs->name, rhs);
 }
 
-/* Into ENTRIES, read all directories entries from the key-value text in
- * STREAM.  If INCREMENTAL is TRUE, read until the end of the STREAM and
+/* Into ENTRIES, parse all directories entries from the serialized form in
+ * DATA.  If INCREMENTAL is TRUE, read until the end of the STREAM and
  * update the data.  ID is provided for nicer error messages.
+ *
+ * The contents of DATA will be shared with the items in ENTRIES, i.e. it
+ * must not be modified afterwards and must remain valid as long as ENTRIES
+ * is valid.  Use SCRATCH_POOL for temporary allocations.
  */
 static svn_error_t *
-read_dir_entries(apr_array_header_t *entries,
-                 svn_stream_t *stream,
-                 svn_boolean_t incremental,
-                 const svn_fs_x__id_t *id,
-                 apr_pool_t *result_pool,
-                 apr_pool_t *scratch_pool)
+parse_dir_entries(apr_array_header_t **entries_p,
+                  const svn_stringbuf_t *data,
+                  svn_boolean_t incremental,
+                  const svn_fs_x__id_t *id,
+                  apr_pool_t *result_pool,
+                  apr_pool_t *scratch_pool)
 {
-  apr_pool_t *iterpool = svn_pool_create(scratch_pool);
+  const apr_byte_t *p = (const apr_byte_t *)data->data;
+  const apr_byte_t *end = p + data->len;
+  apr_uint64_t count;
   apr_hash_t *hash = incremental ? svn_hash__make(scratch_pool) : NULL;
-  const char *terminator = SVN_HASH_TERMINATOR;
-
-  /* Read until the terminator (non-incremental) or the end of STREAM
-     (incremental mode).  In the latter mode, we use a temporary HASH
-     to make updating and removing entries cheaper. */
-  while (1)
-    {
-      svn_hash__entry_t entry;
-      svn_fs_x__dirent_t *dirent;
-      char *str;
-
-      svn_pool_clear(iterpool);
-      SVN_ERR(svn_hash__read_entry(&entry, stream, terminator,
-                                   incremental, iterpool));
+  apr_array_header_t *entries;
 
-      /* End of directory? */
-      if (entry.key == NULL)
-        {
-          /* In incremental mode, we skip the terminator and read the
-             increments following it until the end of the stream. */
-          if (incremental && terminator)
-            terminator = NULL;
-          else
-            break;
-        }
+  /* Construct the resulting container. */
+  p = svn__decode_uint(&count, p, end);
+  if (count > INT_MAX)
+    return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                             _("Directory for '%s' is too large"),
+                             svn_fs_x__id_unparse(id, scratch_pool)->data);
 
-      /* Deleted entry? */
-      if (entry.val == NULL)
-        {
-          /* We must be in incremental mode */
-          assert(hash);
-          apr_hash_set(hash, entry.key, entry.keylen, NULL);
-          continue;
-        }
+  entries = apr_array_make(result_pool, (int)count,
+                           sizeof(svn_fs_x__dirent_t *));
 
-      /* Add a new directory entry. */
+  while (p != end)
+    {
+      apr_size_t len;
+      svn_fs_x__dirent_t *dirent;
       dirent = apr_pcalloc(result_pool, sizeof(*dirent));
-      dirent->name = apr_pstrmemdup(result_pool, entry.key, entry.keylen);
 
-      str = svn_cstring_tokenize(" ", &entry.val);
-      if (str == NULL)
+      /* The part of the serialized entry that is not the name will be
+       * about 6 bytes or less.  Since APR allocates with an 8 byte
+       * alignment (4 bytes loss on average per string), simply using
+       * the name string in DATA already gives us near-optimal memory
+       * usage. */
+      dirent->name = (const char *)p;
+      len = strlen(dirent->name);
+      p += len + 1;
+      if (p == end)
         return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
-                      _("Directory entry corrupt in '%s'"),
-                      svn_fs_x__id_unparse(id, scratch_pool)->data);
+                            _("Directory entry missing kind in '%s'"),
+                            svn_fs_x__id_unparse(id, scratch_pool)->data);
 
-      if (strcmp(str, SVN_FS_X__KIND_FILE) == 0)
-        {
-          dirent->kind = svn_node_file;
-        }
-      else if (strcmp(str, SVN_FS_X__KIND_DIR) == 0)
-        {
-          dirent->kind = svn_node_dir;
-        }
-      else
-        {
-          return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
-                      _("Directory entry corrupt in '%s'"),
-                      svn_fs_x__id_unparse(id, scratch_pool)->data);
-        }
+      dirent->kind = (svn_node_kind_t)*(p++);
+      if (p == end)
+        return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                            _("Directory entry missing change set in '%s'"),
+                            svn_fs_x__id_unparse(id, scratch_pool)->data);
 
-      str = svn_cstring_tokenize(" ", &entry.val);
-      if (str == NULL)
+      p = svn__decode_int(&dirent->id.change_set, p, end);
+      if (p == end)
         return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
-                      _("Directory entry corrupt in '%s'"),
-                      svn_fs_x__id_unparse(id, scratch_pool)->data);
+                            _("Directory entry missing item number in '%s'"),
+                            svn_fs_x__id_unparse(id, scratch_pool)->data);
 
-      SVN_ERR(svn_fs_x__id_parse(&dirent->id, str));
+      p = svn__decode_uint(&dirent->id.number, p, end);
 
       /* In incremental mode, update the hash; otherwise, write to the
        * final array. */
       if (incremental)
-        apr_hash_set(hash, dirent->name, entry.keylen, dirent);
+        {
+          /* Insertion / update or a deletion? */
+          if (svn_fs_x__id_used(&dirent->id))
+            apr_hash_set(hash, dirent->name, len, dirent);
+          else
+            apr_hash_set(hash, dirent->name, len, NULL);
+        }
       else
-        APR_ARRAY_PUSH(entries, svn_fs_x__dirent_t *) = dirent;
+        {
+          APR_ARRAY_PUSH(entries, svn_fs_x__dirent_t *) = dirent;
+        }
     }
 
-  /* Convert container to a sorted array. */
   if (incremental)
     {
+      /* Convert container into a sorted array. */
       apr_hash_index_t *hi;
-      for (hi = apr_hash_first(iterpool, hash); hi; hi = apr_hash_next(hi))
+      for (hi = apr_hash_first(scratch_pool, hash); hi; hi = apr_hash_next(hi))
         APR_ARRAY_PUSH(entries, svn_fs_x__dirent_t *) = apr_hash_this_val(hi);
+
+      if (!sorted(entries))
+        svn_sort__array(entries, compare_dirents);
+    }
+  else
+    {
+      /* Check that we read the expected amount of entries. */
+      if ((apr_uint64_t)entries->nelts != count)
+        return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                            _("Directory length mismatch in '%s'"),
+                            svn_fs_x__id_unparse(id, scratch_pool)->data);
     }
 
-  if (!sorted(entries))
-    svn_sort__array(entries, compare_dirents);
+ *entries_p = entries;
 
-  svn_pool_destroy(iterpool);
+  return SVN_NO_ERROR;
+}
+
+/* For directory NODEREV in FS, return the *FILESIZE of its in-txn
+ * representation.  If the directory representation is comitted data,
+ * set *FILESIZE to SVN_INVALID_FILESIZE. Use SCRATCH_POOL for temporaries.
+ */
+static svn_error_t *
+get_txn_dir_info(svn_filesize_t *filesize,
+                 svn_fs_t *fs,
+                 svn_fs_x__noderev_t *noderev,
+                 apr_pool_t *scratch_pool)
+{
+  if (noderev->data_rep
+      && ! svn_fs_x__is_revision(noderev->data_rep->id.change_set))
+    {
+      const svn_io_dirent2_t *dirent;
+      const char *filename;
+
+      filename = svn_fs_x__path_txn_node_children(fs, &noderev->noderev_id,
+                                                  scratch_pool, scratch_pool);
+
+      SVN_ERR(svn_io_stat_dirent2(&dirent, filename, FALSE, FALSE,
+                                  scratch_pool, scratch_pool));
+      *filesize = dirent->filesize;
+    }
+  else
+    {
+      *filesize = SVN_INVALID_FILESIZE;
+    }
 
   return SVN_NO_ERROR;
 }
 
-/* Fetch the contents of a directory into ENTRIES.  Values are stored
+/* Fetch the contents of a directory into DIR.  Values are stored
    as filename to string mappings; further conversion is necessary to
    convert them into svn_fs_x__dirent_t values. */
 static svn_error_t *
-get_dir_contents(apr_array_header_t **entries,
+get_dir_contents(svn_fs_x__dir_data_t *dir,
                  svn_fs_t *fs,
                  svn_fs_x__noderev_t *noderev,
                  apr_pool_t *result_pool,
@@ -2484,43 +2517,66 @@ get_dir_contents(apr_array_header_t **en
 {
   svn_stream_t *contents;
   const svn_fs_x__id_t *id = &noderev->noderev_id;
+  apr_size_t len;
+  svn_stringbuf_t *text;
+  svn_boolean_t incremental;
+
+  /* Initialize the result. */
+  dir->txn_filesize = SVN_INVALID_FILESIZE;
 
-  *entries = apr_array_make(result_pool, 16, sizeof(svn_fs_x__dirent_t *));
+  /* Read dir contents - unless there is none in which case we are done. */
   if (noderev->data_rep
       && ! svn_fs_x__is_revision(noderev->data_rep->id.change_set))
     {
-      const char *filename
-        = svn_fs_x__path_txn_node_children(fs, id, scratch_pool,
-                                           scratch_pool);
+      /* Get location & current size of the directory representation. */
+      const char *filename;
+      apr_file_t *file;
+
+      filename = svn_fs_x__path_txn_node_children(fs, id, scratch_pool,
+                                                  scratch_pool);
 
       /* The representation is mutable.  Read the old directory
          contents from the mutable children file, followed by the
          changes we've made in this transaction. */
-      SVN_ERR(svn_stream_open_readonly(&contents, filename, scratch_pool,
-                                       scratch_pool));
-      SVN_ERR(read_dir_entries(*entries, contents, TRUE,  id,
-                               result_pool, scratch_pool));
-      SVN_ERR(svn_stream_close(contents));
+      SVN_ERR(svn_io_file_open(&file, filename, APR_READ | APR_BUFFERED,
+                               APR_OS_DEFAULT, scratch_pool));
+
+      /* Obtain txn children file size. */
+      SVN_ERR(svn_io_file_size_get(&dir->txn_filesize, file, scratch_pool));
+      len = (apr_size_t)dir->txn_filesize;
+
+      /* Finally, provide stream access to FILE. */
+      contents = svn_stream_from_aprfile2(file, FALSE, scratch_pool);
+      incremental = TRUE;
     }
   else if (noderev->data_rep)
     {
-      /* Undeltify content before parsing it. Otherwise, we could only
-       * parse it byte-by-byte.
-       */
-      apr_size_t len = noderev->data_rep->expanded_size;
-      svn_stringbuf_t *text;
-
       /* The representation is immutable.  Read it normally. */
+      len = noderev->data_rep->expanded_size;
       SVN_ERR(svn_fs_x__get_contents(&contents, fs, noderev->data_rep,
                                      FALSE, scratch_pool));
-      SVN_ERR(svn_stringbuf_from_stream(&text, contents, len, scratch_pool));
-      SVN_ERR(svn_stream_close(contents));
-
-      /* de-serialize hash */
-      contents = svn_stream_from_stringbuf(text, scratch_pool);
-      SVN_ERR(read_dir_entries(*entries, contents, FALSE,  id,
-                               result_pool, scratch_pool));
+      incremental = FALSE;
     }
+  else
+    {
+      /* Empty representation == empty directory. */
+      dir->entries = apr_array_make(result_pool, 0,
+                                    sizeof(svn_fs_x__dirent_t *));
+      return SVN_NO_ERROR;
+    }
+
+  /* Read the whole stream contents into a single buffer.
+   * Due to our LEN hint, no allocation overhead occurs.
+   *
+   * Also, a large portion of TEXT will be file / dir names which we
+   * directly reference from DIR->ENTRIES instead of copying them.
+   * Hence, we need to use the RESULT_POOL here. */
+  SVN_ERR(svn_stringbuf_from_stream(&text, contents, len, result_pool));
+  SVN_ERR(svn_stream_close(contents));
+
+  /* de-serialize hash */
+  SVN_ERR(parse_dir_entries(&dir->entries, text, incremental, id,
+                            result_pool, scratch_pool));
 
   return SVN_NO_ERROR;
 }
@@ -2535,26 +2591,24 @@ locate_dir_cache(svn_fs_t *fs,
                  svn_fs_x__noderev_t *noderev)
 {
   svn_fs_x__data_t *ffd = fs->fsap_data;
-  if (svn_fs_x__is_txn(noderev->noderev_id.change_set))
+
+  if (!noderev->data_rep)
+    {
+      /* no data rep -> empty directory.
+         Use a key that does definitely not clash with non-NULL reps. */
+      key->change_set = SVN_FS_X__INVALID_CHANGE_SET;
+      key->number = SVN_FS_X__ITEM_INDEX_UNUSED;
+    }
+  else if (svn_fs_x__is_txn(noderev->noderev_id.change_set))
     {
-      /* data in txns must be addressed by ID since the representation has
-         not been created, yet. */
+      /* data in txns must be addressed by noderev ID since the
+         representation has not been created, yet. */
       *key = noderev->noderev_id;
     }
   else
     {
       /* committed data can use simple rev,item pairs */
-      if (noderev->data_rep)
-        {
-          *key = noderev->data_rep->id;
-        }
-      else
-        {
-          /* no data rep -> empty directory.
-             Use a key that does definitely not clash with non-NULL reps. */
-          key->change_set = SVN_FS_X__INVALID_CHANGE_SET;
-          key->number = SVN_FS_X__ITEM_INDEX_UNUSED;
-        }
+      *key = noderev->data_rep->id;
     }
 
   return ffd->dir_cache;
@@ -2568,22 +2622,40 @@ svn_fs_x__rep_contents_dir(apr_array_hea
                            apr_pool_t *scratch_pool)
 {
   svn_fs_x__id_t key;
+  svn_fs_x__dir_data_t *dir;
 
   /* find the cache we may use */
   svn_cache__t *cache = locate_dir_cache(fs, &key, noderev);
   svn_boolean_t found;
 
-  SVN_ERR(svn_cache__get((void **)entries_p, &found, cache, &key,
-                         result_pool));
+  SVN_ERR(svn_cache__get((void **)&dir, &found, cache, &key, result_pool));
   if (found)
-    return SVN_NO_ERROR;
+    {
+      /* Verify that the cached dir info is not stale
+       * (no-op for committed data). */
+      svn_filesize_t filesize;
+      SVN_ERR(get_txn_dir_info(&filesize, fs, noderev, scratch_pool));
+
+      if (filesize == dir->txn_filesize)
+        {
+          /* Still valid. Done. */
+          *entries_p = dir->entries;
+          return SVN_NO_ERROR;
+        }
+    }
 
   /* Read in the directory contents. */
-  SVN_ERR(get_dir_contents(entries_p, fs, noderev, result_pool,
-                           scratch_pool));
+  dir = apr_pcalloc(scratch_pool, sizeof(*dir));
+  SVN_ERR(get_dir_contents(dir, fs, noderev, result_pool, scratch_pool));
+  *entries_p = dir->entries;
 
-  /* Update the cache, if we are to use one. */
-  SVN_ERR(svn_cache__set(cache, &key, *entries_p, scratch_pool));
+  /* Update the cache, if we are to use one.
+   *
+   * Don't even attempt to serialize very large directories; it would cause
+   * an unnecessary memory allocation peak.  100 bytes/entry is about right.
+   */
+  if (svn_cache__is_cachable(cache, 100 * dir->entries->nelts))
+    SVN_ERR(svn_cache__set(cache, &key, dir, scratch_pool));
 
   return SVN_NO_ERROR;
 }
@@ -2613,10 +2685,15 @@ svn_fs_x__rep_contents_dir_entry(svn_fs_
   svn_fs_x__id_t key;
   svn_cache__t *cache = locate_dir_cache(fs, &key, noderev);
   svn_fs_x__ede_baton_t baton;
+
+  svn_filesize_t filesize;
+  SVN_ERR(get_txn_dir_info(&filesize, fs, noderev, scratch_pool));
+
+   /* Cache lookup. */
   baton.hint = *hint;
   baton.name = name;
+  baton.txn_filesize = filesize;
 
-  /* Cache lookup. */
   SVN_ERR(svn_cache__get_partial((void **)dirent,
                                  &found,
                                  cache,
@@ -2630,19 +2707,26 @@ svn_fs_x__rep_contents_dir_entry(svn_fs_
     *hint = baton.hint;
 
   /* fetch data from disk if we did not find it in the cache */
-  if (! found)
+  if (! found || baton.out_of_date)
     {
-      apr_array_header_t *entries;
       svn_fs_x__dirent_t *entry;
       svn_fs_x__dirent_t *entry_copy = NULL;
+      svn_fs_x__dir_data_t dir;
+
+      /* Read in the directory contents. */
+      SVN_ERR(get_dir_contents(&dir, fs, noderev, scratch_pool,
+                               scratch_pool));
 
-      /* read the dir from the file system. It will probably be put it
-         into the cache for faster lookup in future calls. */
-      SVN_ERR(svn_fs_x__rep_contents_dir(&entries, fs, noderev,
-                                         scratch_pool, scratch_pool));
+      /* Update the cache, if we are to use one.
+       *
+       * Don't even attempt to serialize very large directories; it would
+       * cause an unnecessary memory allocation peak.  150 bytes / entry is
+       * about right. */
+      if (cache && svn_cache__is_cachable(cache, 150 * dir.entries->nelts))
+        SVN_ERR(svn_cache__set(cache, &key, &dir, scratch_pool));
 
       /* find desired entry and return a copy in POOL, if found */
-      entry = svn_fs_x__find_dir_entry(entries, name, NULL);
+      entry = svn_fs_x__find_dir_entry(dir.entries, name, NULL);
       if (entry)
         {
           entry_copy = apr_pmemdup(result_pool, entry, sizeof(*entry_copy));
@@ -2656,135 +2740,156 @@ svn_fs_x__rep_contents_dir_entry(svn_fs_
 }
 
 svn_error_t *
-svn_fs_x__get_proplist(apr_hash_t **proplist_p,
+svn_fs_x__get_proplist(apr_hash_t **proplist,
                        svn_fs_t *fs,
                        svn_fs_x__noderev_t *noderev,
                        apr_pool_t *result_pool,
                        apr_pool_t *scratch_pool)
 {
-  apr_hash_t *proplist;
   svn_stream_t *stream;
   const svn_fs_x__id_t *noderev_id = &noderev->noderev_id;
 
   if (noderev->prop_rep
       && !svn_fs_x__is_revision(noderev->prop_rep->id.change_set))
     {
+      svn_stringbuf_t *content;
+      svn_string_t *as_string;
       const char *filename = svn_fs_x__path_txn_node_props(fs, noderev_id,
                                                            scratch_pool,
                                                            scratch_pool);
-      proplist = apr_hash_make(result_pool);
+      SVN_ERR(svn_stringbuf_from_file2(&content, filename, result_pool));
 
-      SVN_ERR(svn_stream_open_readonly(&stream, filename, scratch_pool,
-                                       scratch_pool));
-      SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR,
-                             result_pool));
-      SVN_ERR(svn_stream_close(stream));
+      as_string = svn_stringbuf__morph_into_string(content);
+      SVN_ERR_W(svn_fs_x__parse_properties(proplist, as_string, result_pool),
+                apr_psprintf(scratch_pool,
+                    "malformed property list for node-revision '%s' in '%s'",
+                    svn_fs_x__id_unparse(&noderev->noderev_id,
+                                         scratch_pool)->data,
+                    filename));
     }
   else if (noderev->prop_rep)
     {
       svn_fs_x__data_t *ffd = fs->fsap_data;
       svn_fs_x__representation_t *rep = noderev->prop_rep;
       svn_fs_x__pair_cache_key_t key = { 0 };
+      svn_string_t *content;
+      svn_boolean_t is_cached;
 
       key.revision = svn_fs_x__get_revnum(rep->id.change_set);
       key.second = rep->id.number;
-      if (SVN_IS_VALID_REVNUM(key.revision))
-        {
-          svn_boolean_t is_cached;
-          SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached,
-                                 ffd->properties_cache, &key, result_pool));
-          if (is_cached)
-            return SVN_NO_ERROR;
-        }
+      SVN_ERR(svn_cache__get((void **) proplist, &is_cached,
+                             ffd->properties_cache, &key, result_pool));
+      if (is_cached)
+        return SVN_NO_ERROR;
 
-      proplist = apr_hash_make(result_pool);
-      SVN_ERR(svn_fs_x__get_contents(&stream, fs, noderev->prop_rep, FALSE,
-                                     scratch_pool));
-      SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR,
-                             result_pool));
-      SVN_ERR(svn_stream_close(stream));
+      SVN_ERR(svn_fs_x__get_contents(&stream, fs, rep, FALSE, scratch_pool));
+      SVN_ERR(svn_string_from_stream2(&content, stream, rep->expanded_size,
+                                      result_pool));
+
+      SVN_ERR_W(svn_fs_x__parse_properties(proplist, content, result_pool),
+                apr_psprintf(scratch_pool,
+                    "malformed property list for node-revision '%s'",
+                    svn_fs_x__id_unparse(&noderev->noderev_id,
+                                         scratch_pool)->data));
 
-      if (SVN_IS_VALID_REVNUM(rep->id.change_set))
-        SVN_ERR(svn_cache__set(ffd->properties_cache, &key, proplist,
-                               scratch_pool));
+      SVN_ERR(svn_cache__set(ffd->properties_cache, &key, *proplist,
+                             scratch_pool));
     }
   else
     {
       /* return an empty prop list if the node doesn't have any props */
-      proplist = apr_hash_make(result_pool);
+      *proplist = apr_hash_make(result_pool);
     }
 
-  *proplist_p = proplist;
-
   return SVN_NO_ERROR;
 }
 
+svn_error_t *
+svn_fs_x__create_changes_context(svn_fs_x__changes_context_t **context,
+                                 svn_fs_t *fs,
+                                 svn_revnum_t rev,
+                                 apr_pool_t *result_pool,
+                                 apr_pool_t *scratch_pool)
+{
+  svn_fs_x__changes_context_t *result = apr_pcalloc(result_pool,
+                                                    sizeof(*result));
+  result->fs = fs;
+  result->revision = rev;
+
+  SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, scratch_pool));
+  SVN_ERR(svn_fs_x__rev_file_init(&result->revision_file, fs, rev,
+                                  result_pool));
 
+  *context = result;
+  return SVN_NO_ERROR;
+}
 
 svn_error_t *
 svn_fs_x__get_changes(apr_array_header_t **changes,
-                      svn_fs_t *fs,
-                      svn_revnum_t rev,
-                      apr_pool_t *result_pool)
+                      svn_fs_x__changes_context_t *context,
+                      apr_pool_t *result_pool,
+                      apr_pool_t *scratch_pool)
 {
-  svn_fs_x__revision_file_t *revision_file;
   svn_boolean_t found;
-  svn_fs_x__data_t *ffd = fs->fsap_data;
-  apr_pool_t *scratch_pool = svn_pool_create(result_pool);
+  svn_fs_x__data_t *ffd = context->fs->fsap_data;
 
   svn_fs_x__id_t id;
-  id.change_set = svn_fs_x__change_set_by_rev(rev);
+  id.change_set = svn_fs_x__change_set_by_rev(context->revision);
   id.number = SVN_FS_X__ITEM_INDEX_CHANGES;
 
-  /* Provide revision file. */
-
-  SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, scratch_pool));
-  SVN_ERR(svn_fs_x__rev_file_init(&revision_file, fs, rev, scratch_pool));
-
   /* try cache lookup first */
 
-  if (svn_fs_x__is_packed_rev(fs, rev))
+  if (svn_fs_x__is_packed_rev(context->fs, context->revision))
     {
       apr_off_t offset;
-      apr_uint32_t sub_item;
       svn_fs_x__pair_cache_key_t key;
+      svn_fs_x__changes_get_list_baton_t baton;
+      baton.start = (int)context->next;
+      baton.eol = &context->eol;
 
-      SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, revision_file,
+      SVN_ERR(svn_fs_x__item_offset(&offset, &baton.sub_item, context->fs,
+                                    context->revision_file,
                                     &id, scratch_pool));
-      key.revision = svn_fs_x__packed_base_rev(fs, rev);
+      key.revision = svn_fs_x__packed_base_rev(context->fs,
+                                               context->revision);
       key.second = offset;
 
       SVN_ERR(svn_cache__get_partial((void **)changes, &found,
                                      ffd->changes_container_cache, &key,
                                      svn_fs_x__changes_get_list_func,
-                                     &sub_item, result_pool));
+                                     &baton, result_pool));
     }
   else
     {
-      SVN_ERR(svn_cache__get((void **) changes, &found, ffd->changes_cache,
-                             &rev, result_pool));
+      svn_fs_x__read_changes_block_baton_t baton;
+      baton.start = (int)context->next;
+      baton.eol = &context->eol;
+
+      SVN_ERR(svn_cache__get_partial((void **)changes, &found,
+                                     ffd->changes_cache, &context->revision,
+                                     svn_fs_x__read_changes_block,
+                                     &baton, result_pool));
     }
 
   if (!found)
     {
       /* 'block-read' will also provide us with the desired data */
-      SVN_ERR(block_read((void **)changes, fs, &id, revision_file,
+      SVN_ERR(block_read((void **)changes, context->fs, &id,
+                         context->revision_file, context,
                          result_pool, scratch_pool));
-
-      SVN_ERR(svn_fs_x__close_revision_file(revision_file));
     }
 
-  SVN_ERR(dgb__log_access(fs, &id, *changes, SVN_FS_X__ITEM_TYPE_CHANGES,
-                          scratch_pool));
+  context->next += (*changes)->nelts;
+
+  SVN_ERR(dgb__log_access(context->fs, &id, *changes,
+                          SVN_FS_X__ITEM_TYPE_CHANGES, scratch_pool));
 
-  svn_pool_destroy(scratch_pool);
   return SVN_NO_ERROR;
 }
 
 /* Fetch the representation data (header, txdelta / plain windows)
- * addressed by ENTRY->ITEM in FS and cache it if caches are enabled.
- * Read the data from the already open FILE and the wrapping
- * STREAM object.  If MAX_OFFSET is not -1, don't read windows that start
+ * addressed by ENTRY->ITEM in FS and cache it under KEY.  Read the data
+ * from REV_FILE.  If MAX_OFFSET is not -1, don't read windows that start
  * at or beyond that offset.  Use SCRATCH_POOL for temporary allocations.
  */
 static svn_error_t *
@@ -2814,27 +2919,28 @@ block_read_contents(svn_fs_t *fs,
 
 /* For the given REV_FILE in FS, in *STREAM return a stream covering the
  * item specified by ENTRY.  Also, verify the item's content by low-level
- * checksum.  Allocate the result in POOL.
+ * checksum.  Allocate the result in RESULT_POOL.
  */
 static svn_error_t *
 read_item(svn_stream_t **stream,
           svn_fs_t *fs,
           svn_fs_x__revision_file_t *rev_file,
           svn_fs_x__p2l_entry_t* entry,
-          apr_pool_t *pool)
+          apr_pool_t *result_pool)
 {
   apr_uint32_t digest;
   svn_checksum_t *expected, *actual;
   apr_uint32_t plain_digest;
+  svn_stringbuf_t *text;
 
   /* Read item into string buffer. */
-  svn_stringbuf_t *text = svn_stringbuf_create_ensure(entry->size, pool);
+  text = svn_stringbuf_create_ensure(entry->size, result_pool);
   text->len = entry->size;
   text->data[text->len] = 0;
   SVN_ERR(svn_fs_x__rev_file_read(rev_file, text->data, text->len));
 
   /* Return (construct, calculate) stream and checksum. */
-  *stream = svn_stream_from_stringbuf(text, pool);
+  *stream = svn_stream_from_stringbuf(text, result_pool);
   digest = svn__fnv1a_32x4(text->data, text->len);
 
   /* Checksums will match most of the time. */
@@ -2845,30 +2951,31 @@ read_item(svn_stream_t **stream,
    * nice error messages. */
   plain_digest = htonl(entry->fnv1_checksum);
   expected = svn_checksum__from_digest_fnv1a_32x4(
-                (const unsigned char *)&plain_digest, pool);
+                (const unsigned char *)&plain_digest, result_pool);
   plain_digest = htonl(digest);
   actual = svn_checksum__from_digest_fnv1a_32x4(
-                (const unsigned char *)&plain_digest, pool);
+                (const unsigned char *)&plain_digest, result_pool);
 
   /* Construct the full error message with all the info we have. */
-  return svn_checksum_mismatch_err(expected, actual, pool,
+  return svn_checksum_mismatch_err(expected, actual, result_pool,
                  _("Low-level checksum mismatch while reading\n"
                    "%s bytes of meta data at offset %s "),
-                 apr_psprintf(pool, "%" APR_OFF_T_FMT, entry->size),
-                 apr_psprintf(pool, "%" APR_OFF_T_FMT, entry->offset));
+                 apr_psprintf(result_pool, "%" APR_OFF_T_FMT, entry->size),
+                 apr_psprintf(result_pool, "%" APR_OFF_T_FMT, entry->offset));
 }
 
-/* Read all txdelta / plain windows following REP_HEADER in FS as described
- * by ENTRY.  Read the data from the already open FILE and the wrapping
- * STREAM object.  If MAX_OFFSET is not -1, don't read windows that start
- * at or beyond that offset.  Use SCRATCH_POOL for temporary allocations.
- * If caching is not enabled, this is a no-op.
+/* If not already cached or if MUST_READ is set, read the changed paths
+ * list addressed by ENTRY in FS and retúrn it in *CHANGES.  Cache the
+ * result if caching is enabled.  Read the data from REV_FILE.  Trim the
+ * data in *CHANGES to the range given by CONTEXT.  Allocate *CHANGES in
+ * RESUSLT_POOL and allocate temporaries in SCRATCH_POOL.
  */
 static svn_error_t *
 block_read_changes(apr_array_header_t **changes,
                    svn_fs_t *fs,
                    svn_fs_x__revision_file_t *rev_file,
                    svn_fs_x__p2l_entry_t* entry,
+                   svn_fs_x__changes_context_t *context,
                    svn_boolean_t must_read,
                    apr_pool_t *result_pool,
                    apr_pool_t *scratch_pool)
@@ -2909,15 +3016,43 @@ block_read_changes(apr_array_header_t **
     SVN_ERR(svn_cache__set(ffd->changes_cache, &revision, *changes,
                            scratch_pool));
 
+  /* Trim the result:
+   * Remove the entries that already been reported. */
+
+  /* TODO: This is transitional code.
+   *       The final implementation will read and cache only sections of
+   *       the changes list. */
+  if (must_read)
+    {
+      int i;
+      for (i = 0; i + context->next < (*changes)->nelts; ++i)
+        {
+          APR_ARRAY_IDX(*changes, i, svn_fs_x__change_t *)
+            = APR_ARRAY_IDX(*changes, i + context->next,
+                            svn_fs_x__change_t *);
+        }
+      (*changes)->nelts = i;
+
+      context->eol = TRUE;
+    }
+
   return SVN_NO_ERROR;
 }
 
+/* If not already cached or if MUST_READ is set, read the changed paths
+ * list container addressed by ENTRY in FS.  Return the changes list
+ * identified by SUB_ITEM in *CHANGES, using CONTEXT to select a sub-range
+ * within that list.  Read the data from REV_FILE and cache the result.
+ *
+ * Allocate *CHANGES in RESUSLT_POOL and everything else in SCRATCH_POOL.
+ */
 static svn_error_t *
 block_read_changes_container(apr_array_header_t **changes,
                              svn_fs_t *fs,
                              svn_fs_x__revision_file_t *rev_file,
                              svn_fs_x__p2l_entry_t* entry,
                              apr_uint32_t sub_item,
+                             svn_fs_x__changes_context_t *context,
                              svn_boolean_t must_read,
                              apr_pool_t *result_pool,
                              apr_pool_t *scratch_pool)
@@ -2952,13 +3087,19 @@ block_read_changes_container(apr_array_h
 
   if (must_read)
     SVN_ERR(svn_fs_x__changes_get_list(changes, container, sub_item,
-                                       result_pool));
+                                       context, result_pool));
   SVN_ERR(svn_cache__set(ffd->changes_container_cache, &key, container,
                          scratch_pool));
 
   return SVN_NO_ERROR;
 }
 
+/* If not already cached or if MUST_READ is set, read the node revision
+ * addressed by ENTRY in FS and return it in *NODEREV_P.  Cache the
+ * result under KEY if caching is enabled.  Read the data from REV_FILE.
+ * Allocate *NODEREV_P in RESUSLT_POOL and allocate temporaries in
+ * SCRATCH_POOL.
+ */
 static svn_error_t *
 block_read_noderev(svn_fs_x__noderev_t **noderev_p,
                    svn_fs_t *fs,
@@ -2997,6 +3138,12 @@ block_read_noderev(svn_fs_x__noderev_t *
   return SVN_NO_ERROR;
 }
 
+/* If not already cached or if MUST_READ is set, read the node revision
+ * container addressed by ENTRY in FS.  Return the item identified by
+ * SUB_ITEM in *NODEREV_P.  Read the data from REV_FILE and cache it.
+ * Allocate *NODEREV_P in RESUSLT_POOL and allocate temporaries in
+ * SCRATCH_POOL.
+ */
 static svn_error_t *
 block_read_noderevs_container(svn_fs_x__noderev_t **noderev_p,
                               svn_fs_t *fs,
@@ -3043,6 +3190,12 @@ block_read_noderevs_container(svn_fs_x__
   return SVN_NO_ERROR;
 }
 
+/* If not already cached or if MUST_READ is set, read the representation
+ * container addressed by ENTRY in FS.  Return an extractor object for the
+ * item identified by SUB_ITEM in *EXTRACTOR.  Read the data from REV_FILE
+ * and cache it.  Allocate *EXTRACTOR in RESUSLT_POOL and all temporaries
+ * in SCRATCH_POOL.
+ */
 static svn_error_t *
 block_read_reps_container(svn_fs_x__rep_extractor_t **extractor,
                           svn_fs_t *fs,
@@ -3090,11 +3243,24 @@ block_read_reps_container(svn_fs_x__rep_
   return SVN_NO_ERROR;
 }
 
+/* Read the whole (e.g. 64kB) block containing the item identified by ID in
+ * FS and put all data into cache.  If necessary and depending on heuristics,
+ * neighboring blocks may also get read.  The data is being read from
+ * already open REVISION_FILE, which must be the correct rev / pack file
+ * w.r.t. ID->CHANGE_SET.
+ *
+ * For noderevs and changed path lists, the item fetched can be allocated
+ * RESULT_POOL and returned in *RESULT.  Otherwise, RESULT must be NULL.
+ * The BATON is passed along to the extractor sub-functions and will be
+ * used only when constructing the *RESULT.  SCRATCH_POOL will be used for
+ * all temporary allocations.
+ */
 static svn_error_t *
 block_read(void **result,
            svn_fs_t *fs,
            const svn_fs_x__id_t *id,
            svn_fs_x__revision_file_t *revision_file,
+           void *baton,
            apr_pool_t *result_pool,
            apr_pool_t *scratch_pool)
 {
@@ -3185,7 +3351,7 @@ block_read(void **result,
                   case SVN_FS_X__ITEM_TYPE_CHANGES:
                     SVN_ERR(block_read_changes((apr_array_header_t **)&item,
                                                fs, revision_file,
-                                               entry, is_result,
+                                               entry, baton, is_result,
                                                pool, iterpool));
                     break;
 
@@ -3194,7 +3360,8 @@ block_read(void **result,
                                             ((apr_array_header_t **)&item,
                                              fs, revision_file,
                                              entry, wanted_sub_item,
-                                             is_result, pool, iterpool));
+                                             baton, is_result,
+                                             pool, iterpool));
                     break;
 
                   case SVN_FS_X__ITEM_TYPE_NODEREVS_CONT:
@@ -3223,7 +3390,7 @@ block_read(void **result,
               /* if we crossed a block boundary, read the remainder of
                * the last block as well */
               offset = entry->offset + entry->size;
-              if (offset > block_start + ffd->block_size)
+              if (offset - block_start > ffd->block_size)
                 ++run_count;
 
               svn_pool_clear(iterpool);

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/cached_data.h
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/cached_data.h?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/cached_data.h (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/cached_data.h Fri Apr 29 18:38:53 2016
@@ -168,13 +168,24 @@ svn_fs_x__get_proplist(apr_hash_t **prop
                        apr_pool_t *result_pool,
                        apr_pool_t *scratch_pool);
 
-/* Fetch the list of change in revision REV in FS and return it in *CHANGES.
- * Allocate the result in POOL.
+/* Create a changes retrieval context object in *RESULT_POOL and return it
+ * in *CONTEXT.  It will allow svn_fs_x__get_changes to fetch consecutive
+ * blocks (one per invocation) from REV's changed paths list in FS.
+ * Use SCRATCH_POOL for temporary allocations. */
+svn_error_t *
+svn_fs_x__create_changes_context(svn_fs_x__changes_context_t **context,
+                                 svn_fs_t *fs,
+                                 svn_revnum_t rev,
+                                 apr_pool_t *result_pool,
+                                 apr_pool_t *scratch_pool);
+
+/* Fetch the block of changes from the CONTEXT and return it in *CHANGES.
+ * Allocate the result in RESULT_POOL and use SCRATCH_POOL for temporaries.
  */
 svn_error_t *
 svn_fs_x__get_changes(apr_array_header_t **changes,
-                      svn_fs_t *fs,
-                      svn_revnum_t rev,
-                      apr_pool_t *pool);
+                      svn_fs_x__changes_context_t *context,
+                      apr_pool_t *result_pool,
+                      apr_pool_t *scratch_pool);
 
 #endif

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/caching.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/caching.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/caching.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/caching.c Fri Apr 29 18:38:53 2016
@@ -69,9 +69,9 @@ normalize_key_part(const char *original,
   return normalized->data;
 }
 
-/* *CACHE_TXDELTAS, *CACHE_FULLTEXTS and *CACHE_REVPROPS flags will be set
-   according to FS->CONFIG.  *CACHE_NAMESPACE receives the cache prefix
-   to use.
+/* *CACHE_TXDELTAS, *CACHE_FULLTEXTS, *CACHE_REVPROPS and *CACHE_NODEPROPS
+   flags will be set according to FS->CONFIG.  *CACHE_NAMESPACE receives
+   the cache prefix to use.
 
    Allocate CACHE_NAMESPACE in RESULT_POOL. */
 static svn_error_t *
@@ -79,6 +79,7 @@ read_config(const char **cache_namespace
             svn_boolean_t *cache_txdeltas,
             svn_boolean_t *cache_fulltexts,
             svn_boolean_t *cache_revprops,
+            svn_boolean_t *cache_nodeprops,
             svn_fs_t *fs,
             apr_pool_t *result_pool)
 {
@@ -137,6 +138,15 @@ read_config(const char **cache_namespace
   else
     *cache_revprops = TRUE;
 
+  /* by default, cache nodeprops: this will match pre-1.10
+   * behavior where node properties caching was controlled
+   * by SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS configuration option.
+   */
+  *cache_nodeprops
+    = svn_hash__get_bool(fs->config,
+                         SVN_FS_CONFIG_FSFS_CACHE_NODEPROPS,
+                         TRUE);
+
   return SVN_NO_ERROR;
 }
 
@@ -370,6 +380,7 @@ svn_fs_x__initialize_caches(svn_fs_t *fs
   svn_fs_x__data_t *ffd = fs->fsap_data;
   const char *prefix = apr_pstrcat(scratch_pool,
                                    "fsx:", fs->uuid,
+                                   "--", ffd->instance_id,
                                    "/", normalize_key_part(fs->path,
                                                            scratch_pool),
                                    ":",
@@ -379,6 +390,7 @@ svn_fs_x__initialize_caches(svn_fs_t *fs
   svn_boolean_t cache_txdeltas;
   svn_boolean_t cache_fulltexts;
   svn_boolean_t cache_revprops;
+  svn_boolean_t cache_nodeprops;
   const char *cache_namespace;
   svn_boolean_t has_namespace;
 
@@ -387,6 +399,7 @@ svn_fs_x__initialize_caches(svn_fs_t *fs
                       &cache_txdeltas,
                       &cache_fulltexts,
                       &cache_revprops,
+                      &cache_nodeprops,
                       fs,
                       scratch_pool));
 
@@ -429,7 +442,7 @@ svn_fs_x__initialize_caches(svn_fs_t *fs
                        svn_fs_x__deserialize_dir_entries,
                        sizeof(svn_fs_x__id_t),
                        apr_pstrcat(scratch_pool, prefix, "DIR", SVN_VA_NULL),
-                       SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
+                       SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
                        has_namespace,
                        fs,
                        no_handler, FALSE,
@@ -511,7 +524,7 @@ svn_fs_x__initialize_caches(svn_fs_t *fs
                        SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
                        has_namespace,
                        fs,
-                       no_handler, !cache_fulltexts,
+                       no_handler, !cache_nodeprops,
                        fs->pool, scratch_pool));
 
   /* if enabled, cache revprops */

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/changes.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/changes.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/changes.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/changes.c Fri Apr 29 18:38:53 2016
@@ -21,6 +21,7 @@
  */
 
 #include "svn_private_config.h"
+#include "svn_sorts.h"
 
 #include "private/svn_packed_data.h"
 
@@ -37,8 +38,8 @@
 /* the change contains a property modification */
 #define CHANGE_PROP_MOD     0x00002
 
-/* the last part (rev_id) of node revision ID is a transaction ID */
-#define CHANGE_TXN_NODE     0x00004
+/* the change contains a mergeinfo modification */
+#define CHANGE_MERGEINFO_MOD 0x00004
 
 /* (flags & CHANGE_NODE_MASK) >> CHANGE_NODE_SHIFT extracts the node type */
 #define CHANGE_NODE_SHIFT   0x00003
@@ -52,16 +53,13 @@
 
 /* (flags & CHANGE_KIND_MASK) >> CHANGE_KIND_SHIFT extracts the change type */
 #define CHANGE_KIND_SHIFT   0x00005
-#define CHANGE_KIND_MASK    0x000E0
+#define CHANGE_KIND_MASK    0x00060
 
 /* node types according to svn_fs_path_change_kind_t */
 #define CHANGE_KIND_MODIFY  0x00000
 #define CHANGE_KIND_ADD     0x00020
 #define CHANGE_KIND_DELETE  0x00040
 #define CHANGE_KIND_REPLACE 0x00060
-#define CHANGE_KIND_RESET   0x00080
-#define CHANGE_KIND_MOVE    0x000A0
-#define CHANGE_KIND_MOVEREPLACE 0x000C0
 
 /* Our internal representation of a change */
 typedef struct binary_change_t
@@ -77,10 +75,6 @@ typedef struct binary_change_t
   svn_revnum_t copyfrom_rev;
   apr_size_t copyfrom_path;
 
-  /* Relevant parts of the node revision ID of the change.
-   * Empty, if REV_ID is not "used". */
-  svn_fs_x__id_t noderev_id;
-
 } binary_change_t;
 
 /* The actual container object.  Change lists are concatenated into CHANGES
@@ -138,20 +132,16 @@ append_change(svn_fs_x__changes_t *chang
               svn_fs_x__change_t *change)
 {
   binary_change_t binary_change = { 0 };
-  svn_boolean_t is_txn_id;
 
   /* CHANGE must be sufficiently complete */
   SVN_ERR_ASSERT(change);
   SVN_ERR_ASSERT(change->path.data);
 
-  /* Relevant parts of the revision ID of the change. */
-  binary_change.noderev_id = change->noderev_id;
-
   /* define the kind of change and what specific information is present */
-  is_txn_id = svn_fs_x__is_txn(binary_change.noderev_id.change_set);
   binary_change.flags = (change->text_mod ? CHANGE_TEXT_MOD : 0)
                       | (change->prop_mod ? CHANGE_PROP_MOD : 0)
-                      | (is_txn_id ? CHANGE_TXN_NODE : 0)
+                      | (change->mergeinfo_mod == svn_tristate_true
+                                          ? CHANGE_MERGEINFO_MOD : 0)
                       | ((int)change->change_kind << CHANGE_KIND_SHIFT)
                       | ((int)change->node_kind << CHANGE_NODE_SHIFT);
 
@@ -222,12 +212,19 @@ svn_error_t *
 svn_fs_x__changes_get_list(apr_array_header_t **list,
                            const svn_fs_x__changes_t *changes,
                            apr_size_t idx,
+                           svn_fs_x__changes_context_t *context,
                            apr_pool_t *result_pool)
 {
+  int list_first;
+  int list_last;
   int first;
   int last;
   int i;
 
+  /* Return up to this many entries.  Anything > 0 will do.
+   * At 100..300 bytes per entry, this limits the allocation to ~30kB. */
+  enum { BLOCK_SIZE = 100 };
+
   /* CHANGES must be in 'finalized' mode */
   SVN_ERR_ASSERT(changes->builder == NULL);
   SVN_ERR_ASSERT(changes->paths);
@@ -242,8 +239,16 @@ svn_fs_x__changes_get_list(apr_array_hea
                              idx, changes->offsets->nelts - 1);
 
   /* range of changes to return */
-  first = APR_ARRAY_IDX(changes->offsets, (int)idx, int);
-  last = APR_ARRAY_IDX(changes->offsets, (int)idx + 1, int);
+  list_first = APR_ARRAY_IDX(changes->offsets, (int)idx, int);
+  list_last = APR_ARRAY_IDX(changes->offsets, (int)idx + 1, int);
+
+  /* Restrict it to the sub-range requested by the caller.
+   * Clip the range to never exceed the list's content. */
+  first = MIN(context->next + list_first, list_last);
+  last = MIN(first + BLOCK_SIZE, list_last);
+
+  /* Indicate to the caller whether the end of the list has been reached. */
+  context->eol = last == list_last;
 
   /* construct result */
   *list = apr_array_make(result_pool, last - first,
@@ -260,13 +265,13 @@ svn_fs_x__changes_get_list(apr_array_hea
                                                      &change->path.len,
                                                      result_pool);
 
-      if (binary_change->noderev_id.change_set != SVN_FS_X__INVALID_CHANGE_SET)
-        change->noderev_id = binary_change->noderev_id;
-
       change->change_kind = (svn_fs_path_change_kind_t)
         ((binary_change->flags & CHANGE_KIND_MASK) >> CHANGE_KIND_SHIFT);
       change->text_mod = (binary_change->flags & CHANGE_TEXT_MOD) != 0;
       change->prop_mod = (binary_change->flags & CHANGE_PROP_MOD) != 0;
+      change->mergeinfo_mod = (binary_change->flags & CHANGE_MERGEINFO_MOD)
+                            ? svn_tristate_true
+                            : svn_tristate_false;
       change->node_kind = (svn_node_kind_t)
         ((binary_change->flags & CHANGE_NODE_MASK) >> CHANGE_NODE_SHIFT);
 
@@ -312,8 +317,6 @@ svn_fs_x__write_changes_container(svn_st
   svn_packed__create_int_substream(changes_stream, TRUE, FALSE);
   svn_packed__create_int_substream(changes_stream, TRUE, TRUE);
   svn_packed__create_int_substream(changes_stream, TRUE, FALSE);
-  svn_packed__create_int_substream(changes_stream, TRUE, TRUE);
-  svn_packed__create_int_substream(changes_stream, TRUE, FALSE);
 
   /* serialize offsets array */
   for (i = 0; i < changes->offsets->nelts; ++i)
@@ -331,9 +334,6 @@ svn_fs_x__write_changes_container(svn_st
 
       svn_packed__add_int(changes_stream, change->copyfrom_rev);
       svn_packed__add_uint(changes_stream, change->copyfrom_path);
-
-      svn_packed__add_int(changes_stream, change->noderev_id.change_set);
-      svn_packed__add_uint(changes_stream, change->noderev_id.number);
     }
 
   /* write to disk */
@@ -388,9 +388,6 @@ svn_fs_x__read_changes_container(svn_fs_
       change.copyfrom_rev = (svn_revnum_t)svn_packed__get_int(changes_stream);
       change.copyfrom_path = (apr_size_t)svn_packed__get_uint(changes_stream);
 
-      change.noderev_id.change_set = svn_packed__get_int(changes_stream);
-      change.noderev_id.number = svn_packed__get_uint(changes_stream);
-
       APR_ARRAY_PUSH(changes->changes, binary_change_t) = change;
     }
 
@@ -464,8 +461,10 @@ svn_fs_x__changes_get_list_func(void **o
   int last;
   int i;
   apr_array_header_t *list;
+  enum { BLOCK_SIZE = 100 };
 
-  apr_uint32_t idx = *(apr_uint32_t *)baton;
+  svn_fs_x__changes_get_list_baton_t *b = baton;
+  apr_uint32_t idx = b->sub_item;
   const svn_fs_x__changes_t *container = data;
 
   /* resolve all the sub-container pointers we need */
@@ -496,6 +495,12 @@ svn_fs_x__changes_get_list_func(void **o
   first = offsets[idx];
   last = offsets[idx+1];
 
+  /* Restrict range to the block requested by the BATON.
+   * Tell the caller whether we reached the end of the list. */
+  first = MIN(first + b->start, last);
+  last = MIN(first + BLOCK_SIZE, last);
+  *b->eol = last == offsets[idx+1];
+
   /* construct result */
   list = apr_array_make(pool, last - first, sizeof(svn_fs_x__change_t*));
 
@@ -509,8 +514,6 @@ svn_fs_x__changes_get_list_func(void **o
         = svn_fs_x__string_table_get_func(paths, binary_change->path,
                                           &change->path.len, pool);
 
-      change->noderev_id = binary_change->noderev_id;
-
       change->change_kind = (svn_fs_path_change_kind_t)
         ((binary_change->flags & CHANGE_KIND_MASK) >> CHANGE_KIND_SHIFT);
       change->text_mod = (binary_change->flags & CHANGE_TEXT_MOD) != 0;

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/changes.h
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/changes.h?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/changes.h (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/changes.h Fri Apr 29 18:38:53 2016
@@ -71,13 +71,15 @@ svn_fs_x__changes_estimate_size(const sv
 
 /* Read changes containers. */
 
-/* From CHANGES, extract the change list with the given IDX.  Allocate
- * the result in POOL and return it in *LIST.
+/* From CHANGES, access the change list with the given IDX and extract the
+ * next entries according to CONTEXT.  Allocate the result in RESULT_POOL
+ * and return it in *LIST.
  */
 svn_error_t *
 svn_fs_x__changes_get_list(apr_array_header_t **list,
                            const svn_fs_x__changes_t *changes,
                            apr_size_t idx,
+                           svn_fs_x__changes_context_t *context,
                            apr_pool_t *result_pool);
 
 /* I/O interface. */
@@ -116,11 +118,25 @@ svn_fs_x__deserialize_changes_container(
                                         apr_size_t data_len,
                                         apr_pool_t *result_pool);
 
+/* Baton type to be used with svn_fs_x__changes_get_list_func. */
+typedef struct svn_fs_x__changes_get_list_baton_t
+{
+  /* Sub-item to query */
+  apr_uint32_t sub_item;
+
+  /* Deliver data starting from this index within the changes list. */
+  int start;
+
+  /* To be set by svn_fs_x__changes_get_list_func:
+     Did we deliver the last change in that list? */
+  svn_boolean_t *eol;
+} svn_fs_x__changes_get_list_baton_t;
+
 /* Implements svn_cache__partial_getter_func_t for svn_fs_x__changes_t,
  * setting *OUT to the change list (apr_array_header_t *) selected by
- * the apr_uint32_t index passed in as *BATON.  This function is similar
- * to svn_fs_x__changes_get_list but operates on the cache serialized
- * representation of the container.
+ * the svn_fs_x__changes_get_list_baton_t passed in as *BATON.  This
+ * function is similar to svn_fs_x__changes_get_list but operates on
+ * the cache serialized representation of the container.
  */
 svn_error_t *
 svn_fs_x__changes_get_list_func(void **out,

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/fs.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/fs.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/fs.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/fs.c Fri Apr 29 18:38:53 2016
@@ -137,6 +137,18 @@ x_serialized_init(svn_fs_t *fs,
   return SVN_NO_ERROR;
 }
 
+svn_error_t *
+svn_fs_x__initialize_shared_data(svn_fs_t *fs,
+                                 svn_mutex__t *common_pool_lock,
+                                 apr_pool_t *scratch_pool,
+                                 apr_pool_t *common_pool)
+{
+  SVN_MUTEX__WITH_LOCK(common_pool_lock,
+                       x_serialized_init(fs, common_pool, scratch_pool));
+
+  return SVN_NO_ERROR;
+}
+
 
 
 /* This function is provided for Subversion 1.0.x compatibility.  It
@@ -218,20 +230,11 @@ x_info(const void **fsx_info,
   return SVN_NO_ERROR;
 }
 
-/* Wrapper around svn_fs_x__revision_prop() adapting between function
-   signatures. */
 static svn_error_t *
-x_revision_prop(svn_string_t **value_p,
-                svn_fs_t *fs,
-                svn_revnum_t rev,
-                const char *propname,
-                apr_pool_t *pool)
+x_refresh_revprops(svn_fs_t *fs,
+                   apr_pool_t *scratch_pool)
 {
-  apr_pool_t *scratch_pool = svn_pool_create(pool);
-  SVN_ERR(svn_fs_x__revision_prop(value_p, fs, rev, propname, pool,
-                                  scratch_pool));
-  svn_pool_destroy(scratch_pool);
-
+  svn_fs_x__invalidate_revprop_generation(fs);
   return SVN_NO_ERROR;
 }
 
@@ -241,14 +244,14 @@ static svn_error_t *
 x_revision_proplist(apr_hash_t **proplist_p,
                     svn_fs_t *fs,
                     svn_revnum_t rev,
-                    apr_pool_t *pool)
+                    svn_boolean_t refresh,
+                    apr_pool_t *result_pool,
+                    apr_pool_t *scratch_pool)
 {
-  apr_pool_t *scratch_pool = svn_pool_create(pool);
-
   /* No need to bypass the caches for r/o access to revprops. */
   SVN_ERR(svn_fs_x__get_revision_proplist(proplist_p, fs, rev, FALSE,
-                                          pool, scratch_pool));
-  svn_pool_destroy(scratch_pool);
+                                          refresh, result_pool,
+                                          scratch_pool));
 
   return SVN_NO_ERROR;
 }
@@ -262,7 +265,8 @@ x_set_uuid(svn_fs_t *fs,
 {
   /* Whenever we set a new UUID, imply that FS will also be a different
    * instance (on formats that support this). */
-  return svn_error_trace(svn_fs_x__set_uuid(fs, uuid, NULL, scratch_pool));
+  return svn_error_trace(svn_fs_x__set_uuid(fs, uuid, NULL, TRUE,
+                                            scratch_pool));
 }
 
 /* Wrapper around svn_fs_x__begin_txn() providing the scratch pool. */
@@ -285,7 +289,8 @@ x_begin_txn(svn_fs_txn_t **txn_p,
 /* The vtable associated with a specific open filesystem. */
 static fs_vtable_t fs_vtable = {
   svn_fs_x__youngest_rev,
-  x_revision_prop,
+  x_refresh_revprops,
+  svn_fs_x__revision_prop,
   x_revision_proplist,
   svn_fs_x__change_rev_prop,
   x_set_uuid,
@@ -316,6 +321,8 @@ static svn_error_t *
 initialize_fs_struct(svn_fs_t *fs)
 {
   svn_fs_x__data_t *ffd = apr_pcalloc(fs->pool, sizeof(*ffd));
+  ffd->revprop_generation = -1;
+
   fs->vtable = &fs_vtable;
   fs->fsap_data = ffd;
   return SVN_NO_ERROR;
@@ -536,24 +543,17 @@ x_hotcopy(svn_fs_t *src_fs,
   if (cancel_func)
     SVN_ERR(cancel_func(cancel_baton));
 
-  /* Test target repo when in INCREMENTAL mode, initialize it when not.
-   * For this, we need our FS internal data structures to be temporarily
-   * available. */
+  SVN_ERR(svn_fs__check_fs(dst_fs, FALSE));
   SVN_ERR(initialize_fs_struct(dst_fs));
-  SVN_ERR(svn_fs_x__hotcopy_prepare_target(src_fs, dst_fs, dst_path,
-                                           incremental, scratch_pool));
-  uninitialize_fs_struct(dst_fs);
 
-  /* Now, the destination repo should open just fine. */
-  SVN_ERR(x_open(dst_fs, dst_path, common_pool_lock, scratch_pool,
-                 common_pool));
-  if (cancel_func)
-    SVN_ERR(cancel_func(cancel_baton));
-
-  /* Now, we may copy data as needed ... */
-  return svn_fs_x__hotcopy(src_fs, dst_fs, incremental,
-                           notify_func, notify_baton,
-                           cancel_func, cancel_baton, scratch_pool);
+  /* In INCREMENTAL mode, svn_fs_x__hotcopy() will open DST_FS.
+     Otherwise, it's not an FS yet --- possibly just an empty dir --- so
+     can't be opened.
+   */
+  return svn_fs_x__hotcopy(src_fs, dst_fs, src_path, dst_path,
+                            incremental, notify_func, notify_baton,
+                            cancel_func, cancel_baton, common_pool_lock,
+                            scratch_pool, common_pool);
 }
 
 

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/fs.h
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/fs.h?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/fs.h (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/fs.h Fri Apr 29 18:38:53 2016
@@ -37,7 +37,7 @@
 #include "private/svn_sqlite.h"
 #include "private/svn_mutex.h"
 
-#include "id.h"
+#include "rev_file.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -60,7 +60,6 @@ extern "C" {
 #define PATH_TXNS_DIR         "transactions"     /* Directory of transactions */
 #define PATH_TXN_PROTOS_DIR   "txn-protorevs"    /* Directory of proto-revs */
 #define PATH_TXN_CURRENT      "txn-current"      /* File with next txn key */
-#define PATH_TXN_NEXT         "txn-next"         /* Will become txn-current */
 #define PATH_TXN_CURRENT_LOCK "txn-current-lock" /* Lock for txn-current */
 #define PATH_LOCKS_DIR        "locks"            /* Directory of locks */
 #define PATH_MIN_UNPACKED_REV "min-unpacked-rev" /* Oldest revision which
@@ -295,9 +294,8 @@ typedef struct svn_fs_x__data_t
      rep key (revision/offset) to svn_stringbuf_t. */
   svn_cache__t *fulltext_cache;
 
-  /* Access object to the revprop "generation". Will be NULL until
-     the first access.  May be also get closed and set to NULL again. */
-  apr_file_t *revprop_generation_file;
+  /* Revprop generation number.  Will be -1 if it has to reread from disk. */
+  apr_int64_t revprop_generation;
 
   /* Revision property cache.  Maps from (rev,generation) to apr_hash_t. */
   svn_cache__t *revprop_cache;
@@ -404,19 +402,12 @@ typedef struct svn_fs_x__data_t
   svn_error_t *(*svn_fs_open_)(svn_fs_t **, const char *, apr_hash_t *,
                                apr_pool_t *, apr_pool_t *);
 
-  /* If not 0, this is a pre-allocated transaction ID that can just be
-     used for a new txn without needing to consult 'txn-current'. */
-  apr_uint64_t next_txn_id;
 } svn_fs_x__data_t;
 
 
 /*** Filesystem Transaction ***/
 typedef struct svn_fs_x__transaction_t
 {
-  /* property list (const char * name, svn_string_t * value).
-     may be NULL if there are no properties.  */
-  apr_hash_t *proplist;
-
   /* revision upon which this txn is base.  (unfinished only) */
   svn_revnum_t base_rev;
 
@@ -530,28 +521,38 @@ typedef struct svn_fs_x__dirent_t
 
 
 /*** Change ***/
-typedef struct svn_fs_x__change_t
+typedef svn_fs_path_change3_t svn_fs_x__change_t;
+
+/*** Context for reading changed paths lists iteratively. */
+typedef struct svn_fs_x__changes_context_t
 {
-  /* Path of the change. */
-  svn_string_t path;
+  /* Repository to fetch from. */
+  svn_fs_t *fs;
 
-  /* node revision id of changed path */
-  svn_fs_x__id_t noderev_id;
+  /* Revision that we read from. */
+  svn_revnum_t revision;
 
-  /* See svn_fs_path_change2_t for a description for the remaining elements.
-   */
-  svn_fs_path_change_kind_t change_kind;
-
-  svn_boolean_t text_mod;
-  svn_boolean_t prop_mod;
-  svn_node_kind_t node_kind;
+  /* Revision file object to use when needed. */
+  svn_fs_x__revision_file_t *revision_file;
 
-  svn_boolean_t copyfrom_known;
-  svn_revnum_t copyfrom_rev;
-  const char *copyfrom_path;
+  /* Index of the next change to fetch. */
+  apr_size_t next;
+
+  /* Has the end of the list been reached? */
+  svn_boolean_t eol;
+
+} svn_fs_x__changes_context_t;
+
+/*** Directory (only used at the cache interface) ***/
+typedef struct svn_fs_x__dir_data_t
+{
+  /* Contents, i.e. all directory entries, sorted by name. */
+  apr_array_header_t *entries;
 
-  svn_tristate_t mergeinfo_mod;
-} svn_fs_x__change_t;
+  /* SVN_INVALID_FILESIZE for committed data, otherwise the length of the
+   * in-txn on-disk representation of that directory. */
+  svn_filesize_t txn_filesize;
+} svn_fs_x__dir_data_t;
 
 
 #ifdef __cplusplus

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.c Fri Apr 29 18:38:53 2016
@@ -33,6 +33,7 @@
 
 #include "cached_data.h"
 #include "id.h"
+#include "low_level.h"
 #include "rep-cache.h"
 #include "revprops.h"
 #include "transaction.h"
@@ -602,8 +603,9 @@ svn_fs_x__open(svn_fs_t *fs,
   /* Read the configuration file. */
   SVN_ERR(read_config(ffd, fs->path, fs->pool, scratch_pool));
 
-  return svn_error_trace(svn_fs_x__read_current(&ffd->youngest_rev_cache,
-                                                fs, scratch_pool));
+  ffd->youngest_rev_cache = 0;
+
+  return SVN_NO_ERROR;
 }
 
 /* Baton type bridging svn_fs_x__upgrade and upgrade_body carrying
@@ -855,10 +857,7 @@ static svn_error_t *
 write_revision_zero(svn_fs_t *fs,
                     apr_pool_t *scratch_pool)
 {
-  /* Use an explicit sub-pool to have full control over temp file lifetimes.
-   * Since we have it, use it for everything else as well. */
-  apr_pool_t *subpool = svn_pool_create(scratch_pool);
-  const char *path_revision_zero = svn_fs_x__path_rev(fs, 0, subpool);
+  const char *path_revision_zero = svn_fs_x__path_rev(fs, 0, scratch_pool);
   apr_hash_t *proplist;
   svn_string_t date;
 
@@ -876,63 +875,72 @@ write_revision_zero(svn_fs_t *fs,
                                                 "count: 0\n"
                                                 "cpath: /\n"
                                                 "\n",
-                                                subpool);
+                                                scratch_pool);
   svn_string_t *changes_str = svn_string_create("\n",
-                                                subpool);
-  svn_string_t *r0 = svn_string_createf(subpool, "%s%s",
+                                                scratch_pool);
+  svn_string_t *r0 = svn_string_createf(scratch_pool, "%s%s",
                                         noderev_str->data,
                                         changes_str->data);
 
   /* Write skeleton r0 to disk. */
-  SVN_ERR(svn_io_file_create(path_revision_zero, r0->data, subpool));
+  SVN_ERR(svn_io_file_create(path_revision_zero, r0->data, scratch_pool));
 
   /* Construct the index P2L contents: describe the 2 items we have.
      Be sure to create them in on-disk order. */
-  index_entries = apr_array_make(subpool, 2, sizeof(entry));
+  index_entries = apr_array_make(scratch_pool, 2, sizeof(entry));
 
-  entry = apr_pcalloc(subpool, sizeof(*entry));
+  entry = apr_pcalloc(scratch_pool, sizeof(*entry));
   entry->offset = 0;
   entry->size = (apr_off_t)noderev_str->len;
   entry->type = SVN_FS_X__ITEM_TYPE_NODEREV;
   entry->item_count = 1;
-  entry->items = apr_pcalloc(subpool, sizeof(*entry->items));
+  entry->items = apr_pcalloc(scratch_pool, sizeof(*entry->items));
   entry->items[0].change_set = 0;
   entry->items[0].number = SVN_FS_X__ITEM_INDEX_ROOT_NODE;
   APR_ARRAY_PUSH(index_entries, svn_fs_x__p2l_entry_t *) = entry;
 
-  entry = apr_pcalloc(subpool, sizeof(*entry));
+  entry = apr_pcalloc(scratch_pool, sizeof(*entry));
   entry->offset = (apr_off_t)noderev_str->len;
   entry->size = (apr_off_t)changes_str->len;
   entry->type = SVN_FS_X__ITEM_TYPE_CHANGES;
   entry->item_count = 1;
-  entry->items = apr_pcalloc(subpool, sizeof(*entry->items));
+  entry->items = apr_pcalloc(scratch_pool, sizeof(*entry->items));
   entry->items[0].change_set = 0;
   entry->items[0].number = SVN_FS_X__ITEM_INDEX_CHANGES;
   APR_ARRAY_PUSH(index_entries, svn_fs_x__p2l_entry_t *) = entry;
 
   /* Now re-open r0, create proto-index files from our entries and
-      rewrite the index section of r0. */
+     rewrite the index section of r0. */
   SVN_ERR(svn_fs_x__rev_file_open_writable(&rev_file, fs, 0,
-                                           subpool, subpool));
+                                           scratch_pool, scratch_pool));
   SVN_ERR(svn_fs_x__p2l_index_from_p2l_entries(&p2l_proto_index, fs,
                                                rev_file, index_entries,
-                                               subpool, subpool));
+                                               scratch_pool, scratch_pool));
   SVN_ERR(svn_fs_x__l2p_index_from_p2l_entries(&l2p_proto_index, fs,
                                                index_entries,
-                                               subpool, subpool));
+                                               scratch_pool, scratch_pool));
   SVN_ERR(svn_fs_x__rev_file_get(&apr_file, rev_file));
   SVN_ERR(svn_fs_x__add_index_data(fs, apr_file, l2p_proto_index,
-                                   p2l_proto_index, 0, subpool));
+                                   p2l_proto_index, 0, scratch_pool));
   SVN_ERR(svn_fs_x__close_revision_file(rev_file));
 
-  SVN_ERR(svn_io_set_file_read_only(path_revision_zero, FALSE, fs->pool));
+  SVN_ERR(svn_io_set_file_read_only(path_revision_zero, FALSE, scratch_pool));
 
   /* Set a date on revision 0. */
-  date.data = svn_time_to_cstring(apr_time_now(), fs->pool);
+  date.data = svn_time_to_cstring(apr_time_now(), scratch_pool);
   date.len = strlen(date.data);
-  proplist = apr_hash_make(fs->pool);
+  proplist = apr_hash_make(scratch_pool);
   svn_hash_sets(proplist, SVN_PROP_REVISION_DATE, &date);
-  return svn_fs_x__set_revision_proplist(fs, 0, proplist, fs->pool);
+
+  SVN_ERR(svn_io_file_open(&apr_file,
+                           svn_fs_x__path_revprops(fs, 0, scratch_pool),
+                           APR_WRITE | APR_CREATE, APR_OS_DEFAULT, 
+                           scratch_pool));
+  SVN_ERR(svn_fs_x__write_non_packed_revprops(apr_file, proplist,
+                                              scratch_pool));
+  SVN_ERR(svn_io_file_close(apr_file, scratch_pool));
+
+  return SVN_NO_ERROR;
 }
 
 svn_error_t *
@@ -966,14 +974,13 @@ svn_fs_x__create_file_tree(svn_fs_t *fs,
                             scratch_pool));
 
   /* Create the 'current' file. */
-  SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_current(fs, scratch_pool),
-                                   scratch_pool));
-  SVN_ERR(svn_fs_x__write_current(fs, 0, scratch_pool));
+  SVN_ERR(svn_io_file_create(svn_fs_x__path_current(fs, scratch_pool),
+                             "0\n", scratch_pool));
 
   /* Create the 'uuid' file. */
   SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_lock(fs, scratch_pool),
                                    scratch_pool));
-  SVN_ERR(svn_fs_x__set_uuid(fs, NULL, NULL, scratch_pool));
+  SVN_ERR(svn_fs_x__set_uuid(fs, NULL, NULL, FALSE, scratch_pool));
 
   /* Create the fsfs.conf file. */
   SVN_ERR(write_config(fs, scratch_pool));
@@ -996,6 +1003,9 @@ svn_fs_x__create_file_tree(svn_fs_t *fs,
                           scratch_pool));
 
   /* Initialize the revprop caching info. */
+  SVN_ERR(svn_io_file_create_empty(
+                        svn_fs_x__path_revprop_generation(fs, scratch_pool),
+                        scratch_pool));
   SVN_ERR(svn_fs_x__reset_revprop_generation_file(fs, scratch_pool));
 
   ffd->youngest_rev_cache = 0;
@@ -1052,6 +1062,7 @@ svn_error_t *
 svn_fs_x__set_uuid(svn_fs_t *fs,
                    const char *uuid,
                    const char *instance_id,
+                   svn_boolean_t overwrite,
                    apr_pool_t *scratch_pool)
 {
   svn_fs_x__data_t *ffd = fs->fsap_data;
@@ -1070,11 +1081,23 @@ svn_fs_x__set_uuid(svn_fs_t *fs,
   svn_stringbuf_appendcstr(contents, "\n");
 
   /* We use the permissions of the 'current' file, because the 'uuid'
-     file does not exist during repository creation. */
-  SVN_ERR(svn_io_write_atomic2(uuid_path, contents->data, contents->len,
-                               /* perms */
-                               svn_fs_x__path_current(fs, scratch_pool),
-                               TRUE, scratch_pool));
+     file does not exist during repository creation.
+
+     svn_io_write_atomic2() does a load of magic to allow it to
+     replace version files that already exist.  We only need to do
+     that when we're allowed to overwrite an existing file. */
+  if (! overwrite)
+    {
+      /* Create the file */
+      SVN_ERR(svn_io_file_create(uuid_path, contents->data, scratch_pool));
+    }
+  else
+    {
+      SVN_ERR(svn_io_write_atomic2(uuid_path, contents->data, contents->len,
+                                   /* perms */
+                                   svn_fs_x__path_current(fs, scratch_pool),
+                                   TRUE, scratch_pool));
+    }
 
   fs->uuid = apr_pstrdup(fs->pool, uuid);
   ffd->instance_id = apr_pstrdup(fs->pool, instance_id);
@@ -1112,13 +1135,14 @@ svn_fs_x__revision_prop(svn_string_t **v
                         svn_fs_t *fs,
                         svn_revnum_t rev,
                         const char *propname,
+                        svn_boolean_t refresh,
                         apr_pool_t *result_pool,
                         apr_pool_t *scratch_pool)
 {
   apr_hash_t *table;
 
   SVN_ERR(svn_fs__check_fs(fs, TRUE));
-  SVN_ERR(svn_fs_x__get_revision_proplist(&table, fs, rev, FALSE,
+  SVN_ERR(svn_fs_x__get_revision_proplist(&table, fs, rev, FALSE, refresh,
                                           scratch_pool, scratch_pool));
 
   *value_p = svn_string_dup(svn_hash_gets(table, propname), result_pool);
@@ -1151,7 +1175,7 @@ change_rev_prop_body(void *baton,
      Even if somehow the cache got out of sync, we want to make sure that
      we read, update and write up-to-date data. */
   SVN_ERR(svn_fs_x__get_revision_proplist(&table, cb->fs, cb->rev, TRUE,
-                                          scratch_pool, scratch_pool));
+                                          TRUE, scratch_pool, scratch_pool));
   present_value = svn_hash_gets(table, cb->name);
 
   if (cb->old_value_p)

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.h
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.h?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.h (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.h Fri Apr 29 18:38:53 2016
@@ -41,6 +41,16 @@ svn_fs_x__open(svn_fs_t *fs,
                const char *path,
                apr_pool_t *scratch_pool);
 
+/* Initialize parts of the FS data that are being shared across multiple
+   filesystem objects.  Use COMMON_POOL for process-wide and SCRATCH_POOL
+   for temporary allocations.  Use COMMON_POOL_LOCK to ensure that the
+   initialization is serialized. */
+svn_error_t *
+svn_fs_x__initialize_shared_data(svn_fs_t *fs,
+                                 svn_mutex__t *common_pool_lock,
+                                 apr_pool_t *scratch_pool,
+                                 apr_pool_t *common_pool);
+
 /* Upgrade the fsx filesystem FS.  Indicate progress via the optional
  * NOTIFY_FUNC callback using NOTIFY_BATON.  The optional CANCEL_FUNC
  * will periodically be called with CANCEL_BATON to allow for preemption.
@@ -138,11 +148,16 @@ svn_fs_x__create(svn_fs_t *fs,
 
 /* Set the uuid of repository FS to UUID and the instance ID to INSTANCE_ID.
    If any of them is NULL, use a newly generated UUID / ID instead.
+
+   If OVERWRITE is not set, the uuid file must not exist yet implying this
+   is a fresh repository.
+
    Perform temporary allocations in SCRATCH_POOL. */
 svn_error_t *
 svn_fs_x__set_uuid(svn_fs_t *fs,
                    const char *uuid,
                    const char *instance_id,
+                   svn_boolean_t overwrite,
                    apr_pool_t *scratch_pool);
 
 /* Read the format number and maximum number of files per directory
@@ -160,12 +175,15 @@ svn_fs_x__write_format(svn_fs_t *fs,
 
 /* Find the value of the property named PROPNAME in transaction REV.
    Return the contents in *VALUE_P, allocated from RESULT_POOL.
+   If REFRESH is not set, continue using the potentially outdated
+   revprop generation value in FS->FSAP_DATA.
    Use SCRATCH_POOL for temporary allocations. */
 svn_error_t *
 svn_fs_x__revision_prop(svn_string_t **value_p,
                         svn_fs_t *fs,
                         svn_revnum_t rev,
                         const char *propname,
+                        svn_boolean_t refresh,
                         apr_pool_t *result_pool,
                         apr_pool_t *scratch_pool);
 

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/hotcopy.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/hotcopy.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/hotcopy.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/hotcopy.c Fri Apr 29 18:38:53 2016
@@ -326,6 +326,19 @@ hotcopy_copy_packed_shard(svn_boolean_t
   return SVN_NO_ERROR;
 }
 
+/* Remove file PATH, if it exists - even if it is read-only.
+ * Use SCRATCH_POOL for temporary allocations. */
+static svn_error_t *
+hotcopy_remove_file(const char *path,
+                    apr_pool_t *scratch_pool)
+{
+  /* Make the rev file writable and remove it. */
+  SVN_ERR(svn_io_set_file_read_write(path, TRUE, scratch_pool));
+  SVN_ERR(svn_io_remove_file2(path, TRUE, scratch_pool));
+
+  return SVN_NO_ERROR;
+}
+
 /* Verify that DST_FS is a suitable destination for an incremental
  * hotcopy from SRC_FS. */
 static svn_error_t *
@@ -651,6 +664,10 @@ hotcopy_body(void *baton,
       /* Copy the rep cache and then remove entries for revisions
        * that did not make it into the destination. */
       SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, scratch_pool));
+
+      /* The source might have r/o flags set on it - which would be
+         carried over to the copy. */
+      SVN_ERR(svn_io_set_file_read_write(dst_subdir, FALSE, scratch_pool));
       SVN_ERR(svn_fs_x__del_rep_reference(dst_fs, src_youngest,
                                           scratch_pool));
     }
@@ -665,64 +682,33 @@ hotcopy_body(void *baton,
    * used for the named atomics implementation. */
   SVN_ERR(svn_fs_x__reset_revprop_generation_file(dst_fs, scratch_pool));
 
-  return SVN_NO_ERROR;
-}
-
-/* Wrapper around hotcopy_body taking out all necessary source repository
- * locks.
- */
-static svn_error_t *
-hotcopy_locking_src_body(void *baton,
-                         apr_pool_t *scratch_pool)
-{
-  hotcopy_body_baton_t *hbb = baton;
+  /* Hotcopied FS is complete. Stamp it with a format file. */
+  SVN_ERR(svn_fs_x__write_format(dst_fs, TRUE, scratch_pool));
 
-  return svn_error_trace(svn_fs_x__with_pack_lock(hbb->src_fs, hotcopy_body,
-                                                  baton, scratch_pool));
+  return SVN_NO_ERROR;
 }
 
-/* Create an empty filesystem at DST_FS at DST_PATH with the same
- * configuration as SRC_FS (uuid, format, and other parameters).
- * After creation DST_FS has no revisions, not even revision zero. */
-static svn_error_t *
-hotcopy_create_empty_dest(svn_fs_t *src_fs,
-                          svn_fs_t *dst_fs,
-                          const char *dst_path,
-                          apr_pool_t *scratch_pool)
+svn_error_t *
+svn_fs_x__hotcopy(svn_fs_t *src_fs,
+                  svn_fs_t *dst_fs,
+                  const char *src_path,
+                  const char *dst_path,
+                  svn_boolean_t incremental,
+                  svn_fs_hotcopy_notify_t notify_func,
+                  void *notify_baton,
+                  svn_cancel_func_t cancel_func,
+                  void *cancel_baton,
+                  svn_mutex__t *common_pool_lock,
+                  apr_pool_t *scratch_pool,
+                  apr_pool_t *common_pool)
 {
-  svn_fs_x__data_t *src_ffd = src_fs->fsap_data;
-
-  /* Create the DST_FS repository with the same layout as SRC_FS. */
-  SVN_ERR(svn_fs_x__create_file_tree(dst_fs, dst_path, src_ffd->format,
-                                     src_ffd->max_files_per_dir,
-                                     scratch_pool));
+  hotcopy_body_baton_t hbb;
 
-  /* Copy the UUID.  Hotcopy destination receives a new instance ID, but
-   * has the same filesystem UUID as the source. */
-  SVN_ERR(svn_fs_x__set_uuid(dst_fs, src_fs->uuid, NULL, scratch_pool));
-
-  /* Remove revision 0 contents.  Otherwise, it may not get overwritten
-   * due to having a newer timestamp. */
-  SVN_ERR(svn_io_remove_file2(svn_fs_x__path_rev(dst_fs, 0, scratch_pool),
-                              FALSE, scratch_pool));
-  SVN_ERR(svn_io_remove_file2(svn_fs_x__path_revprops(dst_fs, 0,
-                                                      scratch_pool),
-                              FALSE, scratch_pool));
-
-  /* This filesystem is ready.  Stamp it with a format number.  Fail if
-   * the 'format' file should already exist. */
-  SVN_ERR(svn_fs_x__write_format(dst_fs, FALSE, scratch_pool));
+  if (cancel_func)
+    SVN_ERR(cancel_func(cancel_baton));
 
-  return SVN_NO_ERROR;
-}
+  SVN_ERR(svn_fs_x__open(src_fs, src_path, scratch_pool));
 
-svn_error_t *
-svn_fs_x__hotcopy_prepare_target(svn_fs_t *src_fs,
-                                 svn_fs_t *dst_fs,
-                                 const char *dst_path,
-                                 svn_boolean_t incremental,
-                                 apr_pool_t *scratch_pool)
-{
   if (incremental)
     {
       const char *dst_format_abspath;
@@ -736,40 +722,53 @@ svn_fs_x__hotcopy_prepare_target(svn_fs_
                                 scratch_pool));
       if (dst_format_kind == svn_node_none)
         {
-          /* Destination doesn't exist yet. Perform a normal hotcopy to a
-           * empty destination using the same configuration as the source. */
-          SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path,
-                                            scratch_pool));
-        }
-      else
-        {
-          /* Check the existing repository. */
-          SVN_ERR(svn_fs_x__open(dst_fs, dst_path, scratch_pool));
-          SVN_ERR(hotcopy_incremental_check_preconditions(src_fs, dst_fs));
+          /* No destination?  Fallback to a non-incremental hotcopy. */
+          incremental = FALSE;
         }
     }
+
+  if (incremental)
+    {
+      /* Check the existing repository. */
+      SVN_ERR(svn_fs_x__open(dst_fs, dst_path, scratch_pool));
+      SVN_ERR(hotcopy_incremental_check_preconditions(src_fs, dst_fs));
+
+      SVN_ERR(svn_fs_x__initialize_shared_data(dst_fs, common_pool_lock,
+                                               scratch_pool, common_pool));
+      SVN_ERR(svn_fs_x__initialize_caches(dst_fs, scratch_pool));
+    }
   else
     {
       /* Start out with an empty destination using the same configuration
        * as the source. */
-      SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path,
-                                        scratch_pool));
-    }
+      svn_fs_x__data_t *src_ffd = src_fs->fsap_data;
 
-  return SVN_NO_ERROR;
-}
+      /* Create the DST_FS repository with the same layout as SRC_FS. */
+      SVN_ERR(svn_fs_x__create_file_tree(dst_fs, dst_path, src_ffd->format,
+                                         src_ffd->max_files_per_dir,
+                                         scratch_pool));
+
+      /* Copy the UUID.  Hotcopy destination receives a new instance ID, but
+       * has the same filesystem UUID as the source. */
+      SVN_ERR(svn_fs_x__set_uuid(dst_fs, src_fs->uuid, NULL, TRUE,
+                                 scratch_pool));
+
+      /* Remove revision 0 contents.  Otherwise, it may not get overwritten
+       * due to having a newer timestamp. */
+      SVN_ERR(hotcopy_remove_file(svn_fs_x__path_rev(dst_fs, 0,
+                                                     scratch_pool),
+                                  scratch_pool));
+      SVN_ERR(hotcopy_remove_file(svn_fs_x__path_revprops(dst_fs, 0,
+                                                          scratch_pool),
+                                  scratch_pool));
+
+      SVN_ERR(svn_fs_x__initialize_shared_data(dst_fs, common_pool_lock,
+                                               scratch_pool, common_pool));
+      SVN_ERR(svn_fs_x__initialize_caches(dst_fs, scratch_pool));
+    }
 
-svn_error_t *
-svn_fs_x__hotcopy(svn_fs_t *src_fs,
-                  svn_fs_t *dst_fs,
-                  svn_boolean_t incremental,
-                  svn_fs_hotcopy_notify_t notify_func,
-                  void *notify_baton,
-                  svn_cancel_func_t cancel_func,
-                  void *cancel_baton,
-                  apr_pool_t *scratch_pool)
-{
-  hotcopy_body_baton_t hbb;
+  if (cancel_func)
+    SVN_ERR(cancel_func(cancel_baton));
 
   hbb.src_fs = src_fs;
   hbb.dst_fs = dst_fs;
@@ -778,8 +777,16 @@ svn_fs_x__hotcopy(svn_fs_t *src_fs,
   hbb.notify_baton = notify_baton;
   hbb.cancel_func = cancel_func;
   hbb.cancel_baton = cancel_baton;
-  SVN_ERR(svn_fs_x__with_all_locks(dst_fs, hotcopy_locking_src_body, &hbb,
-                                   scratch_pool));
+
+  /* Lock the destination in the incremental mode.  For a non-incremental
+   * hotcopy, don't take any locks.  In that case the destination cannot be
+   * opened until the hotcopy finishes, and we don't have to worry about
+   * concurrency. */
+  if (incremental)
+    SVN_ERR(svn_fs_x__with_all_locks(dst_fs, hotcopy_body, &hbb,
+                                     scratch_pool));
+  else
+    SVN_ERR(hotcopy_body(&hbb, scratch_pool));
 
   return SVN_NO_ERROR;
 }

Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/hotcopy.h
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/hotcopy.h?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/hotcopy.h (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/hotcopy.h Fri Apr 29 18:38:53 2016
@@ -25,29 +25,24 @@
 
 #include "fs.h"
 
-/* Create an empty copy of the fsfs filesystem SRC_FS into a new DST_FS at
- * DST_PATH.  If INCREMENTAL is TRUE, perform a few pre-checks only if
- * a repo already exists at DST_PATH.
- * Use SCRATCH_POOL for temporary allocations. */
-svn_error_t *
-svn_fs_x__hotcopy_prepare_target(svn_fs_t *src_fs,
-                                 svn_fs_t *dst_fs,
-                                 const char *dst_path,
-                                 svn_boolean_t incremental,
-                                 apr_pool_t *scratch_pool);
-
-/* Copy the fsfs filesystem SRC_FS into DST_FS. If INCREMENTAL is TRUE, do
- * not re-copy data which already exists in DST_FS.  Indicate progress via
- * the optional NOTIFY_FUNC callback using NOTIFY_BATON.
- * Use SCRATCH_POOL for temporary allocations. */
+/* Copy the fsfs filesystem SRC_FS at SRC_PATH into a new copy DST_FS at
+ * DST_PATH.  If INCREMENTAL is TRUE, do not re-copy data which already
+ * exists in DST_FS.  Indicate progress via the optional NOTIFY_FUNC
+ * callback using NOTIFY_BATON.  Use COMMON_POOL for process-wide and
+ * SCRATCH_POOL for temporary allocations.  Use COMMON_POOL_LOCK to ensure
+ * that the initialization of the shared data is serialized. */
 svn_error_t *
 svn_fs_x__hotcopy(svn_fs_t *src_fs,
                   svn_fs_t *dst_fs,
+                  const char *src_path,
+                  const char *dst_path,
                   svn_boolean_t incremental,
                   svn_fs_hotcopy_notify_t notify_func,
                   void *notify_baton,
                   svn_cancel_func_t cancel_func,
                   void *cancel_baton,
-                  apr_pool_t *scratch_pool);
+                  svn_mutex__t *common_pool_lock,
+                  apr_pool_t *scratch_pool,
+                  apr_pool_t *common_pool);
 
 #endif




Mime
View raw message