subversion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From i...@apache.org
Subject svn commit: r1639628 [10/23] - in /subversion/branches/svn-auth-x509: ./ build/ build/generator/ notes/ notes/api-errata/1.8/ notes/api-errata/1.9/ notes/wc-ng/ subversion/ subversion/bindings/ctypes-python/ subversion/bindings/cxxhl/ subversion/bindin...
Date Fri, 14 Nov 2014 13:18:03 GMT
Modified: subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs.c
URL: http://svn.apache.org/viewvc/subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs.c?rev=1639628&r1=1639627&r2=1639628&view=diff
==============================================================================
--- subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs.c (original)
+++ subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs.c Fri Nov 14 13:17:55 2014
@@ -45,6 +45,7 @@
 #include "revprops.h"
 #include "rep-cache.h"
 #include "transaction.h"
+#include "util.h"
 #include "svn_private_config.h"
 #include "private/svn_fs_util.h"
 
@@ -72,20 +73,28 @@ x_serialized_init(svn_fs_t *fs, apr_pool
      each separate repository opened during the lifetime of the
      svn_fs_initialize pool.  It's unlikely that anyone will notice
      the modest expenditure; the alternative is to allocate each structure
-     in a subpool, add a reference-count, and add a serialized deconstructor
+     in a subpool, add a reference-count, and add a serialized destructor
      to the FS vtable.  That's more machinery than it's worth.
 
-     Using the uuid to obtain the lock creates a corner case if a
-     caller uses svn_fs_set_uuid on the repository in a process where
-     other threads might be using the same repository through another
-     FS object.  The only real-world consumer of svn_fs_set_uuid is
-     "svnadmin load", so this is a low-priority problem, and we don't
-     know of a better way of associating such data with the
-     repository. */
+     Picking an appropriate key for the shared data is tricky, because,
+     unfortunately, a filesystem UUID is not really unique.  It is implicitly
+     shared between hotcopied (1), dump / loaded (2) or naively copied (3)
+     filesystems.  We tackle this problem by using a combination of the UUID
+     and an instance ID as the key.  This allows us to avoid key clashing
+     in (1) and (2).
+
+     Speaking of (3), there is not so much we can do about it, except maybe
+     provide a convenient way of fixing things.  Naively copied filesystems
+     have identical filesystem UUIDs *and* instance IDs.  With the key being
+     a combination of these two, clashes can be fixed by changing either of
+     them (or both), e.g. with svn_fs_set_uuid(). */
+
 
   SVN_ERR_ASSERT(fs->uuid);
-  key = apr_pstrcat(pool, SVN_FSX_SHARED_USERDATA_PREFIX, fs->uuid,
-                    SVN_VA_NULL);
+  SVN_ERR_ASSERT(ffd->instance_id);
+
+  key = apr_pstrcat(pool, SVN_FSX_SHARED_USERDATA_PREFIX,
+                    fs->uuid, ":", ffd->instance_id, SVN_VA_NULL);
   status = apr_pool_userdata_get(&val, key, common_pool);
   if (status)
     return svn_error_wrap_apr(status, _("Can't fetch FSX shared data"));
@@ -100,21 +109,19 @@ x_serialized_init(svn_fs_t *fs, apr_pool
          intra-process synchronization when grabbing the repository write
          lock. */
       SVN_ERR(svn_mutex__init(&ffsd->fs_write_lock,
-                              TRUE, TRUE, common_pool));
+                              SVN_FS_X__USE_LOCK_MUTEX, common_pool));
 
       /* ... the pack lock ... */
       SVN_ERR(svn_mutex__init(&ffsd->fs_pack_lock,
-                              TRUE, TRUE, common_pool));
+                              SVN_FS_X__USE_LOCK_MUTEX, common_pool));
 
       /* ... not to mention locking the txn-current file. */
       SVN_ERR(svn_mutex__init(&ffsd->txn_current_lock,
-                              TRUE, TRUE, common_pool));
+                              SVN_FS_X__USE_LOCK_MUTEX, common_pool));
 
       /* We also need a mutex for synchronizing access to the active
-         transaction list and free transaction pointer.  This one is
-         enabled unconditionally. */
-      SVN_ERR(svn_mutex__init(&ffsd->txn_list_lock,
-                              TRUE, TRUE, common_pool));
+         transaction list and free transaction pointer. */
+      SVN_ERR(svn_mutex__init(&ffsd->txn_list_lock, TRUE, common_pool));
 
       key = apr_pstrdup(common_pool, key);
       status = apr_pool_userdata_set(ffsd, key, NULL, common_pool);
@@ -163,6 +170,16 @@ x_freeze_body(void *baton,
 }
 
 static svn_error_t *
+x_freeze_body2(void *baton,
+               apr_pool_t *pool)
+{
+  struct x_freeze_baton_t *b = baton;
+  SVN_ERR(svn_fs_x__with_write_lock(b->fs, x_freeze_body, baton, pool));
+
+  return SVN_NO_ERROR;
+}
+
+static svn_error_t *
 x_freeze(svn_fs_t *fs,
          svn_fs_freeze_func_t freeze_func,
          void *freeze_baton,
@@ -175,7 +192,7 @@ x_freeze(svn_fs_t *fs,
   b.freeze_baton = freeze_baton;
 
   SVN_ERR(svn_fs__check_fs(fs, TRUE));
-  SVN_ERR(svn_fs_x__with_write_lock(fs, x_freeze_body, &b, pool));
+  SVN_ERR(svn_fs_x__with_pack_lock(fs, x_freeze_body2, &b, pool));
 
   return SVN_NO_ERROR;
 }
@@ -195,15 +212,40 @@ x_info(const void **fsx_info,
   return SVN_NO_ERROR;
 }
 
+/* Wrapper around svn_fs_x__get_revision_proplist() adapting between function
+   signatures. */
+static svn_error_t *
+x_revision_proplist(apr_hash_t **proplist_p,
+                    svn_fs_t *fs,
+                    svn_revnum_t rev,
+                    apr_pool_t *pool)
+{
+  /* No need to bypass the caches for r/o access to revprops. */
+  return svn_error_trace(svn_fs_x__get_revision_proplist(proplist_p, fs,
+                                                         rev, FALSE, pool));
+}
+
+/* Wrapper around svn_fs_x__set_uuid() adapting between function
+   signatures. */
+static svn_error_t *
+x_set_uuid(svn_fs_t *fs,
+           const char *uuid,
+           apr_pool_t *pool)
+{
+  /* Whenever we set a new UUID, imply that FS will also be a different
+   * instance (on formats that support this). */
+  return svn_error_trace(svn_fs_x__set_uuid(fs, uuid, NULL, pool));
+}
+
 
 
 /* The vtable associated with a specific open filesystem. */
 static fs_vtable_t fs_vtable = {
   svn_fs_x__youngest_rev,
   svn_fs_x__revision_prop,
-  svn_fs_x__revision_proplist,
+  x_revision_proplist,
   svn_fs_x__change_rev_prop,
-  svn_fs_x__set_uuid,
+  x_set_uuid,
   svn_fs_x__revision_root,
   svn_fs_x__begin_txn,
   svn_fs_x__open_txn,
@@ -236,6 +278,15 @@ initialize_fs_struct(svn_fs_t *fs)
   return SVN_NO_ERROR;
 }
 
+/* Reset vtable and fsap_data fields in FS such that the FS is basically
+ * closed now.  Note that FS must not hold locks when you call this. */
+static void
+uninitialize_fs_struct(svn_fs_t *fs)
+{
+  fs->vtable = NULL;
+  fs->fsap_data = NULL;
+}
+
 /* This implements the fs_library_vtable_t.create() API.  Create a new
    fsx-backed Subversion filesystem at path PATH and link it into
    *FS.  Perform temporary allocations in POOL, and fs-global allocations
@@ -303,20 +354,53 @@ x_open_for_recovery(svn_fs_t *fs,
                     apr_pool_t *pool,
                     apr_pool_t *common_pool)
 {
-  /* Recovery for FSX is currently limited to recreating the 'current'
+  svn_error_t * err;
+  svn_revnum_t youngest_rev;
+  apr_pool_t * subpool = svn_pool_create(pool);
+
+  /* Recovery for FSFS is currently limited to recreating the 'current'
      file from the latest revision. */
 
   /* The only thing we have to watch out for is that the 'current' file
-     might not exist.  So we'll try to create it here unconditionally,
-     and just ignore any errors that might indicate that it's already
-     present. (We'll need it to exist later anyway as a source for the
-     new file's permissions). */
+     might not exist or contain garbage.  So we'll try to read it here
+     and provide or replace the existing file if we couldn't read it.
+     (We'll also need it to exist later anyway as a source for the new
+     file's permissions). */
 
-  /* Use a partly-filled fs pointer first to create 'current'.  This will fail
-     if 'current' already exists, but we don't care about that. */
+  /* Use a partly-filled fs pointer first to create 'current'. */
   fs->path = apr_pstrdup(fs->pool, path);
-  svn_error_clear(svn_io_file_create(svn_fs_x__path_current(fs, pool),
-                                     "0 1 1\n", pool));
+
+  SVN_ERR(initialize_fs_struct(fs));
+
+  /* Figure out the repo format and check that we can even handle it. */
+  SVN_ERR(svn_fs_x__read_format_file(fs, subpool));
+
+  /* Now, read 'current' and try to patch it if necessary. */
+  err = svn_fs_x__youngest_rev(&youngest_rev, fs, subpool);
+  if (err)
+    {
+      const char *file_path;
+
+      /* 'current' file is missing or contains garbage.  Since we are trying
+       * to recover from whatever problem there is, being picky about the
+       * error code here won't do us much good.  If there is a persistent
+       * problem that we can't fix, it will show up when we try rewrite the
+       * file a few lines further below and we will report the failure back
+       * to the caller.
+       *
+       * Start recovery with HEAD = 0. */
+      svn_error_clear(err);
+      file_path = svn_fs_x__path_current(fs, subpool);
+
+      /* Best effort to ensure the file exists and is valid.
+       * This may fail for r/o filesystems etc. */
+      SVN_ERR(svn_io_remove_file2(file_path, TRUE, subpool));
+      SVN_ERR(svn_io_file_create_empty(file_path, subpool));
+      SVN_ERR(svn_fs_x__write_current(fs, 0, subpool));
+    }
+
+  uninitialize_fs_struct(fs);
+  svn_pool_destroy(subpool);
 
   /* Now open the filesystem properly by calling the vtable method directly. */
   return x_open(fs, path, common_pool_lock, pool, common_pool);
@@ -404,11 +488,13 @@ x_hotcopy(svn_fs_t *src_fs,
   if (cancel_func)
     SVN_ERR(cancel_func(cancel_baton));
 
-  /* Provide FFD for DST_FS, test / initialize target repo, remove FFD. */
+  /* Test target repo when in INCREMENTAL mode, initialize it when not.
+   * For this, we need our FS internal data structures to be temporarily
+   * available. */
   SVN_ERR(initialize_fs_struct(dst_fs));
   SVN_ERR(svn_fs_x__hotcopy_prepare_target(src_fs, dst_fs, dst_path,
                                            incremental, pool));
-  dst_fs->fsap_data = NULL;
+  uninitialize_fs_struct(dst_fs);
 
   /* Now, the destination repo should open just fine. */
   SVN_ERR(x_open(dst_fs, dst_path, common_pool_lock, pool, common_pool));
@@ -416,8 +502,9 @@ x_hotcopy(svn_fs_t *src_fs,
     SVN_ERR(cancel_func(cancel_baton));
 
   /* Now, we may copy data as needed ... */
-  return svn_fs_x__hotcopy(src_fs, dst_fs,
-                           incremental, cancel_func, cancel_baton, pool);
+  return svn_fs_x__hotcopy(src_fs, dst_fs, incremental,
+                           notify_func, notify_baton,
+                           cancel_func, cancel_baton, pool);
 }
 
 
@@ -448,7 +535,7 @@ x_delete_fs(const char *path,
             apr_pool_t *pool)
 {
   /* Remove everything. */
-  return svn_io_remove_dir2(path, FALSE, NULL, NULL, pool);
+  return svn_error_trace(svn_io_remove_dir2(path, FALSE, NULL, NULL, pool));
 }
 
 static const svn_version_t *

Modified: subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs.h
URL: http://svn.apache.org/viewvc/subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs.h?rev=1639628&r1=1639627&r2=1639628&view=diff
==============================================================================
--- subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs.h (original)
+++ subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs.h Fri Nov 14 13:17:55 2014
@@ -36,7 +36,6 @@
 #include "private/svn_fs_private.h"
 #include "private/svn_sqlite.h"
 #include "private/svn_mutex.h"
-#include "private/svn_named_atomic.h"
 
 #include "id.h"
 
@@ -127,6 +126,16 @@ extern "C" {
  */
 #define SVN_FS_X__FORMAT_NUMBER   1
 
+/* On most operating systems apr implements file locks per process, not
+   per file.  On Windows apr implements the locking as per file handle
+   locks, so we don't have to add our own mutex for just in-process
+   synchronization. */
+#if APR_HAS_THREADS && !defined(WIN32)
+#define SVN_FS_X__USE_LOCK_MUTEX 1
+#else
+#define SVN_FS_X__USE_LOCK_MUTEX 0
+#endif
+
 /* Private FSX-specific data shared between all svn_txn_t objects that
    relate to a particular transaction in a filesystem (as identified
    by transaction id and filesystem UUID).  Objects of this type are
@@ -246,9 +255,11 @@ typedef struct fs_x_data_t
   /* The maximum number of files to store per directory. */
   int max_files_per_dir;
 
-  /* Rev / pack file read granularity. */
+  /* Rev / pack file read granularity in bytes. */
   apr_int64_t block_size;
 
+  /* Rev / pack file granularity (in bytes) covered by a single phys-to-log
+   * index page. */
   /* Capacity in entries of log-to-phys index pages */
   apr_int64_t l2p_page_size;
 
@@ -288,17 +299,9 @@ typedef struct fs_x_data_t
      rep key (revision/offset) to svn_stringbuf_t. */
   svn_cache__t *fulltext_cache;
 
-  /* Access object to the atomics namespace used by revprop caching.
-     Will be NULL until the first access. */
-  svn_atomic_namespace__t *revprop_namespace;
-
   /* Access object to the revprop "generation". Will be NULL until
-     the first access. */
-  svn_named_atomic__t *revprop_generation;
-
-  /* Access object to the revprop update timeout. Will be NULL until
-     the first access. */
-  svn_named_atomic__t *revprop_timeout;
+     the first access.  May be also get closed and set to NULL again. */
+  apr_file_t *revprop_generation_file;
 
   /* Revision property cache.  Maps from (rev,generation) to apr_hash_t. */
   svn_cache__t *revprop_cache;
@@ -409,6 +412,12 @@ typedef struct fs_x_data_t
   /* Pack after every commit. */
   svn_boolean_t pack_after_commit;
 
+  /* Per-instance filesystem ID, which provides an additional level of
+     uniqueness for filesystems that share the same UUID, but should
+     still be distinguishable (e.g. backups produced by svn_fs_hotcopy()
+     or dump / load cycles). */
+  const char *instance_id;
+
   /* Pointer to svn_fs_open. */
   svn_error_t *(*svn_fs_open_)(svn_fs_t **, const char *, apr_hash_t *,
                                apr_pool_t *, apr_pool_t *);

Modified: subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs_x.c
URL: http://svn.apache.org/viewvc/subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs_x.c?rev=1639628&r1=1639627&r2=1639628&view=diff
==============================================================================
--- subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs_x.c (original)
+++ subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs_x.c Fri Nov 14 13:17:55 2014
@@ -183,25 +183,50 @@ svn_fs_x__write_format(svn_fs_t *fs,
   return svn_io_set_file_read_only(path, FALSE, pool);
 }
 
-/* Find the youngest revision in a repository at path FS_PATH and
-   return it in *YOUNGEST_P.  Perform temporary allocations in
-   POOL. */
+/* Check that BLOCK_SIZE is a valid block / page size, i.e. it is within
+ * the range of what the current system may address in RAM and it is a
+ * power of 2.  Assume that the element size within the block is ITEM_SIZE.
+ * Use SCRATCH_POOL for temporary allocations.
+ */
 static svn_error_t *
-get_youngest(svn_revnum_t *youngest_p,
-             const char *fs_path,
-             apr_pool_t *pool)
-{
-  svn_stringbuf_t *buf;
-  SVN_ERR(svn_fs_x__read_content(&buf,
-                                 svn_dirent_join(fs_path, PATH_CURRENT, pool),
-                                 pool));
-
-  *youngest_p = SVN_STR_TO_REV(buf->data);
+verify_block_size(apr_int64_t block_size,
+                  apr_size_t item_size,
+                  const char *name,
+                  apr_pool_t *scratch_pool
+                 )
+{
+  /* Limit range. */
+  if (block_size <= 0)
+    return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE, NULL,
+                             _("%s is too small for fsfs.conf setting '%s'."),
+                             apr_psprintf(scratch_pool,
+                                          "%" APR_INT64_T_FMT,
+                                          block_size),
+                             name);
+
+  if (block_size > SVN_MAX_OBJECT_SIZE / item_size)
+    return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE, NULL,
+                             _("%s is too large for fsfs.conf setting '%s'."),
+                             apr_psprintf(scratch_pool,
+                                          "%" APR_INT64_T_FMT,
+                                          block_size),
+                             name);
+
+  /* Ensure it is a power of two.
+   * For positive X,  X & (X-1) will reset the lowest bit set.
+   * If the result is 0, at most one bit has been set. */
+  if (0 != (block_size & (block_size - 1)))
+    return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE, NULL,
+                             _("%s is invalid for fsfs.conf setting '%s' "
+                               "because it is not a power of 2."),
+                             apr_psprintf(scratch_pool,
+                                          "%" APR_INT64_T_FMT,
+                                          block_size),
+                             name);
 
   return SVN_NO_ERROR;
 }
 
-
 /* Read the configuration information of the file system at FS_PATH
  * and set the respective values in FFD.  Use pools as usual.
  */
@@ -268,8 +293,20 @@ read_config(fs_x_data_t *ffd,
                                CONFIG_OPTION_P2L_PAGE_SIZE,
                                0x400));
 
+  /* Don't accept unreasonable or illegal values.
+   * Block size and P2L page size are in kbytes;
+   * L2P blocks are arrays of apr_off_t. */
+  SVN_ERR(verify_block_size(ffd->block_size, 0x400,
+                            CONFIG_OPTION_BLOCK_SIZE, scratch_pool));
+  SVN_ERR(verify_block_size(ffd->p2l_page_size, 0x400,
+                            CONFIG_OPTION_P2L_PAGE_SIZE, scratch_pool));
+  SVN_ERR(verify_block_size(ffd->l2p_page_size, sizeof(apr_off_t),
+                            CONFIG_OPTION_L2P_PAGE_SIZE, scratch_pool));
+
+  /* convert kBytes to bytes */
   ffd->block_size *= 0x400;
   ffd->p2l_page_size *= 0x400;
+  /* L2P pages are in entries - not in (k)Bytes */
 
   /* Debug options. */
   SVN_ERR(svn_config_get_bool(config, &ffd->pack_after_commit,
@@ -435,23 +472,18 @@ write_config(svn_fs_t *fs,
 "### For SSD-based storage systems, slightly lower values around 16 kB"      NL
 "### may improve latency while still maximizing throughput."                 NL
 "### Can be changed at any time but must be a power of 2."                   NL
-"### block-size is 64 kBytes by default."                                    NL
+"### block-size is given in kBytes and with a default of 64 kBytes."         NL
 "# " CONFIG_OPTION_BLOCK_SIZE " = 64"                                        NL
 "###"                                                                        NL
 "### The log-to-phys index maps data item numbers to offsets within the"     NL
-"### rev or pack file.  A revision typically contains 2 .. 5 such items"     NL
-"### per changed path.  For each revision, at least one page is being"       NL
-"### allocated in the l2p index with unused parts resulting in no wasted"    NL
-"### space."                                                                 NL
-"### Changing this parameter only affects larger revisions with thousands"   NL
-"### of changed paths.  A smaller value means that more pages need to be"    NL
-"### allocated for such revisions, increasing the size of the page table"    NL
-"### meaning it takes longer to read that table (once).  Access to each"     NL
-"### page is then faster because less data has to read.  So, if you have"    NL
-"### several extremely large revisions (approaching 1 mio changes),  think"  NL
+"### rev or pack file.  This index is organized in pages of a fixed maximum" NL
+"### capacity.  To access an item, the page table and the respective page"   NL
+"### must be read."                                                          NL
+"### This parameter only affects revisions with thousands of changed paths." NL
+"### If you have several extremely large revisions (~1 mio changes), think"  NL
 "### about increasing this setting.  Reducing the value will rarely result"  NL
 "### in a net speedup."                                                      NL
-"### This is an expert setting.  Any non-zero value is possible."            NL
+"### This is an expert setting.  Must be a power of 2."                      NL
 "### l2p-page-size is 8192 entries by default."                              NL
 "# " CONFIG_OPTION_L2P_PAGE_SIZE " = 8192"                                   NL
 "###"                                                                        NL
@@ -460,7 +492,7 @@ write_config(svn_fs_t *fs,
 "### stored at any particular offset.  The index describes the rev file"     NL
 "### in chunks (pages) and keeps a global list of all those pages.  Large"   NL
 "### pages mean a shorter page table but a larger per-page description of"   NL
-"### data items in it.  The latency sweetspot depends on the change size"    NL
+"### data items in it.  The latency sweet spot depends on the change size"   NL
 "### distribution but covers a relatively wide range."                       NL
 "### If the repository contains very large files,  i.e. individual changes"  NL
 "### of tens of MB each,  increasing the page size will shorten the index"   NL
@@ -468,7 +500,7 @@ write_config(svn_fs_t *fs,
 "### smaller changes."                                                       NL
 "### For source code repositories, this should be about 16x the block-size." NL
 "### Must be a power of 2."                                                  NL
-"### p2l-page-size is 1024 kBytes by default."                               NL
+"### p2l-page-size is given in kBytes and with a default of 1024 kBytes."    NL
 "# " CONFIG_OPTION_P2L_PAGE_SIZE " = 1024"                                   NL
 ;
 #undef NL
@@ -476,34 +508,65 @@ write_config(svn_fs_t *fs,
                             fsx_conf_contents, pool);
 }
 
-svn_error_t *
-svn_fs_x__open(svn_fs_t *fs, const char *path, apr_pool_t *pool)
+/* Read FS's UUID file and store the data in the FS struct. */
+static svn_error_t *
+read_uuid(svn_fs_t *fs,
+          apr_pool_t *scratch_pool)
 {
   fs_x_data_t *ffd = fs->fsap_data;
   apr_file_t *uuid_file;
-  int format, max_files_per_dir;
   char buf[APR_UUID_FORMATTED_LENGTH + 2];
   apr_size_t limit;
 
-  fs->path = apr_pstrdup(fs->pool, path);
+  /* Read the repository uuid. */
+  SVN_ERR(svn_io_file_open(&uuid_file, svn_fs_x__path_uuid(fs, scratch_pool),
+                           APR_READ | APR_BUFFERED, APR_OS_DEFAULT,
+                           scratch_pool));
+
+  limit = sizeof(buf);
+  SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit, scratch_pool));
+  fs->uuid = apr_pstrdup(fs->pool, buf);
 
-  /* Read the FS format number. */
+  /* Read the instance ID. */
+  limit = sizeof(buf);
+  SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit,
+                                  scratch_pool));
+  ffd->instance_id = apr_pstrdup(fs->pool, buf);
+
+  SVN_ERR(svn_io_file_close(uuid_file, scratch_pool));
+
+  return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs_x__read_format_file(svn_fs_t *fs,
+                           apr_pool_t *scratch_pool)
+{
+  fs_x_data_t *ffd = fs->fsap_data;
+  int format, max_files_per_dir;
+
+  /* Read info from format file. */
   SVN_ERR(read_format(&format, &max_files_per_dir,
-                      svn_fs_x__path_format(fs, pool), pool));
+                      svn_fs_x__path_format(fs, scratch_pool), scratch_pool));
 
-  /* Now we've got a format number no matter what. */
+  /* Now that we've got *all* info, store / update values in FFD. */
   ffd->format = format;
   ffd->max_files_per_dir = max_files_per_dir;
 
-  /* Read in and cache the repository uuid. */
-  SVN_ERR(svn_io_file_open(&uuid_file, svn_fs_x__path_uuid(fs, pool),
-                           APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
+  return SVN_NO_ERROR;
+}
 
-  limit = sizeof(buf);
-  SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit, pool));
-  fs->uuid = apr_pstrdup(fs->pool, buf);
+svn_error_t *
+svn_fs_x__open(svn_fs_t *fs, const char *path, apr_pool_t *pool)
+{
+  fs_x_data_t *ffd = fs->fsap_data;
+  fs->path = apr_pstrdup(fs->pool, path);
 
-  SVN_ERR(svn_io_file_close(uuid_file, pool));
+  /* Read the FS format file. */
+  SVN_ERR(svn_fs_x__read_format_file(fs, pool));
+
+  /* Read in and cache the repository uuid. */
+  SVN_ERR(read_uuid(fs, pool));
 
   /* Read the min unpacked revision. */
   SVN_ERR(svn_fs_x__update_min_unpacked_rev(fs, pool));
@@ -511,7 +574,8 @@ svn_fs_x__open(svn_fs_t *fs, const char 
   /* Read the configuration file. */
   SVN_ERR(read_config(ffd, fs->path, fs->pool, pool));
 
-  return get_youngest(&(ffd->youngest_rev_cache), path, pool);
+  return svn_error_trace(svn_fs_x__read_current(&ffd->youngest_rev_cache,
+                                                fs, pool));
 }
 
 /* Baton type bridging svn_fs_x__upgrade and upgrade_body carrying 
@@ -566,12 +630,11 @@ svn_fs_x__upgrade(svn_fs_t *fs,
 
 svn_error_t *
 svn_fs_x__youngest_rev(svn_revnum_t *youngest_p,
-                        svn_fs_t *fs,
-                        apr_pool_t *pool)
+                       svn_fs_t *fs,
+                       apr_pool_t *pool)
 {
   fs_x_data_t *ffd = fs->fsap_data;
-
-  SVN_ERR(get_youngest(youngest_p, fs->path, pool));
+  SVN_ERR(svn_fs_x__read_current(youngest_p, fs, pool));
   ffd->youngest_rev_cache = *youngest_p;
 
   return SVN_NO_ERROR;
@@ -594,7 +657,7 @@ svn_fs_x__ensure_revision_exists(svn_rev
   if (rev <= ffd->youngest_rev_cache)
     return SVN_NO_ERROR;
 
-  SVN_ERR(get_youngest(&(ffd->youngest_rev_cache), fs->path, pool));
+  SVN_ERR(svn_fs_x__read_current(&ffd->youngest_rev_cache, fs, pool));
 
   /* Check again. */
   if (rev <= ffd->youngest_rev_cache)
@@ -604,67 +667,13 @@ svn_fs_x__ensure_revision_exists(svn_rev
                            _("No such revision %ld"), rev);
 }
 
-/* Open the correct revision file for REV.  If the filesystem FS has
-   been packed, *FILE will be set to the packed file; otherwise, set *FILE
-   to the revision file for REV.  Return SVN_ERR_FS_NO_SUCH_REVISION if the
-   file doesn't exist.
-
-   TODO: Consider returning an indication of whether this is a packed rev
-         file, so the caller need not rely on is_packed_rev() which in turn
-         relies on the cached FFD->min_unpacked_rev value not having changed
-         since the rev file was opened.
-
-   Use POOL for allocations. */
-svn_error_t *
-svn_fs_x__open_pack_or_rev_file(apr_file_t **file,
-                                svn_fs_t *fs,
-                                svn_revnum_t rev,
-                                apr_pool_t *pool)
-{
-  svn_error_t *err;
-  svn_boolean_t retry = FALSE;
-
-  do
-    {
-      const char *path = svn_fs_x__path_rev_absolute(fs, rev, pool);
-
-      /* open the revision file in buffered r/o mode */
-      err = svn_io_file_open(file, path,
-                            APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool);
-      if (err && APR_STATUS_IS_ENOENT(err->apr_err))
-        {
-          /* Could not open the file. This may happen if the
-            * file once existed but got packed later. */
-          svn_error_clear(err);
-
-          /* if that was our 2nd attempt, leave it at that. */
-          if (retry)
-            return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
-                                     _("No such revision %ld"), rev);
-
-          /* We failed for the first time. Refresh cache & retry. */
-          SVN_ERR(svn_fs_x__update_min_unpacked_rev(fs, pool));
-
-          retry = TRUE;
-        }
-      else
-        {
-          retry = FALSE;
-        }
-    }
-  while (retry);
-
-  return svn_error_trace(err);
-}
-
-
 svn_error_t *
 svn_fs_x__revision_proplist(apr_hash_t **proplist_p,
                             svn_fs_t *fs,
                             svn_revnum_t rev,
                             apr_pool_t *pool)
 {
-  SVN_ERR(svn_fs_x__get_revision_proplist(proplist_p, fs, rev, pool));
+  SVN_ERR(svn_fs_x__get_revision_proplist(proplist_p, fs, rev, FALSE, pool));
 
   return SVN_NO_ERROR;
 }
@@ -818,19 +827,27 @@ svn_fs_x__rep_copy(representation_t *rep
 }
 
 
-/* Write out the zeroth revision for filesystem FS. */
+/* Write out the zeroth revision for filesystem FS.
+   Perform temporary allocations in SCRATCH_POOL. */
 static svn_error_t *
-write_revision_zero(svn_fs_t *fs)
+write_revision_zero(svn_fs_t *fs,
+                    apr_pool_t *scratch_pool)
 {
-  const char *path_revision_zero = svn_fs_x__path_rev(fs, 0, fs->pool);
+  /* Use an explicit sub-pool to have full control over temp file lifetimes.
+   * Since we have it, use it for everything else as well. */
+  apr_pool_t *subpool = svn_pool_create(scratch_pool);
+  const char *path_revision_zero = svn_fs_x__path_rev(fs, 0, subpool);
   apr_hash_t *proplist;
   svn_string_t date;
-  const char *path;
 
-  /* Write out a rev file for revision 0. */
+  apr_array_header_t *index_entries;
+  svn_fs_x__p2l_entry_t *entry;
+  svn_fs_x__revision_file_t *rev_file;
+  const char *l2p_proto_index, *p2l_proto_index;
+
+  /* Write a skeleton r0 with no indexes. */
   SVN_ERR(svn_io_file_create_binary
               (path_revision_zero,
-/*             "DELTA\nSVN\4END\nENDREP\n"*/
                "DELTA\nSVN\1" /* txdelta v1 */
                  "\0\0\4\2\5" /* sview offset, sview len,
                                  tview len, instr len, newlen */
@@ -844,36 +861,57 @@ write_revision_zero(svn_fs_t *fs)
                "2d2977d1c96f487abe4a1e202dd03b4e\n"
                "cpath: /\n"
                "\n\n",
-               0x7b, fs->pool));
+               0x7b, subpool));
 
-  SVN_ERR(svn_io_set_file_read_only(path_revision_zero, FALSE, fs->pool));
+  /* Construct the index P2L contents: describe the 3 items we have.
+     Be sure to create them in on-disk order. */
+  index_entries = apr_array_make(subpool, 3, sizeof(entry));
+
+  entry = apr_pcalloc(subpool, sizeof(*entry));
+  entry->offset = 0;
+  entry->size = 0x1d;
+  entry->type = SVN_FS_X__ITEM_TYPE_DIR_REP;
+  entry->item_count = 1;
+  entry->items = apr_pcalloc(subpool, sizeof(*entry->items));
+  entry->items[0].change_set = 0;
+  entry->items[0].number = SVN_FS_X__ITEM_INDEX_FIRST_USER;
+  APR_ARRAY_PUSH(index_entries, svn_fs_x__p2l_entry_t *) = entry;
+
+  entry = apr_pcalloc(subpool, sizeof(*entry));
+  entry->offset = 0x1d;
+  entry->size = 0x5d;
+  entry->type = SVN_FS_X__ITEM_TYPE_NODEREV;
+  entry->item_count = 1;
+  entry->items = apr_pcalloc(subpool, sizeof(*entry->items));
+  entry->items[0].change_set = 0;
+  entry->items[0].number = SVN_FS_X__ITEM_INDEX_ROOT_NODE;
+  APR_ARRAY_PUSH(index_entries, svn_fs_x__p2l_entry_t *) = entry;
+
+  entry = apr_pcalloc(subpool, sizeof(*entry));
+  entry->offset = 0x1d + 0x5d;
+  entry->size = 1;
+  entry->type = SVN_FS_X__ITEM_TYPE_CHANGES;
+  entry->item_count = 1;
+  entry->items = apr_pcalloc(subpool, sizeof(*entry->items));
+  entry->items[0].change_set = 0;
+  entry->items[0].number = SVN_FS_X__ITEM_INDEX_CHANGES;
+  APR_ARRAY_PUSH(index_entries, svn_fs_x__p2l_entry_t *) = entry;
+
+  /* Now re-open r0, create proto-index files from our entries and
+      rewrite the index section of r0. */
+  SVN_ERR(svn_fs_x__open_pack_or_rev_file_writable(&rev_file, fs, 0,
+                                                   subpool, subpool));
+  SVN_ERR(svn_fs_x__p2l_index_from_p2l_entries(&p2l_proto_index, fs,
+                                               rev_file, index_entries,
+                                               subpool, subpool));
+  SVN_ERR(svn_fs_x__l2p_index_from_p2l_entries(&l2p_proto_index, fs,
+                                               index_entries,
+                                               subpool, subpool));
+  SVN_ERR(svn_fs_x__add_index_data(fs, rev_file->file, l2p_proto_index,
+                                   p2l_proto_index, 0, subpool));
+  SVN_ERR(svn_fs_x__close_revision_file(rev_file));
 
-  path = svn_fs_x__path_l2p_index(fs, 0, fs->pool);
-  SVN_ERR(svn_io_file_create_binary
-              (path,
-              "\0\1\x80\x40\1\1" /* rev 0, single page */
-              "\5\4"             /* page size: bytes, count */
-              "\0"               /* 0 container offsets in list */
-              "\0\x7b\x1e\1",    /* phys offsets + 1 */
-              13,
-              fs->pool));
-  SVN_ERR(svn_io_set_file_read_only(path, FALSE, fs->pool));
-
-  path = svn_fs_x__path_p2l_index(fs, 0, fs->pool);
-  SVN_ERR(svn_io_file_create_binary
-              (path,
-              "\0\x7b"            /* start rev, rev file size */
-              "\x80\x80\4\1\x21"  /* 64k pages, 1 page using 33 bytes */
-              "\0"                /* offset entry 0 page 1 */
-                                  /* len, type & count, checksum,
-                                     (rev, 2*item)* */
-              "\x1d\x11\x8e\xef\xf2\xd6\x01\0\6"
-              "\x5d\x15\xb6\xea\x97\xe0\x0f\0\4"
-              "\1\x16\x9d\x9e\xa9\x94\x0f\0\2"
-              "\x85\xff\3\0\0",   /* last entry fills up 64k page */
-              40,
-              fs->pool));
-  SVN_ERR(svn_io_set_file_read_only(path, FALSE, fs->pool));
+  SVN_ERR(svn_io_set_file_read_only(path_revision_zero, FALSE, fs->pool));
 
   /* Set a date on revision 0. */
   date.data = svn_time_to_cstring(apr_time_now(), fs->pool);
@@ -884,40 +922,19 @@ write_revision_zero(svn_fs_t *fs)
 }
 
 svn_error_t *
-svn_fs_x__create(svn_fs_t *fs,
-                 const char *path,
-                 apr_pool_t *pool)
+svn_fs_x__create_file_tree(svn_fs_t *fs,
+                           const char *path,
+                           int format,
+                           int shard_size,
+                           apr_pool_t *pool)
 {
-  int format = SVN_FS_X__FORMAT_NUMBER;
   fs_x_data_t *ffd = fs->fsap_data;
 
-  fs->path = apr_pstrdup(pool, path);
-  /* See if compatibility with older versions was explicitly requested. */
-  if (fs->config)
-    {
-      svn_version_t *compatible_version;
-      SVN_ERR(svn_fs__compatible_version(&compatible_version, fs->config,
-                                         pool));
-
-      /* select format number */
-      switch(compatible_version->minor)
-        {
-          case 0:
-          case 1:
-          case 2:
-          case 3:
-          case 4:
-          case 5:
-          case 6:
-          case 7:
-          case 8: return svn_error_create(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL,
-                  _("FSX is not compatible with Subversion prior to 1.9"));
-
-          default:format = SVN_FS_X__FORMAT_NUMBER;
-        }
-    }
+  fs->path = apr_pstrdup(fs->pool, path);
   ffd->format = format;
-  ffd->max_files_per_dir = SVN_FS_X_DEFAULT_MAX_FILES_PER_DIR;
+
+  /* Use an appropriate sharding mode if supported by the format. */
+  ffd->max_files_per_dir = shard_size;
 
   /* Create the revision data directories. */
   SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_rev_shard(fs, 0, pool),
@@ -929,29 +946,31 @@ svn_fs_x__create(svn_fs_t *fs,
                                       pool));
 
   /* Create the transaction directory. */
-  SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path, PATH_TXNS_DIR,
-                                                      pool),
+  SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_txns_dir(fs, pool),
                                       pool));
 
   /* Create the protorevs directory. */
-  SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path, PATH_TXN_PROTOS_DIR,
-                                                      pool),
+  SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_txn_proto_revs(fs, pool),
                                       pool));
 
   /* Create the 'current' file. */
-  SVN_ERR(svn_io_file_create(svn_fs_x__path_current(fs, pool), "0\n", pool));
-  SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_lock(fs, pool), pool));
-  SVN_ERR(svn_fs_x__set_uuid(fs, NULL, pool));
+  SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_current(fs, pool), pool));
+  SVN_ERR(svn_fs_x__write_current(fs, 0, pool));
 
-  SVN_ERR(write_revision_zero(fs));
+  /* Create the 'uuid' file. */
+  SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_lock(fs, pool), pool));
+  SVN_ERR(svn_fs_x__set_uuid(fs, NULL, NULL, pool));
 
+  /* Create the fsfs.conf file. */
   SVN_ERR(write_config(fs, pool));
-
   SVN_ERR(read_config(ffd, fs->path, fs->pool, pool));
 
+  /* Add revision 0. */
+  SVN_ERR(write_revision_zero(fs, pool));
+
   /* Create the min unpacked rev file. */
   SVN_ERR(svn_io_file_create(svn_fs_x__path_min_unpacked_rev(fs, pool),
-                              "0\n", pool));
+                             "0\n", pool));
 
   /* Create the txn-current file if the repository supports
      the transaction sequence file. */
@@ -960,6 +979,52 @@ svn_fs_x__create(svn_fs_t *fs,
   SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_txn_current_lock(fs, pool),
                                    pool));
 
+  /* Initialize the revprop caching info. */
+  SVN_ERR(svn_fs_x__reset_revprop_generation_file(fs, pool));
+
+  ffd->youngest_rev_cache = 0;
+  return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs_x__create(svn_fs_t *fs,
+                 const char *path,
+                 apr_pool_t *pool)
+{
+  int format = SVN_FS_X__FORMAT_NUMBER;
+  fs_x_data_t *ffd = fs->fsap_data;
+
+  fs->path = apr_pstrdup(fs->pool, path);
+  /* See if compatibility with older versions was explicitly requested. */
+  if (fs->config)
+    {
+      svn_version_t *compatible_version;
+      SVN_ERR(svn_fs__compatible_version(&compatible_version, fs->config,
+                                         pool));
+
+      /* select format number */
+      switch(compatible_version->minor)
+        {
+          case 0:
+          case 1:
+          case 2:
+          case 3:
+          case 4:
+          case 5:
+          case 6:
+          case 7:
+          case 8: return svn_error_create(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL,
+                  _("FSX is not compatible with Subversion prior to 1.9"));
+
+          default:format = SVN_FS_X__FORMAT_NUMBER;
+        }
+    }
+
+  /* Actual FS creation. */
+  SVN_ERR(svn_fs_x__create_file_tree(fs, path, format,
+                                     SVN_FS_X_DEFAULT_MAX_FILES_PER_DIR,
+                                     pool));
+
   /* This filesystem is ready.  Stamp it with a format number. */
   SVN_ERR(svn_fs_x__write_format(fs, FALSE, pool));
 
@@ -970,28 +1035,32 @@ svn_fs_x__create(svn_fs_t *fs,
 svn_error_t *
 svn_fs_x__set_uuid(svn_fs_t *fs,
                    const char *uuid,
+                   const char *instance_id,
                    apr_pool_t *pool)
 {
-  char *my_uuid;
-  apr_size_t my_uuid_len;
+  fs_x_data_t *ffd = fs->fsap_data;
   const char *uuid_path = svn_fs_x__path_uuid(fs, pool);
+  svn_stringbuf_t *contents = svn_stringbuf_create_empty(pool);
 
   if (! uuid)
     uuid = svn_uuid_generate(pool);
 
-  /* Make sure we have a copy in FS->POOL, and append a newline. */
-  my_uuid = apr_pstrcat(fs->pool, uuid, "\n", SVN_VA_NULL);
-  my_uuid_len = strlen(my_uuid);
+  if (! instance_id)
+    instance_id = svn_uuid_generate(pool);
+
+  svn_stringbuf_appendcstr(contents, uuid);
+  svn_stringbuf_appendcstr(contents, "\n");
+  svn_stringbuf_appendcstr(contents, instance_id);
+  svn_stringbuf_appendcstr(contents, "\n");
 
   /* We use the permissions of the 'current' file, because the 'uuid'
      file does not exist during repository creation. */
-  SVN_ERR(svn_io_write_atomic(uuid_path, my_uuid, my_uuid_len,
+  SVN_ERR(svn_io_write_atomic(uuid_path, contents->data, contents->len,
                               svn_fs_x__path_current(fs, pool) /* perms */,
                               pool));
 
-  /* Remove the newline we added, and stash the UUID. */
-  my_uuid[my_uuid_len - 1] = '\0';
-  fs->uuid = my_uuid;
+  fs->uuid = apr_pstrdup(fs->pool, uuid);
+  ffd->instance_id = apr_pstrdup(fs->pool, instance_id);
 
   return SVN_NO_ERROR;
 }

Modified: subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs_x.h
URL: http://svn.apache.org/viewvc/subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs_x.h?rev=1639628&r1=1639627&r2=1639628&view=diff
==============================================================================
--- subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs_x.h (original)
+++ subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/fs_x.h Fri Nov 14 13:17:55 2014
@@ -25,6 +25,11 @@
 
 #include "fs.h"
 
+/* Read the 'format' file of fsx filesystem FS and store its info in FS.
+ * Use SCRATCH_POOL for temporary allocations. */
+svn_error_t *
+svn_fs_x__read_format_file(svn_fs_t *fs, apr_pool_t *scratch_pool);
+
 /* Open the fsx filesystem pointed to by PATH and associate it with
    filesystem object FS.  Use POOL for temporary allocations.
 
@@ -51,15 +56,6 @@ svn_error_t *svn_fs_x__youngest_rev(svn_
                                     svn_fs_t *fs,
                                     apr_pool_t *pool);
 
-/* For revision REV in fileysystem FS, open the revision (or packed rev)
-   file and seek to the start of the revision.  Return it in *FILE, and
-   use POOL for allocations. */
-svn_error_t *
-svn_fs_x__open_pack_or_rev_file(apr_file_t **file,
-                                svn_fs_t *fs,
-                                svn_revnum_t rev,
-                                apr_pool_t *pool);
-
 /* Return SVN_ERR_FS_NO_SUCH_REVISION if the given revision REV is newer
    than the current youngest revision in FS or is simply not a valid
    revision number, else return success. */
@@ -118,6 +114,23 @@ svn_error_t *svn_fs_x__file_checksum(svn
                                      svn_checksum_kind_t kind,
                                      apr_pool_t *pool);
 
+/* Under the repository db PATH, create a FSFS repository with FORMAT,
+ * the given SHARD_SIZE.  If not supported by the respective format,
+ * the latter two parameters will be ignored.  FS will be updated.
+ *
+ * The only file not being written is the 'format' file.  This allows
+ * callers such as hotcopy to modify the contents before turning the
+ * tree into an accessible repository.
+ *
+ * Use POOL for temporary allocations.
+ */
+svn_error_t *
+svn_fs_x__create_file_tree(svn_fs_t *fs,
+                           const char *path,
+                           int format,
+                           int shard_size,
+                           apr_pool_t *pool);
+
 /* Create a fs_x fileysystem referenced by FS at path PATH.  Get any
    temporary allocations from POOL.
 
@@ -127,11 +140,12 @@ svn_error_t *svn_fs_x__create(svn_fs_t *
                               const char *path,
                               apr_pool_t *pool);
 
-/* Set the uuid of repository FS to UUID, if UUID is not NULL;
-   otherwise, set the uuid of FS to a newly generated UUID.  Perform
-   temporary allocations in POOL. */
+/* Set the uuid of repository FS to UUID and the instance ID to INSTANCE_ID.
+   If any of them is NULL, use a newly generated UUID / ID instead.
+   Perform temporary allocations in POOL. */
 svn_error_t *svn_fs_x__set_uuid(svn_fs_t *fs,
                                 const char *uuid,
+                                const char *instance_id,
                                 apr_pool_t *pool);
 
 /* Set *PATH to the path of REV in FS, whether in a pack file or not.

Modified: subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/hotcopy.c
URL: http://svn.apache.org/viewvc/subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/hotcopy.c?rev=1639628&r1=1639627&r2=1639628&view=diff
==============================================================================
--- subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/hotcopy.c (original)
+++ subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/hotcopy.c Fri Nov 14 13:17:55 2014
@@ -36,9 +36,13 @@
 #include "svn_private_config.h"
 
 /* Like svn_io_dir_file_copy(), but doesn't copy files that exist at
- * the destination and do not differ in terms of kind, size, and mtime. */
+ * the destination and do not differ in terms of kind, size, and mtime.
+ * Set *SKIPPED_P to FALSE only if the file was copied, do not change
+ * the value in *SKIPPED_P otherwise. SKIPPED_P may be NULL if not
+ * required. */
 static svn_error_t *
-hotcopy_io_dir_file_copy(const char *src_path,
+hotcopy_io_dir_file_copy(svn_boolean_t *skipped_p,
+                         const char *src_path,
                          const char *dst_path,
                          const char *file,
                          apr_pool_t *scratch_pool)
@@ -66,6 +70,9 @@ hotcopy_io_dir_file_copy(const char *src
         return SVN_NO_ERROR;
     }
 
+  if (skipped_p)
+    *skipped_p = FALSE;
+
   return svn_error_trace(svn_io_dir_file_copy(src_path, dst_path, file,
                                               scratch_pool));
 }
@@ -107,9 +114,12 @@ entry_name_to_utf8(const char **name_p,
 
 /* Like svn_io_copy_dir_recursively() but doesn't copy regular files that
  * exist in the destination and do not differ from the source in terms of
- * kind, size, and mtime. */
+ * kind, size, and mtime. Set *SKIPPED_P to FALSE only if at least one
+ * file was copied, do not change the value in *SKIPPED_P otherwise.
+ * SKIPPED_P may be NULL if not required. */
 static svn_error_t *
-hotcopy_io_copy_dir_recursively(const char *src,
+hotcopy_io_copy_dir_recursively(svn_boolean_t *skipped_p,
+                                const char *src,
                                 const char *dst_parent,
                                 const char *dst_basename,
                                 svn_boolean_t copy_perms,
@@ -175,8 +185,8 @@ hotcopy_io_copy_dir_recursively(const ch
                                      src, subpool));
           if (this_entry.filetype == APR_REG) /* regular file */
             {
-              SVN_ERR(hotcopy_io_dir_file_copy(src, dst_path, entryname_utf8,
-                                               subpool));
+              SVN_ERR(hotcopy_io_dir_file_copy(skipped_p, src, dst_path,
+                                               entryname_utf8, subpool));
             }
           else if (this_entry.filetype == APR_LNK) /* symlink */
             {
@@ -199,7 +209,8 @@ hotcopy_io_copy_dir_recursively(const ch
                 continue;
 
               src_target = svn_dirent_join(src, entryname_utf8, subpool);
-              SVN_ERR(hotcopy_io_copy_dir_recursively(src_target,
+              SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p,
+                                                      src_target,
                                                       dst_path,
                                                       entryname_utf8,
                                                       copy_perms,
@@ -229,14 +240,15 @@ hotcopy_io_copy_dir_recursively(const ch
 
 /* Copy an un-packed revision or revprop file for revision REV from SRC_SUBDIR
  * to DST_SUBDIR. Assume a sharding layout based on MAX_FILES_PER_DIR.
- * If INCLUDE_INDEXES is set, copy rev index files as well.
+ * Set *SKIPPED_P to FALSE only if the file was copied, do not change the
+ * value in *SKIPPED_P otherwise. SKIPPED_P may be NULL if not required.
  * Use SCRATCH_POOL for temporary allocations. */
 static svn_error_t *
-hotcopy_copy_shard_file(const char *src_subdir,
+hotcopy_copy_shard_file(svn_boolean_t *skipped_p,
+                        const char *src_subdir,
                         const char *dst_subdir,
                         svn_revnum_t rev,
                         int max_files_per_dir,
-                        svn_boolean_t include_indexes,
                         apr_pool_t *scratch_pool)
 {
   const char *src_subdir_shard = src_subdir,
@@ -254,21 +266,10 @@ hotcopy_copy_shard_file(const char *src_
                                 scratch_pool));
     }
 
-  SVN_ERR(hotcopy_io_dir_file_copy(src_subdir_shard, dst_subdir_shard,
+  SVN_ERR(hotcopy_io_dir_file_copy(skipped_p,
+                                   src_subdir_shard, dst_subdir_shard,
                                    apr_psprintf(scratch_pool, "%ld", rev),
                                    scratch_pool));
-  if (include_indexes)
-    {
-      SVN_ERR(hotcopy_io_dir_file_copy(src_subdir_shard, dst_subdir_shard,
-                                       apr_psprintf(scratch_pool, "%ld.l2p",
-                                                    rev),
-                                       scratch_pool));
-      SVN_ERR(hotcopy_io_dir_file_copy(src_subdir_shard, dst_subdir_shard,
-                                       apr_psprintf(scratch_pool, "%ld.p2l",
-                                                    rev),
-                                       scratch_pool));
-    }
-
   return SVN_NO_ERROR;
 }
 
@@ -277,9 +278,13 @@ hotcopy_copy_shard_file(const char *src_
  * MAX_FILES_PER_DIR revisions, from SRC_FS to DST_FS.
  * Update *DST_MIN_UNPACKED_REV in case the shard is new in DST_FS.
  * Do not re-copy data which already exists in DST_FS.
+ * Set *SKIPPED_P to FALSE only if at least one part of the shard
+ * was copied, do not change the value in *SKIPPED_P otherwise.
+ * SKIPPED_P may be NULL if not required.
  * Use SCRATCH_POOL for temporary allocations. */
 static svn_error_t *
-hotcopy_copy_packed_shard(svn_revnum_t *dst_min_unpacked_rev,
+hotcopy_copy_packed_shard(svn_boolean_t *skipped_p,
+                          svn_revnum_t *dst_min_unpacked_rev,
                           svn_fs_t *src_fs,
                           svn_fs_t *dst_fs,
                           svn_revnum_t rev,
@@ -301,7 +306,7 @@ hotcopy_copy_packed_shard(svn_revnum_t *
                               rev / max_files_per_dir);
   src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard,
                                             scratch_pool);
-  SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard,
+  SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p, src_subdir_packed_shard,
                                           dst_subdir, packed_shard,
                                           TRUE /* copy_perms */,
                                           NULL /* cancel_func */, NULL,
@@ -321,9 +326,9 @@ hotcopy_copy_packed_shard(svn_revnum_t *
         {
           svn_pool_clear(iterpool);
 
-          SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir,
+          SVN_ERR(hotcopy_copy_shard_file(skipped_p, src_subdir, dst_subdir,
                                           revprop_rev, max_files_per_dir,
-                                          FALSE, iterpool));
+                                          iterpool));
         }
       svn_pool_destroy(iterpool);
     }
@@ -331,8 +336,8 @@ hotcopy_copy_packed_shard(svn_revnum_t *
     {
       /* revprop for revision 0 will never be packed */
       if (rev == 0)
-        SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir,
-                                        0, max_files_per_dir, FALSE,
+        SVN_ERR(hotcopy_copy_shard_file(skipped_p, src_subdir, dst_subdir,
+                                        0, max_files_per_dir,
                                         scratch_pool));
 
       /* packed revprops folder */
@@ -340,7 +345,8 @@ hotcopy_copy_packed_shard(svn_revnum_t *
                                   rev / max_files_per_dir);
       src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard,
                                                 scratch_pool);
-      SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard,
+      SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p,
+                                              src_subdir_packed_shard,
                                               dst_subdir, packed_shard,
                                               TRUE /* copy_perms */,
                                               NULL /* cancel_func */, NULL,
@@ -359,53 +365,30 @@ hotcopy_copy_packed_shard(svn_revnum_t *
   return SVN_NO_ERROR;
 }
 
-/* If NEW_YOUNGEST is younger than *DST_YOUNGEST, update the 'current'
- * file in DST_FS and set *DST_YOUNGEST to NEW_YOUNGEST.
- * Use SCRATCH_POOL for temporary allocations. */
-static svn_error_t *
-hotcopy_update_current(svn_revnum_t *dst_youngest,
-                       svn_fs_t *dst_fs,
-                       svn_revnum_t new_youngest,
-                       apr_pool_t *scratch_pool)
-{
-  if (*dst_youngest >= new_youngest)
-    return SVN_NO_ERROR;
-
-  /* Update 'current'. */
-  SVN_ERR(svn_fs_x__write_current(dst_fs, new_youngest, scratch_pool));
-
-  *dst_youngest = new_youngest;
-
-  return SVN_NO_ERROR;
-}
-
-/* Remove FILE in SHARD folder.  Use POOL for temporary allocations. */
+/* Remove file PATH, if it exists - even if it is read-only. 
+ * Use POOL for temporary allocations. */
 static svn_error_t *
-hotcopy_remove_file(const char *shard,
-                    const char *file,
+hotcopy_remove_file(const char *path,
                     apr_pool_t *pool)
 {
-  const char *rev_path = svn_dirent_join(shard, file, pool);
-
   /* Make the rev file writable and remove it. */
-  SVN_ERR(svn_io_set_file_read_write(rev_path, TRUE, pool));
-  SVN_ERR(svn_io_remove_file2(rev_path, TRUE, pool));
+  SVN_ERR(svn_io_set_file_read_write(path, TRUE, pool));
+  SVN_ERR(svn_io_remove_file2(path, TRUE, pool));
 
   return SVN_NO_ERROR;
 }
 
 
 /* Remove revision or revprop files between START_REV (inclusive) and
- * END_REV (non-inclusive) from folder DST_SUBDIR in DST_FS.  Also,
- * remove index files if REMOVE_INDEXES is set.  Assume sharding as per
- * MAX_FILES_PER_DIR.  Use SCRATCH_POOL for temporary allocations. */
+ * END_REV (non-inclusive) from folder DST_SUBDIR in DST_FS.  Assume
+ * sharding as per MAX_FILES_PER_DIR.
+ * Use SCRATCH_POOL for temporary allocations. */
 static svn_error_t *
 hotcopy_remove_files(svn_fs_t *dst_fs,
                      const char *dst_subdir,
                      svn_revnum_t start_rev,
                      svn_revnum_t end_rev,
                      int max_files_per_dir,
-                     svn_boolean_t remove_indexes,
                      apr_pool_t *scratch_pool)
 {
   const char *shard;
@@ -430,18 +413,11 @@ hotcopy_remove_files(svn_fs_t *dst_fs,
         }
 
       /* remove files for REV */
-      SVN_ERR(hotcopy_remove_file(dst_subdir_shard,
-                                  apr_psprintf(iterpool, "%ld", rev),
+      SVN_ERR(hotcopy_remove_file(svn_dirent_join(dst_subdir_shard,
+                                                  apr_psprintf(iterpool,
+                                                               "%ld", rev),
+                                                  iterpool),
                                   iterpool));
-      if (remove_indexes)
-        {
-          SVN_ERR(hotcopy_remove_file(dst_subdir_shard,
-                                      apr_psprintf(iterpool, "%ld.p2l", rev),
-                                      iterpool));
-          SVN_ERR(hotcopy_remove_file(dst_subdir_shard,
-                                      apr_psprintf(iterpool, "%ld.l2p", rev),
-                                      iterpool));
-        }
     }
 
   svn_pool_destroy(iterpool);
@@ -465,7 +441,7 @@ hotcopy_remove_rev_files(svn_fs_t *dst_f
                                                PATH_REVS_DIR,
                                                scratch_pool),
                                start_rev, end_rev,
-                               max_files_per_dir, TRUE, scratch_pool));
+                               max_files_per_dir, scratch_pool));
 
   return SVN_NO_ERROR;
 }
@@ -489,7 +465,7 @@ hotcopy_remove_revprop_files(svn_fs_t *d
                                                PATH_REVPROPS_DIR,
                                                scratch_pool),
                                start_rev ? start_rev : 1, end_rev,
-                               max_files_per_dir, FALSE, scratch_pool));
+                               max_files_per_dir, scratch_pool));
 
   return SVN_NO_ERROR;
 }
@@ -552,135 +528,36 @@ remove_folder(const char *path,
   return svn_error_trace(err);
 }
 
-/* Baton for hotcopy_body(). */
-struct hotcopy_body_baton {
-  svn_fs_t *src_fs;
-  svn_fs_t *dst_fs;
-  svn_boolean_t incremental;
-  svn_cancel_func_t cancel_func;
-  void *cancel_baton;
-};
-
-/* Perform a hotcopy, either normal or incremental.
- *
- * Normal hotcopy assumes that the destination exists as an empty
- * directory. It behaves like an incremental hotcopy except that
- * none of the copied files already exist in the destination.
- *
- * An incremental hotcopy copies only changed or new files to the destination,
- * and removes files from the destination no longer present in the source.
- * While the incremental hotcopy is running, readers should still be able
- * to access the destintation repository without error and should not see
- * revisions currently in progress of being copied. Readers are able to see
- * new fully copied revisions even if the entire incremental hotcopy procedure
- * has not yet completed.
- *
- * Writers are blocked out completely during the entire incremental hotcopy
- * process to ensure consistency. This function assumes that the repository
- * write-lock is held.
+/* Copy the revision and revprop files (possibly sharded / packed) from
+ * SRC_FS to DST_FS.  Do not re-copy data which already exists in DST_FS.
+ * When copying packed or unpacked shards, checkpoint the result in DST_FS
+ * for every shard by updating the 'current' file if necessary.  Assume
+ * the >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT filesystem format without
+ * global next-ID counters.  Indicate progress via the optional NOTIFY_FUNC
+ * callback using NOTIFY_BATON.  Use POOL for temporary allocations.
  */
 static svn_error_t *
-hotcopy_body(void *baton, apr_pool_t *pool)
+hotcopy_revisions(svn_fs_t *src_fs,
+                  svn_fs_t *dst_fs,
+                  svn_revnum_t src_youngest,
+                  svn_revnum_t dst_youngest,
+                  svn_boolean_t incremental,
+                  const char *src_revs_dir,
+                  const char *dst_revs_dir,
+                  const char *src_revprops_dir,
+                  const char *dst_revprops_dir,
+                  svn_fs_hotcopy_notify_t notify_func,
+                  void* notify_baton,
+                  svn_cancel_func_t cancel_func,
+                  void* cancel_baton,
+                  apr_pool_t *pool)
 {
-  struct hotcopy_body_baton *hbb = baton;
-  svn_fs_t *src_fs = hbb->src_fs;
   fs_x_data_t *src_ffd = src_fs->fsap_data;
-  svn_fs_t *dst_fs = hbb->dst_fs;
   int max_files_per_dir = src_ffd->max_files_per_dir;
-  svn_boolean_t incremental = hbb->incremental;
-  svn_cancel_func_t cancel_func = hbb->cancel_func;
-  void* cancel_baton = hbb->cancel_baton;
-  svn_revnum_t src_youngest;
-  svn_revnum_t dst_youngest;
-  svn_revnum_t rev;
   svn_revnum_t src_min_unpacked_rev;
   svn_revnum_t dst_min_unpacked_rev;
-  const char *src_subdir;
-  const char *dst_subdir;
-  const char *revprop_src_subdir;
-  const char *revprop_dst_subdir;
+  svn_revnum_t rev;
   apr_pool_t *iterpool;
-  svn_node_kind_t kind;
-
-  /* Try to copy the config.
-   *
-   * ### We try copying the config file before doing anything else,
-   * ### because higher layers will abort the hotcopy if we throw
-   * ### an error from this function, and that renders the hotcopy
-   * ### unusable anyway. */
-  svn_error_t *err;
-
-  err = svn_io_dir_file_copy(src_fs->path, dst_fs->path, PATH_CONFIG,
-                             pool);
-  if (err)
-    {
-      if (APR_STATUS_IS_ENOENT(err->apr_err))
-        {
-          /* 1.6.0 to 1.6.11 did not copy the configuration file during
-            * hotcopy. So if we're hotcopying a repository which has been
-            * created as a hotcopy itself, it's possible that fsx.conf
-            * does not exist. Ask the user to re-create it.
-            *
-            * ### It would be nice to make this a non-fatal error,
-            * ### but this function does not get an svn_fs_t object
-            * ### so we have no way of just printing a warning via
-            * ### the fs->warning() callback. */
-
-          const char *msg;
-          const char *src_abspath;
-          const char *dst_abspath;
-          const char *config_relpath;
-          svn_error_t *err2;
-
-          config_relpath = svn_dirent_join(src_fs->path, PATH_CONFIG, pool);
-          err2 = svn_dirent_get_absolute(&src_abspath, src_fs->path, pool);
-          if (err2)
-            return svn_error_trace(svn_error_compose_create(err, err2));
-          err2 = svn_dirent_get_absolute(&dst_abspath, dst_fs->path, pool);
-          if (err2)
-            return svn_error_trace(svn_error_compose_create(err, err2));
-
-          /* ### hack: strip off the 'db/' directory from paths so
-            * ### they make sense to the user */
-          src_abspath = svn_dirent_dirname(src_abspath, pool);
-          dst_abspath = svn_dirent_dirname(dst_abspath, pool);
-
-          msg = apr_psprintf(pool,
-                             _("Failed to create hotcopy at '%s'. "
-                               "The file '%s' is missing from the source "
-                               "repository. Please create this file, for "
-                               "instance by running 'svnadmin upgrade %s'"),
-                             dst_abspath, config_relpath, src_abspath);
-          return svn_error_quick_wrap(err, msg);
-        }
-      else
-        return svn_error_trace(err);
-    }
-
-  if (cancel_func)
-    SVN_ERR(cancel_func(cancel_baton));
-
-  /* Find the youngest revision in the source and destination.
-   * We only support hotcopies from sources with an equal or greater amount
-   * of revisions than the destination.
-   * This also catches the case where users accidentally swap the
-   * source and destination arguments. */
-  SVN_ERR(svn_fs_x__youngest_rev(&src_youngest, src_fs, pool));
-  if (incremental)
-    {
-      SVN_ERR(svn_fs_x__youngest_rev(&dst_youngest, dst_fs, pool));
-      if (src_youngest < dst_youngest)
-        return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
-                 _("The hotcopy destination already contains more revisions "
-                   "(%lu) than the hotcopy source contains (%lu); are source "
-                   "and destination swapped?"),
-                  dst_youngest, src_youngest);
-    }
-  else
-    dst_youngest = 0;
-
-  if (cancel_func)
-    SVN_ERR(cancel_func(cancel_baton));
 
   /* Copy the min unpacked rev, and read its value. */
   SVN_ERR(svn_fs_x__read_min_unpacked_rev(&src_min_unpacked_rev, src_fs,
@@ -710,30 +587,41 @@ hotcopy_body(void *baton, apr_pool_t *po
    * Copy the necessary rev files.
    */
 
-  src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, pool);
-  dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, pool);
-  SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool));
-
   iterpool = svn_pool_create(pool);
   /* First, copy packed shards. */
   for (rev = 0; rev < src_min_unpacked_rev; rev += max_files_per_dir)
     {
+      svn_boolean_t skipped = TRUE;
+      svn_revnum_t pack_end_rev;
+
       svn_pool_clear(iterpool);
 
       if (cancel_func)
         SVN_ERR(cancel_func(cancel_baton));
 
       /* Copy the packed shard. */
-      SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev,
+      SVN_ERR(hotcopy_copy_packed_shard(&skipped, &dst_min_unpacked_rev,
                                         src_fs, dst_fs,
                                         rev, max_files_per_dir,
                                         iterpool));
 
-      /* If necessary, update 'current' to the most recent packed rev,
-       * so readers can see new revisions which arrived in this pack. */
-      SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs,
-                                     rev + max_files_per_dir - 1,
-                                     iterpool));
+      pack_end_rev = rev + max_files_per_dir - 1;
+
+      /* Whenever this pack did not previously exist in the destination,
+       * update 'current' to the most recent packed rev (so readers can see
+       * new revisions which arrived in this pack). */
+      if (pack_end_rev > dst_youngest)
+        {
+          SVN_ERR(svn_fs_x__write_current(dst_fs, pack_end_rev, iterpool));
+        }
+
+      /* When notifying about packed shards, make things simpler by either
+       * reporting a full revision range, i.e [pack start, pack end] or
+       * reporting nothing. There is one case when this approach might not
+       * be exact (incremental hotcopy with a pack replacing last unpacked
+       * revisions), but generally this is good enough. */
+      if (notify_func && !skipped)
+        notify_func(notify_baton, rev, pack_end_rev, iterpool);
 
       /* Remove revision files which are now packed. */
       if (incremental)
@@ -760,90 +648,170 @@ hotcopy_body(void *baton, apr_pool_t *po
   if (cancel_func)
     SVN_ERR(cancel_func(cancel_baton));
 
-  /* Now, copy pairs of non-packed revisions and revprop files.
-   * If necessary, update 'current' after copying all files from a shard. */
   SVN_ERR_ASSERT(rev == src_min_unpacked_rev);
   SVN_ERR_ASSERT(src_min_unpacked_rev == dst_min_unpacked_rev);
-  revprop_src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, pool);
-  revprop_dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, pool);
-  SVN_ERR(svn_io_make_dir_recursively(revprop_dst_subdir, pool));
+
+  /* Now, copy pairs of non-packed revisions and revprop files.
+   * If necessary, update 'current' after copying all files from a shard. */
   for (; rev <= src_youngest; rev++)
     {
+      svn_boolean_t skipped = TRUE;
+
       svn_pool_clear(iterpool);
 
       if (cancel_func)
         SVN_ERR(cancel_func(cancel_baton));
 
+      /* Copying non-packed revisions is racy in case the source repository is
+       * being packed concurrently with this hotcopy operation.  With the pack
+       * lock, however, the race is impossible, because hotcopy and pack
+       * operations block each other.
+       *
+       * We assume that all revisions coming after 'min-unpacked-rev' really
+       * are unpacked and that's not necessarily true with concurrent packing.
+       * Don't try to be smart in this edge case, because handling it properly
+       * might require copying *everything* from the start. Just abort the
+       * hotcopy with an ENOENT (revision file moved to a pack, so it is no
+       * longer where we expect it to be). */
+
       /* Copy the rev file. */
-      err = hotcopy_copy_shard_file(src_subdir, dst_subdir,
-                                    rev, max_files_per_dir, TRUE,
-                                    iterpool);
-      if (err)
+      SVN_ERR(hotcopy_copy_shard_file(&skipped, src_revs_dir, dst_revs_dir,
+                                      rev, max_files_per_dir,
+                                      iterpool));
+
+      /* Copy the revprop file. */
+      SVN_ERR(hotcopy_copy_shard_file(&skipped, src_revprops_dir,
+                                      dst_revprops_dir,
+                                      rev, max_files_per_dir, 
+                                      iterpool));
+
+      /* Whenever this revision did not previously exist in the destination,
+       * checkpoint the progress via 'current' (do that once per full shard
+       * in order not to slow things down). */
+      if (rev > dst_youngest)
         {
-          if (APR_STATUS_IS_ENOENT(err->apr_err))
+          if (max_files_per_dir && (rev % max_files_per_dir == 0))
             {
-              svn_error_clear(err);
-
-              /* The source rev file does not exist. This can happen if the
-               * source repository is being packed concurrently with this
-               * hotcopy operation.
-               *
-               * If the new revision is now packed, and the youngest revision
-               * we're interested in is not inside this pack, try to copy the
-               * pack instead.
-               *
-               * If the youngest revision ended up being packed, don't try
-               * to be smart and work around this. Just abort the hotcopy. */
-              SVN_ERR(svn_fs_x__update_min_unpacked_rev(src_fs, pool));
-              if (svn_fs_x__is_packed_rev(src_fs, rev))
-                {
-                  if (svn_fs_x__is_packed_rev(src_fs, src_youngest))
-                    return svn_error_createf(
-                             SVN_ERR_FS_NO_SUCH_REVISION, NULL,
-                             _("The assumed HEAD revision (%lu) of the "
-                               "hotcopy source has been packed while the "
-                               "hotcopy was in progress; please restart "
-                               "the hotcopy operation"),
-                             src_youngest);
-
-                  SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev,
-                                                    src_fs, dst_fs,
-                                                    rev, max_files_per_dir,
-                                                    iterpool));
-                  rev = dst_min_unpacked_rev;
-                  continue;
-                }
-              else
-                return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
-                                         _("Revision %lu disappeared from the "
-                                           "hotcopy source while hotcopy was "
-                                           "in progress"), rev);
+              SVN_ERR(svn_fs_x__write_current(dst_fs, rev, iterpool));
             }
-          else
-            return svn_error_trace(err);
         }
 
-      /* Copy the revprop file. */
-      SVN_ERR(hotcopy_copy_shard_file(revprop_src_subdir,
-                                      revprop_dst_subdir,
-                                      rev, max_files_per_dir, FALSE,
-                                      iterpool));
-
-      /* After completing a full shard, update 'current'. */
-      if (max_files_per_dir && rev % max_files_per_dir == 0)
-        SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, rev, iterpool));
+      if (notify_func && !skipped)
+        notify_func(notify_baton, rev, rev, iterpool);
     }
   svn_pool_destroy(iterpool);
 
-  if (cancel_func)
-    SVN_ERR(cancel_func(cancel_baton));
-
   /* We assume that all revisions were copied now, i.e. we didn't exit the
    * above loop early. 'rev' was last incremented during exit of the loop. */
   SVN_ERR_ASSERT(rev == src_youngest + 1);
 
-  /* All revisions were copied. Update 'current'. */
-  SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, src_youngest, pool));
+  return SVN_NO_ERROR;
+}
+
+/* Baton for hotcopy_body(). */
+struct hotcopy_body_baton {
+  svn_fs_t *src_fs;
+  svn_fs_t *dst_fs;
+  svn_boolean_t incremental;
+  svn_fs_hotcopy_notify_t notify_func;
+  void *notify_baton;
+  svn_cancel_func_t cancel_func;
+  void *cancel_baton;
+};
+
+/* Perform a hotcopy, either normal or incremental.
+ *
+ * Normal hotcopy assumes that the destination exists as an empty
+ * directory. It behaves like an incremental hotcopy except that
+ * none of the copied files already exist in the destination.
+ *
+ * An incremental hotcopy copies only changed or new files to the destination,
+ * and removes files from the destination no longer present in the source.
+ * While the incremental hotcopy is running, readers should still be able
+ * to access the destintation repository without error and should not see
+ * revisions currently in progress of being copied. Readers are able to see
+ * new fully copied revisions even if the entire incremental hotcopy procedure
+ * has not yet completed.
+ *
+ * Writers are blocked out completely during the entire incremental hotcopy
+ * process to ensure consistency. This function assumes that the repository
+ * write-lock is held.
+ */
+static svn_error_t *
+hotcopy_body(void *baton, apr_pool_t *pool)
+{
+  struct hotcopy_body_baton *hbb = baton;
+  svn_fs_t *src_fs = hbb->src_fs;
+  svn_fs_t *dst_fs = hbb->dst_fs;
+  svn_boolean_t incremental = hbb->incremental;
+  svn_fs_hotcopy_notify_t notify_func = hbb->notify_func;
+  void* notify_baton = hbb->notify_baton;
+  svn_cancel_func_t cancel_func = hbb->cancel_func;
+  void* cancel_baton = hbb->cancel_baton;
+  svn_revnum_t src_youngest;
+  svn_revnum_t dst_youngest;
+  const char *src_revprops_dir;
+  const char *dst_revprops_dir;
+  const char *src_revs_dir;
+  const char *dst_revs_dir;
+  const char *src_subdir;
+  const char *dst_subdir;
+  svn_node_kind_t kind;
+
+  /* Try to copy the config.
+   *
+   * ### We try copying the config file before doing anything else,
+   * ### because higher layers will abort the hotcopy if we throw
+   * ### an error from this function, and that renders the hotcopy
+   * ### unusable anyway. */
+  SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path, PATH_CONFIG,
+                               pool));
+
+  if (cancel_func)
+    SVN_ERR(cancel_func(cancel_baton));
+
+  /* Find the youngest revision in the source and destination.
+   * We only support hotcopies from sources with an equal or greater amount
+   * of revisions than the destination.
+   * This also catches the case where users accidentally swap the
+   * source and destination arguments. */
+  SVN_ERR(svn_fs_x__read_current(&src_youngest, src_fs, pool));
+  if (incremental)
+    {
+      SVN_ERR(svn_fs_x__youngest_rev(&dst_youngest, dst_fs, pool));
+      if (src_youngest < dst_youngest)
+        return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
+                 _("The hotcopy destination already contains more revisions "
+                   "(%lu) than the hotcopy source contains (%lu); are source "
+                   "and destination swapped?"),
+                   dst_youngest, src_youngest);
+    }
+  else
+    dst_youngest = 0;
+
+  src_revs_dir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, pool);
+  dst_revs_dir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, pool);
+  src_revprops_dir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, pool);
+  dst_revprops_dir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, pool);
+
+  /* Ensure that the required folders exist in the destination
+   * before actually copying the revisions and revprops. */
+  SVN_ERR(svn_io_make_dir_recursively(dst_revs_dir, pool));
+  SVN_ERR(svn_io_make_dir_recursively(dst_revprops_dir, pool));
+
+  if (cancel_func)
+    SVN_ERR(cancel_func(cancel_baton));
+
+  /* Split the logic for new and old FS formats. The latter is much simpler
+   * due to the absense of sharding and packing. However, it requires special
+   * care when updating the 'current' file (which contains not just the
+   * revision number, but also the next-ID counters). */
+  SVN_ERR(hotcopy_revisions(src_fs, dst_fs, src_youngest, dst_youngest,
+                            incremental, src_revs_dir, dst_revs_dir,
+                            src_revprops_dir, dst_revprops_dir,
+                            notify_func, notify_baton,
+                            cancel_func, cancel_baton, pool));
+  SVN_ERR(svn_fs_x__write_current(dst_fs, src_youngest, pool));
 
   /* Replace the locks tree.
    * This is racy in case readers are currently trying to list locks in
@@ -863,7 +831,7 @@ hotcopy_body(void *baton, apr_pool_t *po
   src_subdir = svn_dirent_join(src_fs->path, PATH_NODE_ORIGINS_DIR, pool);
   SVN_ERR(svn_io_check_path(src_subdir, &kind, pool));
   if (kind == svn_node_dir)
-    SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir, dst_fs->path,
+    SVN_ERR(hotcopy_io_copy_dir_recursively(NULL, src_subdir, dst_fs->path,
                                             PATH_NODE_ORIGINS_DIR, TRUE,
                                             cancel_func, cancel_baton, pool));
 
@@ -879,8 +847,16 @@ hotcopy_body(void *baton, apr_pool_t *po
   SVN_ERR(svn_io_check_path(src_subdir, &kind, pool));
   if (kind == svn_node_file)
     {
-      SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, pool));
-      SVN_ERR(svn_fs_x__del_rep_reference(dst_fs, dst_youngest, pool));
+      /* Copy the rep cache and then remove entries for revisions
+       * that did not make it into the destination. */
+      src_subdir = svn_dirent_join(src_fs->path, REP_CACHE_DB_NAME, pool);
+      dst_subdir = svn_dirent_join(dst_fs->path, REP_CACHE_DB_NAME, pool);
+      SVN_ERR(svn_io_check_path(src_subdir, &kind, pool));
+      if (kind == svn_node_file)
+        {
+          SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, pool));
+          SVN_ERR(svn_fs_x__del_rep_reference(dst_fs, src_youngest, pool));
+        }
     }
 
   /* Copy the txn-current file. */
@@ -891,17 +867,13 @@ hotcopy_body(void *baton, apr_pool_t *po
    * reset it to zero (since this is on a different path, it will not
    * overlap with data already in cache).  Also, clean up stale files
    * used for the named atomics implementation. */
-  SVN_ERR(svn_io_check_path(svn_fs_x__path_revprop_generation(src_fs, pool),
-                            &kind, pool));
-  if (kind == svn_node_file)
-    SVN_ERR(svn_fs_x__write_revprop_generation_file(dst_fs, 0, pool));
-
-  SVN_ERR(svn_fs_x__cleanup_revprop_namespace(dst_fs));
+  SVN_ERR(svn_fs_x__reset_revprop_generation_file(dst_fs, pool));
 
   return SVN_NO_ERROR;
 }
 
-/* Wrapper around hotcopy_body taking out all necessary source repositories.
+/* Wrapper around hotcopy_body taking out all necessary source repository
+ * locks.
  */
 static svn_error_t *
 hotcopy_locking_src_body(void *baton, apr_pool_t *pool)
@@ -922,56 +894,24 @@ hotcopy_create_empty_dest(svn_fs_t *src_
                           apr_pool_t *pool)
 {
   fs_x_data_t *src_ffd = src_fs->fsap_data;
-  fs_x_data_t *dst_ffd = dst_fs->fsap_data;
-
-  dst_fs->path = apr_pstrdup(pool, dst_path);
-
-  dst_ffd->max_files_per_dir = src_ffd->max_files_per_dir;
-  dst_ffd->format = src_ffd->format;
 
-  /* Create the revision data directories. */
-  SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_rev_shard(dst_fs, 0,
-                                                               pool),
-                                      pool));
-
-  /* Create the revprops directory. */
-  SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_revprops_shard(dst_fs,
-                                                                    0, pool),
-                                      pool));
-
-  /* Create the transaction directory. */
-  SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, PATH_TXNS_DIR,
-                                                      pool),
-                                      pool));
-
-  /* Create the protorevs directory. */
-  SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path,
-                                                      PATH_TXN_PROTOS_DIR,
-                                                      pool),
-                                      pool));
-
-  /* Create the 'current' file. */
-  SVN_ERR(svn_io_file_create(svn_fs_x__path_current(dst_fs, pool), "0\n", 
-                             pool));
-
-  /* Create lock file and UUID. */
-  SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_lock(dst_fs, pool), pool));
-  SVN_ERR(svn_fs_x__set_uuid(dst_fs, src_fs->uuid, pool));
-
-  /* Create the min unpacked rev file. */
-  SVN_ERR(svn_io_file_create(svn_fs_x__path_min_unpacked_rev(dst_fs, pool),
-                              "0\n", pool));
-
-  /* Create the txn-current file if the repository supports
-     the transaction sequence file. */
-  SVN_ERR(svn_io_file_create(svn_fs_x__path_txn_current(dst_fs, pool),
-                             "0\n", pool));
-  SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_txn_current_lock(dst_fs,
-                                                                   pool),
-                                   pool));
-
-  /* FS creation is complete. Stamp it with a format file. */
-  SVN_ERR(svn_fs_x__write_format(dst_fs, TRUE, pool));
+  /* Create the DST_FS repository with the same layout as SRC_FS. */
+  SVN_ERR(svn_fs_x__create_file_tree(dst_fs, dst_path, src_ffd->format,
+                                     src_ffd->max_files_per_dir, pool));
+
+  /* Copy the UUID.  Hotcopy destination receives a new instance ID, but
+   * has the same filesystem UUID as the source. */
+  SVN_ERR(svn_fs_x__set_uuid(dst_fs, src_fs->uuid, NULL, pool));
+
+  /* Remove revision 0 contents.  Otherwise, it may not get overwritten
+   * due to having a newer timestamp. */
+  SVN_ERR(hotcopy_remove_file(svn_fs_x__path_rev(dst_fs, 0, pool), pool));
+  SVN_ERR(hotcopy_remove_file(svn_fs_x__path_revprops(dst_fs, 0, pool),
+                              pool));
+
+  /* This filesystem is ready.  Stamp it with a format number.  Fail if
+   * the 'format' file should already exist. */
+  SVN_ERR(svn_fs_x__write_format(dst_fs, FALSE, pool));
 
   return SVN_NO_ERROR;
 }
@@ -1020,6 +960,8 @@ svn_error_t *
 svn_fs_x__hotcopy(svn_fs_t *src_fs,
                   svn_fs_t *dst_fs,
                   svn_boolean_t incremental,
+                  svn_fs_hotcopy_notify_t notify_func,
+                  void *notify_baton,
                   svn_cancel_func_t cancel_func,
                   void *cancel_baton,
                   apr_pool_t *pool)
@@ -1029,10 +971,12 @@ svn_fs_x__hotcopy(svn_fs_t *src_fs,
   hbb.src_fs = src_fs;
   hbb.dst_fs = dst_fs;
   hbb.incremental = incremental;
+  hbb.notify_func = notify_func;
+  hbb.notify_baton = notify_baton;
   hbb.cancel_func = cancel_func;
   hbb.cancel_baton = cancel_baton;
-  SVN_ERR(svn_fs_x__with_write_lock(dst_fs, hotcopy_locking_src_body, &hbb,
-                                    pool));
+  SVN_ERR(svn_fs_x__with_all_locks(dst_fs, hotcopy_locking_src_body, &hbb,
+                                   pool));
 
   return SVN_NO_ERROR;
 }

Modified: subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/hotcopy.h
URL: http://svn.apache.org/viewvc/subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/hotcopy.h?rev=1639628&r1=1639627&r2=1639628&view=diff
==============================================================================
--- subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/hotcopy.h (original)
+++ subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/hotcopy.h Fri Nov 14 13:17:55 2014
@@ -36,11 +36,14 @@ svn_fs_x__hotcopy_prepare_target(svn_fs_
                                  apr_pool_t *pool);
 
 /* Copy the fsfs filesystem SRC_FS into DST_FS. If INCREMENTAL is TRUE, do
- * not re-copy data which already exists in DST_FS. Use POOL for temporary
- * allocations. */
+ * not re-copy data which already exists in DST_FS.  Indicate progress via
+ * the optional NOTIFY_FUNC callback using NOTIFY_BATON.  Use POOL for
+ * temporary allocations. */
 svn_error_t * svn_fs_x__hotcopy(svn_fs_t *src_fs,
                                 svn_fs_t *dst_fs,
                                 svn_boolean_t incremental,
+                                svn_fs_hotcopy_notify_t notify_func,
+                                void *notify_baton,
                                 svn_cancel_func_t cancel_func,
                                 void *cancel_baton,
                                 apr_pool_t *pool);

Modified: subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/id.c
URL: http://svn.apache.org/viewvc/subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/id.c?rev=1639628&r1=1639627&r2=1639628&view=diff
==============================================================================
--- subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/id.c (original)
+++ subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/id.c Fri Nov 14 13:17:55 2014
@@ -246,8 +246,9 @@ svn_fs_x__id_eq(const svn_fs_id_t *a,
   if (a == b)
     return TRUE;
 
-  return memcmp(&id_a->node_id, &id_b->node_id,
-                3 * sizeof(svn_fs_x__id_part_t)) == 0;
+  return svn_fs_x__id_part_eq(&id_a->node_id, &id_b->node_id)
+      && svn_fs_x__id_part_eq(&id_a->copy_id, &id_b->copy_id)
+      && svn_fs_x__id_part_eq(&id_a->noderev_id, &id_b->noderev_id);
 }
 
 
@@ -264,7 +265,7 @@ svn_fs_x__id_check_related(const svn_fs_
   /* Items from different txns are unrelated. */
   if (   svn_fs_x__is_txn(id_a->noderev_id.change_set)
       && svn_fs_x__is_txn(id_b->noderev_id.change_set)
-      && id_a->noderev_id.change_set != id_a->noderev_id.change_set)
+      && id_a->noderev_id.change_set != id_b->noderev_id.change_set)
     return FALSE;
 
   /* related if they trace back to the same node creation */
@@ -385,7 +386,7 @@ svn_fs_id_t *
 svn_fs_x__id_copy(const svn_fs_id_t *source, apr_pool_t *pool)
 {
   const fs_x__id_t *id = (const fs_x__id_t *)source;
-  fs_x__id_t *new_id = apr_pmemdup(pool, id, sizeof(*id));
+  fs_x__id_t *new_id = apr_pmemdup(pool, id, sizeof(*new_id));
 
   new_id->generic_id.fsap_data = new_id;
   new_id->pool = pool;
@@ -393,18 +394,14 @@ svn_fs_x__id_copy(const svn_fs_id_t *sou
   return (svn_fs_id_t *)new_id;
 }
 
-
-svn_fs_id_t *
-svn_fs_x__id_parse(const char *data,
-                   apr_size_t len,
-                   apr_pool_t *pool)
+/* Return an ID resulting from parsing the string DATA, or NULL if DATA is
+   an invalid ID string. *DATA will be modified / invalidated by this call. */
+static svn_fs_id_t *
+id_parse(char *data,
+         apr_pool_t *pool)
 {
   fs_x__id_t *id;
-  char *data_copy, *str;
-
-  /* Dup the ID data into POOL.  Our returned ID will have references
-     into this memory. */
-  data_copy = apr_pstrmemdup(pool, data, len);
+  char *str;
 
   /* Alloc a new svn_fs_id_t structure. */
   id = apr_pcalloc(pool, sizeof(*id));
@@ -419,21 +416,21 @@ svn_fs_x__id_parse(const char *data,
      string.*/
 
   /* Node Id */
-  str = svn_cstring_tokenize(".", &data_copy);
+  str = svn_cstring_tokenize(".", &data);
   if (str == NULL)
     return NULL;
   if (! part_parse(&id->node_id, str))
     return NULL;
 
   /* Copy Id */
-  str = svn_cstring_tokenize(".", &data_copy);
+  str = svn_cstring_tokenize(".", &data);
   if (str == NULL)
     return NULL;
   if (! part_parse(&id->copy_id, str))
     return NULL;
 
   /* NodeRev Id */
-  str = svn_cstring_tokenize(".", &data_copy);
+  str = svn_cstring_tokenize(".", &data);
   if (str == NULL)
     return NULL;
 
@@ -443,6 +440,21 @@ svn_fs_x__id_parse(const char *data,
   return (svn_fs_id_t *)id;
 }
 
+svn_error_t *
+svn_fs_x__id_parse(const svn_fs_id_t **id_p,
+                   char *data,
+                   apr_pool_t *pool)
+{
+  svn_fs_id_t *id = id_parse(data, pool);
+  if (id == NULL)
+    return svn_error_createf(SVN_ERR_FS_MALFORMED_NODEREV_ID, NULL,
+                             "Malformed node revision ID string");
+
+  *id_p = id;
+
+  return SVN_NO_ERROR;
+}
+
 /* (de-)serialization support */
 
 /* Serialize an ID within the serialization CONTEXT.

Modified: subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/id.h
URL: http://svn.apache.org/viewvc/subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/id.h?rev=1639628&r1=1639627&r2=1639628&view=diff
==============================================================================
--- subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/id.h (original)
+++ subversion/branches/svn-auth-x509/subversion/libsvn_fs_x/id.h Fri Nov 14 13:17:55 2014
@@ -161,11 +161,13 @@ svn_fs_id_t *svn_fs_x__id_create(const s
 svn_fs_id_t *svn_fs_x__id_copy(const svn_fs_id_t *id,
                                apr_pool_t *pool);
 
-/* Return an ID resulting from parsing the string DATA (with length
-   LEN), or NULL if DATA is an invalid ID string. */
-svn_fs_id_t *svn_fs_x__id_parse(const char *data,
-                                apr_size_t len,
-                                apr_pool_t *pool);
+/* Return an ID in *ID_P resulting from parsing the string DATA, or an error
+   if DATA is an invalid ID string. *DATA will be modified / invalidated by
+   this call. */
+svn_error_t *
+svn_fs_x__id_parse(const svn_fs_id_t **id_p,
+                   char *data,
+                   apr_pool_t *pool);
 
 
 /* (de-)serialization support*/



Mime
View raw message