subversion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From stef...@apache.org
Subject svn commit: r1546928 [4/5] - in /subversion/branches/fsfs-improvements: ./ subversion/libsvn_fs_fs/ subversion/tests/cmdline/ subversion/tests/cmdline/svntest/ subversion/tests/libsvn_fs_fs/ tools/server-side/
Date Mon, 02 Dec 2013 08:48:01 GMT
Modified: subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/transaction.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/transaction.c?rev=1546928&r1=1546927&r2=1546928&view=diff
==============================================================================
--- subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/transaction.c (original)
+++ subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/transaction.c Mon Dec  2 08:48:00 2013
@@ -32,6 +32,7 @@
 #include "svn_dirent_uri.h"
 
 #include "fs_fs.h"
+#include "index.h"
 #include "tree.h"
 #include "util.h"
 #include "id.h"
@@ -403,7 +404,8 @@ get_writable_proto_rev_body(svn_fs_t *fs
   /* Now open the prototype revision file and seek to the end. */
   err = svn_io_file_open(file,
                          svn_fs_fs__path_txn_proto_rev(fs, &b->txn_id, pool),
-                         APR_WRITE | APR_BUFFERED, APR_OS_DEFAULT, pool);
+                         APR_READ | APR_WRITE | APR_BUFFERED, APR_OS_DEFAULT,
+                         pool);
 
   /* You might expect that we could dispense with the following seek
      and achieve the same thing by opening the file using APR_APPEND.
@@ -1583,6 +1585,183 @@ svn_fs_fs__add_change(svn_fs_t *fs,
   return svn_io_file_close(file, pool);
 }
 
+/* If the transaction TXN_ID in FS uses logical addressing, store the
+ * (ITEM_INDEX, OFFSET) pair in the txn's log-to-phys proto index file.
+ * If FINAL_REVISION is not SVN_INVALID_REVNUM, use it to determine whether
+ * to actually write to the proto-index.  Use POOL for allocations.
+ */
+static svn_error_t *
+store_l2p_index_entry(svn_fs_t *fs,
+                      const svn_fs_fs__id_part_t *txn_id,
+                      svn_revnum_t final_revision,
+                      apr_off_t offset,
+                      apr_uint64_t item_index,
+                      apr_pool_t *pool)
+{
+  if (final_revision == SVN_INVALID_REVNUM)
+    final_revision = txn_id->revision + 1;
+
+  if (svn_fs_fs__use_log_addressing(fs, final_revision))
+    {
+      const char *path = svn_fs_fs__path_l2p_proto_index(fs, txn_id, pool);
+      apr_file_t *file;
+      SVN_ERR(svn_fs_fs__l2p_proto_index_open(&file, path, pool));
+      SVN_ERR(svn_fs_fs__l2p_proto_index_add_entry(file, offset,
+                                                   item_index, pool));
+      SVN_ERR(svn_io_file_close(file, pool));
+    }
+
+  return SVN_NO_ERROR;
+}
+
+/* If the transaction TXN_ID in FS uses logical addressing, store ENTRY
+ * in the phys-to-log proto index file of transaction TXN_ID.
+ * If FINAL_REVISION is not SVN_INVALID_REVNUM, use it to determine whether
+ * to actually write to the proto-index.  Use POOL for allocations.
+ */
+static svn_error_t *
+store_p2l_index_entry(svn_fs_t *fs,
+                      const svn_fs_fs__id_part_t *txn_id,
+                      svn_revnum_t final_revision,
+                      svn_fs_fs__p2l_entry_t *entry,
+                      apr_pool_t *pool)
+{
+  if (final_revision == SVN_INVALID_REVNUM)
+    final_revision = txn_id->revision + 1;
+
+  if (svn_fs_fs__use_log_addressing(fs, final_revision))
+    {
+      const char *path = svn_fs_fs__path_p2l_proto_index(fs, txn_id, pool);
+      apr_file_t *file;
+      SVN_ERR(svn_fs_fs__p2l_proto_index_open(&file, path, pool));
+      SVN_ERR(svn_fs_fs__p2l_proto_index_add_entry(file, entry, pool));
+      SVN_ERR(svn_io_file_close(file, pool));
+    }
+
+  return SVN_NO_ERROR;
+}
+
+/* Allocate an item index for the given MY_OFFSET in the transaction TXN_ID
+ * of file system FS and return it in *ITEM_INDEX.  For old formats, it
+ * will simply return the offset as item index; in new formats, it will
+ * increment the txn's item index counter file and store the mapping in
+ * the proto index file.  If FINAL_REVISION is not SVN_INVALID_REVNUM, use
+ * it to determine whether to actually write to the proto-index.
+ * Use POOL for allocations.
+ */
+static svn_error_t *
+allocate_item_index(apr_uint64_t *item_index,
+                    svn_fs_t *fs,
+                    const svn_fs_fs__id_part_t *txn_id,
+                    svn_revnum_t final_revision,
+                    apr_off_t my_offset,
+                    apr_pool_t *pool)
+{
+  if (final_revision == SVN_INVALID_REVNUM)
+    final_revision = txn_id->revision + 1;
+
+  if (svn_fs_fs__use_log_addressing(fs, final_revision))
+    {
+      apr_file_t *file;
+      char buffer[SVN_INT64_BUFFER_SIZE] = { 0 };
+      svn_boolean_t eof = FALSE;
+      apr_size_t to_write;
+      apr_size_t read;
+      apr_off_t offset = 0;
+
+      /* read number, increment it and write it back to disk */
+      SVN_ERR(svn_io_file_open(&file,
+                         svn_fs_fs__path_txn_item_index(fs, txn_id, pool),
+                         APR_READ | APR_WRITE | APR_CREATE | APR_BUFFERED,
+                         APR_OS_DEFAULT, pool));
+      SVN_ERR(svn_io_file_read_full2(file, buffer, sizeof(buffer)-1,
+                                     &read, &eof, pool));
+      if (read)
+        SVN_ERR(svn_cstring_atoui64(item_index, buffer));
+      else
+        *item_index = SVN_FS_FS__ITEM_INDEX_FIRST_USER;
+
+      to_write = svn__ui64toa(buffer, *item_index + 1);
+      SVN_ERR(svn_io_file_seek(file, SEEK_SET, &offset, pool));
+      SVN_ERR(svn_io_file_write_full(file, buffer, to_write, NULL, pool));
+      SVN_ERR(svn_io_file_close(file, pool));
+
+      /* write log-to-phys index */
+      SVN_ERR(store_l2p_index_entry(fs, txn_id, final_revision,
+                                    my_offset, *item_index, pool));
+    }
+  else
+    {
+      *item_index = (apr_uint64_t)my_offset;
+    }
+
+  return SVN_NO_ERROR;
+}
+
+/* Baton used by fnv1a_write_handler to calculate the FNV checksum
+ * before passing the data on to the INNER_STREAM.
+ */
+typedef struct fnv1a_stream_baton_t
+{
+  svn_stream_t *inner_stream;
+  svn_checksum_ctx_t *context;
+} fnv1a_stream_baton_t;
+
+/* Implement svn_write_fn_t.
+ * Update checksum and pass data on to inner stream.
+ */
+static svn_error_t *
+fnv1a_write_handler(void *baton,
+                    const char *data,
+                    apr_size_t *len)
+{
+  fnv1a_stream_baton_t *b = baton;
+
+  SVN_ERR(svn_checksum_update(b->context, data, *len));
+  SVN_ERR(svn_stream_write(b->inner_stream, data, len));
+
+  return SVN_NO_ERROR;
+}
+
+/* Return a stream that calculates a FNV checksum in *CONTEXT
+ * over all data written to the stream and passes that data on
+ * to INNER_STREAM.  Allocate objects in POOL.
+ */
+static svn_stream_t *
+fnv1a_wrap_stream(svn_checksum_ctx_t **context,
+                  svn_stream_t *inner_stream,
+                  apr_pool_t *pool)
+{
+  svn_stream_t *outer_stream;
+
+  fnv1a_stream_baton_t *baton = apr_pcalloc(pool, sizeof(*baton));
+  baton->inner_stream = inner_stream;
+  baton->context = svn_checksum_ctx_create(svn_checksum_fnv1a_32x4, pool);
+  *context = baton->context;
+
+  outer_stream = svn_stream_create(baton, pool);
+  svn_stream_set_write(outer_stream, fnv1a_write_handler);
+
+  return outer_stream;
+}
+
+/* Set *DIGEST to the FNV checksum calculated in CONTEXT.
+ * Use SCRATCH_POOL for temporary allocations.
+ */
+static svn_error_t *
+fnv1a_checksum_finalize(apr_uint32_t *digest,
+                        svn_checksum_ctx_t *context,
+                        apr_pool_t *scratch_pool)
+{
+  svn_checksum_t *checksum;
+
+  SVN_ERR(svn_checksum_final(&checksum, context, scratch_pool));
+  SVN_ERR_ASSERT(checksum->kind == svn_checksum_fnv1a_32x4);
+  *digest =  *(apr_uint32_t *)(checksum->digest);
+
+  return SVN_NO_ERROR;
+}
+
 /* This baton is used by the representation writing streams.  It keeps
    track of the checksum information as well as the total size of the
    representation so far. */
@@ -1619,6 +1798,9 @@ struct rep_write_baton
   svn_checksum_ctx_t *md5_checksum_ctx;
   svn_checksum_ctx_t *sha1_checksum_ctx;
 
+  /* calculate a modified FNV-1a checksum of the on-disk representation */
+  svn_checksum_ctx_t *fnv1a_checksum_ctx;
+
   apr_pool_t *pool;
 
   apr_pool_t *parent_pool;
@@ -1787,7 +1969,6 @@ rep_write_cleanup(void *data)
   return APR_SUCCESS;
 }
 
-
 /* Get a rep_write_baton and store it in *WB_P for the representation
    indicated by NODEREV in filesystem FS.  Perform allocations in
    POOL.  Only appropriate for file contents, not for props or
@@ -1825,7 +2006,10 @@ rep_write_get_baton(struct rep_write_bat
                                  b->pool));
 
   b->file = file;
-  b->rep_stream = svn_stream_from_aprfile2(file, TRUE, b->pool);
+  b->rep_stream = fnv1a_wrap_stream(&b->fnv1a_checksum_ctx,
+                                    svn_stream_from_aprfile2(file, TRUE,
+                                                             b->pool),
+                                    b->pool);
 
   SVN_ERR(svn_fs_fs__get_file_offset(&b->rep_offset, file, b->pool));
 
@@ -1837,7 +2021,7 @@ rep_write_get_baton(struct rep_write_bat
   if (base_rep)
     {
       header.base_revision = base_rep->revision;
-      header.base_offset = base_rep->offset;
+      header.base_item_index = base_rep->item_index;
       header.base_length = base_rep->size;
       header.type = svn_fs_fs__rep_delta;
     }
@@ -2003,7 +2187,6 @@ rep_write_contents_close(void *baton)
   apr_off_t offset;
 
   rep = apr_pcalloc(b->parent_pool, sizeof(*rep));
-  rep->offset = b->rep_offset;
 
   /* Close our delta stream so the last bits of svndiff are written
      out. */
@@ -2040,6 +2223,9 @@ rep_write_contents_close(void *baton)
     {
       /* Write out our cosmetic end marker. */
       SVN_ERR(svn_stream_puts(b->rep_stream, "ENDREP\n"));
+      SVN_ERR(allocate_item_index(&rep->item_index, b->fs, &rep->txn_id,
+                                  SVN_INVALID_REVNUM, b->rep_offset,
+                                  b->pool));
 
       b->noderev->data_rep = rep;
     }
@@ -2051,7 +2237,23 @@ rep_write_contents_close(void *baton)
   SVN_ERR(svn_fs_fs__put_node_revision(b->fs, b->noderev->id, b->noderev,
                                        FALSE, b->pool));
   if (!old_rep)
-    SVN_ERR(store_sha1_rep_mapping(b->fs, b->noderev, b->pool));
+    {
+      svn_fs_fs__p2l_entry_t entry;
+
+      entry.offset = b->rep_offset;
+      SVN_ERR(svn_fs_fs__get_file_offset(&offset, b->file, b->pool));
+      entry.size = offset - b->rep_offset;
+      entry.type = SVN_FS_FS__ITEM_TYPE_FILE_REP;
+      entry.item.revision = SVN_INVALID_REVNUM;
+      entry.item.number = rep->item_index;
+      SVN_ERR(fnv1a_checksum_finalize(&entry.fnv1_checksum,
+                                      b->fnv1a_checksum_ctx,
+                                      b->pool));
+
+      SVN_ERR(store_sha1_rep_mapping(b->fs, b->noderev, b->pool));
+      SVN_ERR(store_p2l_index_entry(b->fs, &rep->txn_id, SVN_INVALID_REVNUM,
+                                    &entry, b->pool));
+    }
 
   SVN_ERR(svn_io_file_close(b->file, b->pool));
   SVN_ERR(unlock_proto_rev(b->fs, &rep->txn_id, b->lockcookie, b->pool));
@@ -2236,21 +2438,27 @@ write_hash_handler(void *baton,
 
 /* Write out the hash HASH as a text representation to file FILE.  In
    the process, record position, the total size of the dump and MD5 as
-   well as SHA1 in REP.   If rep sharing has been enabled and REPS_HASH
+   well as SHA1 in REP.   Add the representation of type ITEM_TYPE to
+   the indexes if necessary.  If rep sharing has been enabled and REPS_HASH
    is not NULL, it will be used in addition to the on-disk cache to find
    earlier reps with the same content.  When such existing reps can be
    found, we will truncate the one just written from the file and return
-   the existing rep.  Perform temporary allocations in POOL. */
+   the existing rep.  If FINAL_REVISION is not SVN_INVALID_REVNUM, use it
+   to determine whether to write to the proto-index files.
+   Perform temporary allocations in POOL. */
 static svn_error_t *
 write_hash_rep(representation_t *rep,
                apr_file_t *file,
                apr_hash_t *hash,
                svn_fs_t *fs,
                apr_hash_t *reps_hash,
+               int item_type,
+               svn_revnum_t final_revision,
                apr_pool_t *pool)
 {
   svn_stream_t *stream;
   struct write_hash_baton *whb;
+  svn_checksum_ctx_t *fnv1a_checksum_ctx;
   representation_t *old_rep;
   apr_off_t offset = 0;
 
@@ -2258,7 +2466,9 @@ write_hash_rep(representation_t *rep,
 
   whb = apr_pcalloc(pool, sizeof(*whb));
 
-  whb->stream = svn_stream_from_aprfile2(file, TRUE, pool);
+  whb->stream = fnv1a_wrap_stream(&fnv1a_checksum_ctx,
+                                  svn_stream_from_aprfile2(file, TRUE, pool),
+                                  pool);
   whb->size = 0;
   whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool);
   whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool);
@@ -2287,10 +2497,26 @@ write_hash_rep(representation_t *rep,
     }
   else
     {
+      svn_fs_fs__p2l_entry_t entry;
+
       /* Write out our cosmetic end marker. */
       SVN_ERR(svn_stream_puts(whb->stream, "ENDREP\n"));
 
-      rep->offset = offset;
+      SVN_ERR(allocate_item_index(&rep->item_index, fs, &rep->txn_id,
+                                  final_revision, offset, pool));
+
+      entry.offset = offset;
+      SVN_ERR(svn_fs_fs__get_file_offset(&offset, file, pool));
+      entry.size = offset - entry.offset;
+      entry.type = item_type;
+      entry.item.revision = SVN_INVALID_REVNUM;
+      entry.item.number = rep->item_index;
+      SVN_ERR(fnv1a_checksum_finalize(&entry.fnv1_checksum,
+                                      fnv1a_checksum_ctx,
+                                      pool));
+
+      SVN_ERR(store_p2l_index_entry(fs, &rep->txn_id, final_revision,
+                                    &entry, pool));
 
       /* update the representation */
       rep->size = whb->size;
@@ -2302,12 +2528,16 @@ write_hash_rep(representation_t *rep,
 
 /* Write out the hash HASH pertaining to the NODEREV in FS as a deltified
    text representation to file FILE.  In the process, record the total size
-   and the md5 digest in REP.  If rep sharing has been enabled and REPS_HASH
+   and the md5 digest in REP and add the representation of type ITEM_TYPE
+   to the indexes if necessary.  If rep sharing has been enabled and REPS_HASH
    is not NULL, it will be used in addition to the on-disk cache to find
    earlier reps with the same content.  When such existing reps can be found,
    we will truncate the one just written from the file and return the existing
-   rep.  If IS_PROPS is set, assume that we want to a props representation as
-   the base for our delta.  Perform temporary allocations in POOL. */
+   rep.  If ITEM_TYPE is IS_PROPS equals SVN_FS_FS__ITEM_TYPE_*_PROPS, assume
+   that we want to a props representation as the base for our delta.
+   If FINAL_REVISION is not SVN_INVALID_REVNUM, use it to determine whether
+   to write to the proto-index files.  Perform temporary allocations in POOL.
+ */
 static svn_error_t *
 write_hash_delta_rep(representation_t *rep,
                      apr_file_t *file,
@@ -2315,7 +2545,8 @@ write_hash_delta_rep(representation_t *r
                      svn_fs_t *fs,
                      node_revision_t *noderev,
                      apr_hash_t *reps_hash,
-                     svn_boolean_t is_props,
+                     int item_type,
+                     svn_revnum_t final_revision,
                      apr_pool_t *pool)
 {
   svn_txdelta_window_handler_t diff_wh;
@@ -2325,6 +2556,7 @@ write_hash_delta_rep(representation_t *r
   svn_stream_t *stream;
   representation_t *base_rep;
   representation_t *old_rep;
+  svn_checksum_ctx_t *fnv1a_checksum_ctx;
   svn_stream_t *source;
   svn_fs_fs__rep_header_t header = { 0 };
 
@@ -2335,6 +2567,8 @@ write_hash_delta_rep(representation_t *r
   struct write_hash_baton *whb;
   fs_fs_data_t *ffd = fs->fsap_data;
   int diff_version = ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT ? 1 : 0;
+  svn_boolean_t is_props = (item_type == SVN_FS_FS__ITEM_TYPE_FILE_PROPS)
+                        || (item_type == SVN_FS_FS__ITEM_TYPE_DIR_PROPS);
 
   /* Get the base for this delta. */
   SVN_ERR(choose_delta_base(&base_rep, fs, noderev, is_props, pool));
@@ -2346,7 +2580,7 @@ write_hash_delta_rep(representation_t *r
   if (base_rep)
     {
       header.base_revision = base_rep->revision;
-      header.base_offset = base_rep->offset;
+      header.base_item_index = base_rep->item_index;
       header.base_length = base_rep->size;
       header.type = svn_fs_fs__rep_delta;
     }
@@ -2355,7 +2589,9 @@ write_hash_delta_rep(representation_t *r
       header.type = svn_fs_fs__rep_self_delta;
     }
 
-  file_stream = svn_stream_from_aprfile2(file, TRUE, pool);
+  file_stream = fnv1a_wrap_stream(&fnv1a_checksum_ctx,
+                                  svn_stream_from_aprfile2(file, TRUE, pool),
+                                  pool);
   SVN_ERR(svn_fs_fs__write_rep_header(&header, file_stream, pool));
   SVN_ERR(svn_fs_fs__get_file_offset(&delta_start, file, pool));
 
@@ -2397,11 +2633,27 @@ write_hash_delta_rep(representation_t *r
     }
   else
     {
+      svn_fs_fs__p2l_entry_t entry;
+
       /* Write out our cosmetic end marker. */
       SVN_ERR(svn_fs_fs__get_file_offset(&rep_end, file, pool));
       SVN_ERR(svn_stream_puts(file_stream, "ENDREP\n"));
 
-      rep->offset = offset;
+      SVN_ERR(allocate_item_index(&rep->item_index, fs, &rep->txn_id,
+                                  final_revision, offset, pool));
+
+      entry.offset = offset;
+      SVN_ERR(svn_fs_fs__get_file_offset(&offset, file, pool));
+      entry.size = offset - entry.offset;
+      entry.type = item_type;
+      entry.item.revision = SVN_INVALID_REVNUM;
+      entry.item.number = rep->item_index;
+      SVN_ERR(fnv1a_checksum_finalize(&entry.fnv1_checksum,
+                                      fnv1a_checksum_ctx,
+                                      pool));
+
+      SVN_ERR(store_p2l_index_entry(fs, &rep->txn_id, final_revision,
+                                    &entry, pool));
 
       /* update the representation */
       rep->expanded_size = whb->size;
@@ -2540,8 +2792,12 @@ write_final_rev(const svn_fs_id_t **new_
   node_revision_t *noderev;
   apr_off_t my_offset;
   const svn_fs_id_t *new_id;
-  svn_fs_fs__id_part_t node_id, copy_id, rev_offset;
+  svn_fs_fs__id_part_t node_id, copy_id, rev_item;
   fs_fs_data_t *ffd = fs->fsap_data;
+  const svn_fs_fs__id_part_t *txn_id = svn_fs_fs__id_txn_id(id);
+  svn_stream_t *file_stream;
+  svn_checksum_ctx_t *fnv1a_checksum_ctx;
+
   *new_id_p = NULL;
 
   /* Check to see if this is a transaction node. */
@@ -2587,16 +2843,19 @@ write_final_rev(const svn_fs_id_t **new_
           /* Write out the contents of this directory as a text rep. */
           SVN_ERR(unparse_dir_entries(&str_entries, entries, pool));
 
-          reset_txn_in_rep(noderev->data_rep);
           noderev->data_rep->revision = rev;
 
           if (ffd->deltify_directories)
             SVN_ERR(write_hash_delta_rep(noderev->data_rep, file,
                                          str_entries, fs, noderev, NULL,
-                                         FALSE, pool));
+                                         SVN_FS_FS__ITEM_TYPE_DIR_REP,
+                                         rev, pool));
           else
             SVN_ERR(write_hash_rep(noderev->data_rep, file, str_entries,
-                                   fs, NULL, pool));
+                                   fs, NULL, SVN_FS_FS__ITEM_TYPE_DIR_REP,
+                                   rev, pool));
+
+          reset_txn_in_rep(noderev->data_rep);
         }
     }
   else
@@ -2610,13 +2869,16 @@ write_final_rev(const svn_fs_id_t **new_
           reset_txn_in_rep(noderev->data_rep);
           noderev->data_rep->revision = rev;
 
-          /* See issue 3845.  Some unknown mechanism caused the
-             protorev file to get truncated, so check for that
-             here.  */
-          if (noderev->data_rep->offset + noderev->data_rep->size
-              > initial_offset)
-            return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
-                                    _("Truncated protorev file detected"));
+          if (!svn_fs_fs__use_log_addressing(fs, rev))
+            {
+              /* See issue 3845.  Some unknown mechanism caused the
+                 protorev file to get truncated, so check for that
+                 here.  */
+              if (noderev->data_rep->item_index + noderev->data_rep->size
+                  > initial_offset)
+                return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
+                                        _("Truncated protorev file detected"));
+            }
         }
     }
 
@@ -2624,18 +2886,22 @@ write_final_rev(const svn_fs_id_t **new_
   if (noderev->prop_rep && is_txn_rep(noderev->prop_rep))
     {
       apr_hash_t *proplist;
+      int item_type = noderev->kind == svn_node_dir
+                    ? SVN_FS_FS__ITEM_TYPE_DIR_PROPS
+                    : SVN_FS_FS__ITEM_TYPE_FILE_PROPS;
       SVN_ERR(svn_fs_fs__get_proplist(&proplist, fs, noderev, pool));
 
-      reset_txn_in_rep(noderev->prop_rep);
       noderev->prop_rep->revision = rev;
 
       if (ffd->deltify_properties)
         SVN_ERR(write_hash_delta_rep(noderev->prop_rep, file,
                                      proplist, fs, noderev, reps_hash,
-                                     TRUE, pool));
+                                     item_type, rev, pool));
       else
         SVN_ERR(write_hash_rep(noderev->prop_rep, file, proplist,
-                               fs, reps_hash, pool));
+                               fs, reps_hash, item_type, rev, pool));
+
+      reset_txn_in_rep(noderev->prop_rep);
     }
 
   /* Convert our temporary ID into a permanent revision one. */
@@ -2647,11 +2913,21 @@ write_final_rev(const svn_fs_id_t **new_
   if (noderev->copyroot_rev == SVN_INVALID_REVNUM)
     noderev->copyroot_rev = rev;
 
+  /* root nodes have a fixed ID in log addressing mode */
   SVN_ERR(svn_fs_fs__get_file_offset(&my_offset, file, pool));
-  rev_offset.number = my_offset;
+  if (svn_fs_fs__use_log_addressing(fs, rev) && at_root)
+    {
+      /* reference the root noderev from the log-to-phys index */
+      rev_item.number = SVN_FS_FS__ITEM_INDEX_ROOT_NODE;
+      SVN_ERR(store_l2p_index_entry(fs, txn_id, rev, my_offset,
+                                    rev_item.number, pool));
+    }
+  else
+    SVN_ERR(allocate_item_index(&rev_item.number, fs, txn_id, rev,
+                                my_offset, pool));
 
-  rev_offset.revision = rev;
-  new_id = svn_fs_fs__id_rev_create(&node_id, &copy_id, &rev_offset, pool);
+  rev_item.revision = rev;
+  new_id = svn_fs_fs__id_rev_create(&node_id, &copy_id, &rev_item, pool);
 
   noderev->id = new_id;
 
@@ -2697,30 +2973,54 @@ write_final_rev(const svn_fs_id_t **new_
   if (at_root)
     SVN_ERR(validate_root_noderev(fs, noderev, rev, pool));
 
-  SVN_ERR(svn_fs_fs__write_noderev(svn_stream_from_aprfile2(file, TRUE, pool),
-                                   noderev, ffd->format,
+  file_stream = fnv1a_wrap_stream(&fnv1a_checksum_ctx,
+                                  svn_stream_from_aprfile2(file, TRUE, pool),
+                                  pool);
+  SVN_ERR(svn_fs_fs__write_noderev(file_stream, noderev, ffd->format,
                                    svn_fs_fs__fs_supports_mergeinfo(fs),
                                    pool));
 
+  /* reference the root noderev from the log-to-phys index */
+  if (svn_fs_fs__use_log_addressing(fs, rev))
+    {
+      svn_fs_fs__p2l_entry_t entry;
+      rev_item.revision = SVN_INVALID_REVNUM;
+
+      entry.offset = my_offset;
+      SVN_ERR(svn_fs_fs__get_file_offset(&my_offset, file, pool));
+      entry.size = my_offset - entry.offset;
+      entry.type = SVN_FS_FS__ITEM_TYPE_NODEREV;
+      entry.item = rev_item;
+      SVN_ERR(fnv1a_checksum_finalize(&entry.fnv1_checksum,
+                                      fnv1a_checksum_ctx,
+                                      pool));
+
+      SVN_ERR(store_p2l_index_entry(fs, txn_id, rev, &entry, pool));
+    }
+
   /* Return our ID that references the revision file. */
   *new_id_p = noderev->id;
 
   return SVN_NO_ERROR;
 }
 
-/* Write the changed path info CHANGED_PATHS to the permanent rev-file FILE
-   representing NEW_REV in filesystem FS.  *OFFSET_P is set the to offset in
-   the file of the beginning of this information.
+/* Write the changed path info CHANGED_PATHS from transaction TXN_ID to the
+   permanent rev-file FILE representing NEW_REV in filesystem FS.  *OFFSET_P
+   is set the to offset in the file of the beginning of this information.
+   NEW_REV is the revision currently being committed.
    Perform temporary allocations in POOL. */
 static svn_error_t *
 write_final_changed_path_info(apr_off_t *offset_p,
                               apr_file_t *file,
                               svn_fs_t *fs,
+                              const svn_fs_fs__id_part_t *txn_id,
                               apr_hash_t *changed_paths,
                               svn_revnum_t new_rev,
                               apr_pool_t *pool)
 {
   apr_off_t offset;
+  svn_stream_t *stream;
+  svn_checksum_ctx_t *fnv1a_checksum_ctx;
   apr_hash_index_t *hi;
 
   SVN_ERR(svn_fs_fs__get_file_offset(&offset, file, pool));
@@ -2740,11 +3040,34 @@ write_final_changed_path_info(apr_off_t 
         }
     }
 
-  SVN_ERR(svn_fs_fs__write_changes(svn_stream_from_aprfile2(file, TRUE, pool),
-                                   fs, changed_paths, TRUE, pool));
+  /* write to target file & calculate checksum */
+  stream = fnv1a_wrap_stream(&fnv1a_checksum_ctx,
+                             svn_stream_from_aprfile2(file, TRUE, pool),
+                             pool);
+  SVN_ERR(svn_fs_fs__write_changes(stream, fs, changed_paths, TRUE, pool));
 
   *offset_p = offset;
 
+  /* reference changes from the indexes */
+  if (svn_fs_fs__use_log_addressing(fs, new_rev))
+    {
+      svn_fs_fs__p2l_entry_t entry;
+
+      entry.offset = offset;
+      SVN_ERR(svn_fs_fs__get_file_offset(&offset, file, pool));
+      entry.size = offset - entry.offset;
+      entry.type = SVN_FS_FS__ITEM_TYPE_CHANGES;
+      entry.item.revision = SVN_INVALID_REVNUM;
+      entry.item.number = SVN_FS_FS__ITEM_INDEX_CHANGES;
+      SVN_ERR(fnv1a_checksum_finalize(&entry.fnv1_checksum,
+                                      fnv1a_checksum_ctx,
+                                      pool));
+
+      SVN_ERR(store_p2l_index_entry(fs, txn_id, new_rev, &entry, pool));
+      SVN_ERR(store_l2p_index_entry(fs, txn_id, new_rev, entry.offset,
+                                    SVN_FS_FS__ITEM_INDEX_CHANGES, pool));
+    }
+
   return SVN_NO_ERROR;
 }
 
@@ -3061,6 +3384,242 @@ verify_moves(svn_fs_t *fs,
   return SVN_NO_ERROR;
 }
 
+/* Return TRUE, if transaction TXN_ID in FS definitively uses logical
+ * addressing mode.  Use POOL for temporary allocations.
+ */
+static svn_boolean_t
+using_log_addressing(svn_fs_t *fs,
+                     const svn_fs_fs__id_part_t *txn_id,
+                     apr_pool_t *pool)
+{
+  /* As long as we don't write new data representations, we won't allocate
+     IDs and there is no difference between log & phys mode.
+
+     After the first ID got allocated, it is logical mode and the proto-
+     index file does exist.
+   */
+  svn_node_kind_t kind;
+  const char *path = svn_fs_fs__path_l2p_proto_index(fs, txn_id, pool);
+
+  svn_error_t *err = svn_io_check_path(path, &kind, pool);
+  if (err)
+    {
+      /* We couldn't check for the presence of the index file.
+
+         So, we probably won't be able to access it during later stages
+         of the commit.
+       */
+      svn_error_clear(err);
+      return FALSE;
+    }
+
+  return kind == svn_node_file;
+}
+
+/* Return TRUE, if the file with FILENAME contains a node revision.
+ */
+static svn_boolean_t
+is_noderev_file(const char *filename)
+{
+  apr_size_t dot_count = 0;
+
+  /* all interesting files start with "node." */
+  if (strncmp(filename, PATH_PREFIX_NODE, strlen(PATH_PREFIX_NODE)))
+    return FALSE;
+
+  for (; *filename; ++filename)
+    if (*filename == '.')
+      ++dot_count;
+
+  return dot_count == 2;
+}
+
+/* Determine the checksum for the SIZE bytes of data starting at START
+ * in FILE and return the result in *FNV1_CHECKSUM.
+ * Use POOL for tempoary allocations.
+ */
+static svn_error_t *
+fnv1a_checksum_on_file_range(apr_uint32_t *fnv1_checksum,
+                             apr_file_t *file,
+                             apr_off_t start,
+                             apr_off_t size,
+                             apr_pool_t *pool)
+{
+  char buffer[4096];
+
+  svn_checksum_ctx_t *checksum_ctx
+    = svn_checksum_ctx_create(svn_checksum_fnv1a_32x4, pool);
+
+  SVN_ERR(svn_io_file_seek(file, APR_SET, &start, pool));
+  while (size > 0)
+    {
+      apr_size_t to_read = MIN(size, sizeof(buffer));
+
+      SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, &to_read,
+                                     NULL, pool));
+      SVN_ERR(svn_checksum_update(checksum_ctx, buffer, to_read));
+      size -= to_read;
+    }
+  SVN_ERR(fnv1a_checksum_finalize(fnv1_checksum, checksum_ctx, pool));
+
+  return SVN_NO_ERROR;
+}
+
+/* qsort()-compatible comparision function sorting svn_fs_fs__p2l_entry_t
+ * by offset.
+ */
+static int
+compare_sort_p2l_entry(const void *a,
+                       const void *b)
+{
+  apr_off_t lhs = ((const svn_fs_fs__p2l_entry_t *)a)->offset;
+  apr_off_t rhs = ((const svn_fs_fs__p2l_entry_t *)b)->offset;
+
+  return lhs < rhs ? -1 : rhs == lhs ? 0 : 1;
+}
+
+
+/* Upgrade the transaction TXN_ID in FS from physical addressing mode
+ * to logical addressing mode.  FINAL_REVISION is the revision that this
+ * txn is being committed to.  Use POOL for temporary allocations.
+ */
+static svn_error_t *
+upgrade_transaction(svn_fs_t *fs,
+                    const svn_fs_fs__id_part_t *txn_id,
+                    svn_revnum_t final_revision,
+                    apr_file_t *proto_file,
+                    apr_pool_t *pool)
+{
+  fs_fs_data_t *ffd = fs->fsap_data;
+  apr_hash_t *dirents;
+  int i;
+  apr_hash_index_t* hi;
+
+  /* we allocate large temporary data and want to release it asap */
+
+  apr_pool_t *subpool = svn_pool_create(pool);
+  apr_pool_t *iterpool = svn_pool_create(pool);
+
+  apr_hash_t *id_map = apr_hash_make(subpool);
+  const char *txn_dir = svn_fs_fs__path_txn_dir(fs, txn_id, subpool);
+  apr_array_header_t *p2l_entries
+    = apr_array_make(subpool, 16, sizeof(svn_fs_fs__p2l_entry_t));
+
+  /* scan the txn directory for noderev files and patch them up */
+
+  SVN_ERR(svn_io_get_dirents3(&dirents, txn_dir, TRUE, subpool, iterpool));
+  for (hi = apr_hash_first(subpool, dirents); hi; hi = apr_hash_next(hi))
+    {
+      apr_file_t *file;
+      const char *filename;
+      node_revision_t *noderev;
+      svn_stream_t *stream;
+      const char *name;
+      apr_uint64_t *old_index, *item_index, *new_index;
+
+      svn_pool_clear(iterpool);
+
+      /* We are only interested in file data reps of this txns.
+         Older IDs remain valid because they are already committed.
+         Other IDs (noderevs and their usage in directories) will only be
+         assigned later anyways. */
+
+      name = svn__apr_hash_index_key(hi);
+      if (!is_noderev_file(name))
+        continue;
+
+      filename = svn_dirent_join(txn_dir, name, iterpool);
+      SVN_ERR(svn_io_file_open(&file, filename,
+                               APR_READ | APR_WRITE | APR_BUFFERED,
+                               APR_OS_DEFAULT,
+                               iterpool));
+      stream = svn_stream_from_aprfile2(file, TRUE, iterpool);
+      SVN_ERR(svn_fs_fs__read_noderev(&noderev, stream, iterpool));
+      if (   noderev->data_rep == NULL
+          || noderev->data_rep->revision != SVN_INVALID_REVNUM
+          || noderev->kind != svn_node_file)
+        continue;
+
+      /* We need to assign an id.
+         We might already have one because of rep sharing. */
+
+      item_index = &noderev->data_rep->item_index;
+      new_index = apr_hash_get(id_map, item_index, sizeof(*item_index));
+
+      if (new_index)
+        {
+          *item_index = *new_index;
+        }
+      else
+        {
+          svn_fs_fs__rep_header_t *header;
+          svn_fs_fs__p2l_entry_t *entry;
+
+          /* assign a logical ID and write the L2P proto-index */
+
+          old_index = apr_pmemdup(subpool, item_index, sizeof(*item_index));
+          SVN_ERR(allocate_item_index(item_index, fs, txn_id, final_revision,
+                                      *old_index, iterpool));
+
+          new_index = apr_pmemdup(subpool, item_index, sizeof(*item_index));
+          apr_hash_set(id_map, old_index, sizeof(*old_index), new_index);
+
+          /* we need to know the length of the representation header
+             because it is not accounted for by the representation length */
+
+          entry = apr_array_push(p2l_entries);
+          entry->offset = *old_index;
+          SVN_ERR(svn_io_file_seek(proto_file, APR_SET, &entry->offset,
+                                   iterpool));
+          SVN_ERR(svn_fs_fs__read_rep_header(&header,
+                    svn_stream_from_aprfile2(proto_file, TRUE, iterpool),
+                    iterpool));
+
+          /* Create the corresponding entry for the P2L proto-index.
+
+             We need to write that proto-index in strict offset order but
+             we have no control over the order in which we traverse the
+             data reps.   Thus, we collect the entries in an array. */
+
+          entry->size = noderev->data_rep->size + header->header_size + 7;
+                        /* 7 for the "ENDREP\n" */
+          entry->type = SVN_FS_FS__ITEM_TYPE_FILE_REP;
+          entry->item.revision = SVN_INVALID_REVNUM;
+          entry->item.number = *new_index;
+          SVN_ERR(fnv1a_checksum_on_file_range(&entry->fnv1_checksum,
+                                               proto_file,
+                                               entry->offset, entry->size,
+                                               iterpool));
+        }
+
+      /* write the updated noderev to disk */
+
+      SVN_ERR(svn_io_file_trunc(file, 0, iterpool));
+      SVN_ERR(svn_fs_fs__write_noderev(stream, noderev, ffd->format,
+                                       TRUE, iterpool));
+    }
+
+  /* Finally, write all P2L proto-index entries ordered by item offset. */
+
+  qsort(p2l_entries->elts, p2l_entries->nelts, p2l_entries->elt_size,
+        compare_sort_p2l_entry);
+  for (i = 0; i < p2l_entries->nelts; ++i)
+    {
+      svn_fs_fs__p2l_entry_t *entry;
+
+      svn_pool_clear(iterpool);
+
+      entry = &APR_ARRAY_IDX(p2l_entries, i, svn_fs_fs__p2l_entry_t);
+      SVN_ERR(store_p2l_index_entry(fs, txn_id, final_revision,
+                                    entry, iterpool));
+    }
+
+  svn_pool_clear(iterpool);
+  svn_pool_clear(subpool);
+
+  return SVN_NO_ERROR;
+}
+
 /* Baton used for commit_body below. */
 struct commit_baton {
   svn_revnum_t *new_rev_p;
@@ -3093,7 +3652,6 @@ commit_body(void *baton, apr_pool_t *poo
   apr_array_header_t *txnprop_list;
   svn_prop_t prop;
   const svn_fs_fs__id_part_t *txn_id = svn_fs_fs__txn_get_id(cb->txn);
-  svn_stringbuf_t *trailer;
   apr_hash_t *changed_paths;
 
   /* Get the current youngest revision. */
@@ -3134,6 +3692,18 @@ commit_body(void *baton, apr_pool_t *poo
                                  cb->fs, txn_id, pool));
   SVN_ERR(svn_fs_fs__get_file_offset(&initial_offset, proto_file, pool));
 
+  /* Make sure that we don't try to commit an old txn that used physical
+     addressing but will be committed into the revision range that requires
+     logical addressing.
+     */
+  if (svn_fs_fs__use_log_addressing(cb->fs, new_rev)
+      && !svn_fs_fs__use_log_addressing(cb->fs, txn_id->revision)
+      && !using_log_addressing(cb->fs, txn_id, pool))
+    {
+      SVN_ERR(upgrade_transaction(cb->fs, txn_id, new_rev, proto_file, pool));
+      SVN_ERR(svn_io_file_seek(proto_file, APR_SET, &initial_offset, pool));
+    }
+
   /* Write out all the node-revisions and directory contents. */
   root_id = svn_fs_fs__id_txn_create_root(txn_id, pool);
   SVN_ERR(write_final_rev(&new_root_id, proto_file, new_rev, cb->fs, root_id,
@@ -3143,16 +3713,21 @@ commit_body(void *baton, apr_pool_t *poo
 
   /* Write the changed-path information. */
   SVN_ERR(write_final_changed_path_info(&changed_path_offset, proto_file,
-                                        cb->fs, changed_paths, new_rev,
-                                        pool));
+                                        cb->fs, txn_id, changed_paths,
+                                        new_rev, pool));
 
-  /* Write the final line. */
-  trailer = svn_fs_fs__unparse_revision_trailer
-               (svn_fs_fs__id_offset(new_root_id),
-                changed_path_offset,
-                pool);
-  SVN_ERR(svn_io_file_write_full(proto_file, trailer->data, trailer->len,
-                                 NULL, pool));
+  if (!svn_fs_fs__use_log_addressing(cb->fs, new_rev))
+    {
+      /* Write the final line. */
+
+      svn_stringbuf_t *trailer
+        = svn_fs_fs__unparse_revision_trailer
+                  ((apr_off_t)svn_fs_fs__id_item(new_root_id),
+                   changed_path_offset,
+                   pool);
+      SVN_ERR(svn_io_file_write_full(proto_file, trailer->data, trailer->len,
+                                     NULL, pool));
+    }
 
   SVN_ERR(svn_io_file_flush_to_disk(proto_file, pool));
   SVN_ERR(svn_io_file_close(proto_file, pool));
@@ -3222,6 +3797,20 @@ commit_body(void *baton, apr_pool_t *poo
         }
     }
 
+  if (svn_fs_fs__use_log_addressing(cb->fs, new_rev))
+    {
+      /* Convert the index files from the proto format into their form
+         in their final location */
+      SVN_ERR(svn_fs_fs__l2p_index_create(cb->fs,
+                      svn_fs_fs__path_l2p_index(cb->fs, new_rev, FALSE, pool),
+                      svn_fs_fs__path_l2p_proto_index(cb->fs, txn_id, pool),
+                      new_rev, pool));
+      SVN_ERR(svn_fs_fs__p2l_index_create(cb->fs,
+                      svn_fs_fs__path_p2l_index(cb->fs, new_rev, FALSE, pool),
+                      svn_fs_fs__path_p2l_proto_index(cb->fs, txn_id, pool),
+                      new_rev, pool));
+    }
+
   /* Move the finished rev file into place.
 
      ### This "breaks" the transaction by removing the protorev file

Modified: subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/tree.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/tree.c?rev=1546928&r1=1546927&r2=1546928&view=diff
==============================================================================
--- subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/tree.c (original)
+++ subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/tree.c Mon Dec  2 08:48:00 2013
@@ -2248,7 +2248,10 @@ fs_dir_optimal_order(apr_array_header_t 
                      apr_hash_t *entries,
                      apr_pool_t *pool)
 {
-  *ordered_p = svn_fs_fs__order_dir_entries(root->fs, entries, pool);
+  *ordered_p
+    = svn_fs_fs__order_dir_entries(root->fs, entries,
+                                   root->rev,
+                                   pool);
 
   return SVN_NO_ERROR;
 }

Modified: subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/util.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/util.c?rev=1546928&r1=1546927&r2=1546928&view=diff
==============================================================================
--- subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/util.c (original)
+++ subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/util.c Mon Dec  2 08:48:00 2013
@@ -131,17 +131,51 @@ svn_fs_fs__path_rev(svn_fs_t *fs, svn_re
                               apr_psprintf(pool, "%ld", rev), SVN_VA_NULL);
 }
 
+/* Set *PATH to the path of REV in FS with PACKED selecting whether the
+   (potential) pack file or single revision file name is returned.
+   Allocate *PATH in POOL.
+*/
+static const char *
+path_rev_absolute_internal(svn_fs_t *fs,
+                           svn_revnum_t rev,
+                           svn_boolean_t packed,
+                           apr_pool_t *pool)
+{
+  return packed
+       ? svn_fs_fs__path_rev_packed(fs, rev, PATH_PACKED, pool)
+       : svn_fs_fs__path_rev(fs, rev, pool);
+}
+
+const char *
+svn_fs_fs__path_l2p_index(svn_fs_t *fs,
+                          svn_revnum_t rev,
+                          svn_boolean_t packed,
+                          apr_pool_t *pool)
+{
+  return apr_psprintf(pool, "%s" PATH_EXT_L2P_INDEX,
+                      path_rev_absolute_internal(fs, rev, packed, pool));
+}
+
+const char *
+svn_fs_fs__path_p2l_index(svn_fs_t *fs,
+                          svn_revnum_t rev,
+                          svn_boolean_t packed,
+                          apr_pool_t *pool)
+{
+  return apr_psprintf(pool, "%s" PATH_EXT_P2L_INDEX,
+                      path_rev_absolute_internal(fs, rev, packed, pool));
+}
+
 const char *
 svn_fs_fs__path_rev_absolute(svn_fs_t *fs,
                              svn_revnum_t rev,
                              apr_pool_t *pool)
 {
   fs_fs_data_t *ffd = fs->fsap_data;
+  svn_boolean_t is_packed = ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT
+                         && svn_fs_fs__is_packed_rev(fs, rev);
 
-  return (   ffd->format < SVN_FS_FS__MIN_PACKED_FORMAT
-          || ! svn_fs_fs__is_packed_rev(fs, rev))
-       ? svn_fs_fs__path_rev(fs, rev, pool)
-       : svn_fs_fs__path_rev_packed(fs, rev, PATH_PACKED, pool);
+  return path_rev_absolute_internal(fs, rev, is_packed, pool);
 }
 
 const char *
@@ -214,6 +248,33 @@ svn_fs_fs__path_txn_dir(svn_fs_t *fs,
                               SVN_VA_NULL);
 }
 
+const char*
+svn_fs_fs__path_l2p_proto_index(svn_fs_t *fs,
+                                const svn_fs_fs__id_part_t *txn_id,
+                                apr_pool_t *pool)
+{
+  return svn_dirent_join(svn_fs_fs__path_txn_dir(fs, txn_id, pool),
+                         PATH_INDEX PATH_EXT_L2P_INDEX, pool);
+}
+
+const char*
+svn_fs_fs__path_p2l_proto_index(svn_fs_t *fs,
+                                const svn_fs_fs__id_part_t *txn_id,
+                                apr_pool_t *pool)
+{
+  return svn_dirent_join(svn_fs_fs__path_txn_dir(fs, txn_id, pool),
+                         PATH_INDEX PATH_EXT_P2L_INDEX, pool);
+}
+
+const char *
+svn_fs_fs__path_txn_item_index(svn_fs_t *fs,
+                               const svn_fs_fs__id_part_t *txn_id,
+                               apr_pool_t *pool)
+{
+  return svn_dirent_join(svn_fs_fs__path_txn_dir(fs, txn_id, pool),
+                         PATH_TXN_ITEM_INDEX, pool);
+}
+
 const char *
 svn_fs_fs__path_txn_proto_rev(svn_fs_t *fs,
                               const svn_fs_fs__id_part_t *txn_id,
@@ -230,6 +291,7 @@ svn_fs_fs__path_txn_proto_rev(svn_fs_t *
                            PATH_REV, pool);
 }
 
+
 const char *
 svn_fs_fs__path_txn_proto_rev_lock(svn_fs_t *fs,
                                    const svn_fs_fs__id_part_t *txn_id,
@@ -567,77 +629,13 @@ svn_fs_fs__move_into_place(const char *o
   return SVN_NO_ERROR;
 }
 
-svn_error_t *
-svn_fs_fs__open_pack_or_rev_file(apr_file_t **file,
-                                 svn_fs_t *fs,
-                                 svn_revnum_t rev,
-                                 apr_pool_t *pool)
+svn_boolean_t
+svn_fs_fs__use_log_addressing(svn_fs_t *fs,
+                              svn_revnum_t rev)
 {
   fs_fs_data_t *ffd = fs->fsap_data;
-  svn_error_t *err;
-  svn_boolean_t retry = FALSE;
-
-  do
-    {
-      const char *path = svn_fs_fs__path_rev_absolute(fs, rev, pool);
-
-      /* open the revision file in buffered r/o mode */
-      err = svn_io_file_open(file, path,
-                            APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool);
-      if (err && APR_STATUS_IS_ENOENT(err->apr_err))
-        {
-          if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
-            {
-              /* Could not open the file. This may happen if the
-               * file once existed but got packed later. */
-              svn_error_clear(err);
-
-              /* if that was our 2nd attempt, leave it at that. */
-              if (retry)
-                return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
-                                         _("No such revision %ld"), rev);
-
-              /* We failed for the first time. Refresh cache & retry. */
-              SVN_ERR(svn_fs_fs__update_min_unpacked_rev(fs, pool));
-
-              retry = TRUE;
-            }
-          else
-            {
-              svn_error_clear(err);
-              return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
-                                       _("No such revision %ld"), rev);
-            }
-        }
-      else
-        {
-          retry = FALSE;
-        }
-    }
-  while (retry);
-
-  return svn_error_trace(err);
-}
-
-svn_error_t *
-svn_fs_fs__item_offset(apr_off_t *absolute_position,
-                       svn_fs_t *fs,
-                       svn_revnum_t rev,
-                       apr_off_t offset,
-                       apr_pool_t *pool)
-{
-  if (svn_fs_fs__is_packed_rev(fs, rev))
-    {
-      apr_off_t rev_offset;
-      SVN_ERR(svn_fs_fs__get_packed_offset(&rev_offset, fs, rev, pool));
-      *absolute_position = rev_offset + offset;
-    }
-  else
-    {
-      *absolute_position = offset;
-    }
-
-  return SVN_NO_ERROR;
+  return ffd->min_log_addressing_rev != SVN_INVALID_REVNUM
+      && ffd->min_log_addressing_rev <= rev;
 }
 
 svn_boolean_t

Modified: subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/util.h
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/util.h?rev=1546928&r1=1546927&r2=1546928&view=diff
==============================================================================
--- subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/util.h (original)
+++ subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/util.h Mon Dec  2 08:48:00 2013
@@ -169,6 +169,24 @@ svn_fs_fs__path_revprops(svn_fs_t *fs,
                          svn_revnum_t rev,
                          apr_pool_t *pool);
 
+/* Return the path of the file containing the log-to-phys index for the
+ * file containing revision REV in FS. The result will be allocated in POOL.
+ */
+const char *
+svn_fs_fs__path_l2p_index(svn_fs_t *fs,
+                          svn_revnum_t rev,
+                          svn_boolean_t packed,
+                          apr_pool_t *pool);
+
+/* Return the path of the file containing the phys-to-log index for the
+ * file containing revision REV in FS. The result will be allocated in POOL.
+ */
+const char *
+svn_fs_fs__path_p2l_index(svn_fs_t *fs,
+                          svn_revnum_t rev,
+                          svn_boolean_t packed,
+                          apr_pool_t *pool);
+
 /* Return the path of the file storing the oldest non-packed revision in FS.
  * The result will be allocated in POOL.
  */
@@ -225,6 +243,33 @@ svn_fs_fs__path_txn_node_children(svn_fs
                                   const svn_fs_id_t *id,
                                   apr_pool_t *pool);
 
+/* Return the path of the file containing the log-to-phys index for
+ * the transaction identified by TXN_ID in FS.
+ * The result will be allocated in POOL.
+ */
+const char*
+svn_fs_fs__path_l2p_proto_index(svn_fs_t *fs,
+                                const svn_fs_fs__id_part_t *txn_id,
+                                apr_pool_t *pool);
+
+/* Return the path of the file containing the phys-to-log index for
+ * the transaction identified by TXN_ID in FS.
+ * The result will be allocated in POOL.
+ */
+const char*
+svn_fs_fs__path_p2l_proto_index(svn_fs_t *fs,
+                                const svn_fs_fs__id_part_t *txn_id,
+                                apr_pool_t *pool);
+
+/* Return the path of the file containing item_index counter for
+ * the transaction identified by TXN_ID in FS.
+ * The result will be allocated in POOL.
+ */
+const char *
+svn_fs_fs__path_txn_item_index(svn_fs_t *fs,
+                               const svn_fs_fs__id_part_t *txn_id,
+                               apr_pool_t *pool);
+
 /* Return the path of the file containing the node origins cachs for
  * the given NODE_ID in FS.  The result will be allocated in POOL.
  */
@@ -335,34 +380,13 @@ svn_fs_fs__move_into_place(const char *o
                            const char *perms_reference,
                            apr_pool_t *pool);
 
-/* Open the correct revision file for REV.  If the filesystem FS has
-   been packed, *FILE will be set to the packed file; otherwise, set *FILE
-   to the revision file for REV.  Return SVN_ERR_FS_NO_SUCH_REVISION if the
-   file doesn't exist.
-
-   TODO: Consider returning an indication of whether this is a packed rev
-         file, so the caller need not rely on is_packed_rev() which in turn
-         relies on the cached FFD->min_unpacked_rev value not having changed
-         since the rev file was opened.
-
-   Use POOL for allocations. */
-svn_error_t *
-svn_fs_fs__open_pack_or_rev_file(apr_file_t **file,
-                                 svn_fs_t *fs,
-                                 svn_revnum_t rev,
-                                 apr_pool_t *pool);
-
-/* For OFFSET within REV in FS, return the position in the respective rev
-   or pack file in *ABSOLUTE_POSITION.  Use POOL for allocations. */
-svn_error_t *
-svn_fs_fs__item_offset(apr_off_t *absolute_position,
-                       svn_fs_t *fs,
-                       svn_revnum_t rev,
-                       apr_off_t offset,
-                       apr_pool_t *pool);
+/* Return TRUE, iff revision REV in FS requires logical addressing. */
+svn_boolean_t
+svn_fs_fs__use_log_addressing(svn_fs_t *fs,
+                              svn_revnum_t rev);
 
 /* Return TRUE if FS's format supports moves to be recorded natively. */
 svn_boolean_t
 svn_fs_fs__supports_move(svn_fs_t *fs);
 
-#endif
+#endif
\ No newline at end of file

Modified: subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/verify.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/verify.c?rev=1546928&r1=1546927&r2=1546928&view=diff
==============================================================================
--- subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/verify.c (original)
+++ subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/verify.c Mon Dec  2 08:48:00 2013
@@ -20,12 +20,17 @@
  * ====================================================================
  */
 
+#include "svn_sorts.h"
+#include "svn_checksum.h"
+#include "private/svn_subr_private.h"
+
 #include "verify.h"
 #include "fs_fs.h"
 
 #include "cached_data.h"
 #include "rep-cache.h"
 #include "util.h"
+#include "index.h"
 
 #include "../libsvn_fs/fs-loader.h"
 
@@ -154,6 +159,517 @@ verify_rep_cache(svn_fs_t *fs,
   return SVN_NO_ERROR;
 }
 
+/* Verify that for all log-to-phys index entries for revisions START to
+ * START + COUNT-1 in FS there is a consistent entry in the phys-to-log
+ * index.  If given, invoke CANCEL_FUNC with CANCEL_BATON at regular
+ * intervals. Use POOL for allocations.
+ */
+static svn_error_t *
+compare_l2p_to_p2l_index(svn_fs_t *fs,
+                         svn_revnum_t start,
+                         svn_revnum_t count,
+                         svn_cancel_func_t cancel_func,
+                         void *cancel_baton,
+                         apr_pool_t *pool)
+{
+  svn_revnum_t i;
+  apr_pool_t *iterpool = svn_pool_create(pool);
+  apr_array_header_t *max_ids;
+
+  /* common file access structure */
+  svn_fs_fs__revision_file_t rev_file;
+  svn_fs_fs__init_revision_file(&rev_file, fs, start, pool);
+
+  /* determine the range of items to check for each revision */
+  SVN_ERR(svn_fs_fs__l2p_get_max_ids(&max_ids, fs, start, count, pool));
+
+  /* check all items in all revisions if the given range */
+  for (i = 0; i < max_ids->nelts; ++i)
+    {
+      apr_uint64_t k;
+      apr_uint64_t max_id = APR_ARRAY_IDX(max_ids, i, apr_uint64_t);
+      svn_revnum_t revision = start + i;
+
+      for (k = 0; k < max_id; ++k)
+        {
+          apr_off_t offset;
+          svn_fs_fs__p2l_entry_t *p2l_entry;
+          svn_pool_clear(iterpool);
+
+          /* get L2P entry.  Ignore unused entries. */
+          SVN_ERR(svn_fs_fs__item_offset(&offset, fs, &rev_file, revision,
+                                         NULL, k, iterpool));
+          if (offset == -1)
+            continue;
+
+          /* find the corresponding P2L entry */
+          SVN_ERR(svn_fs_fs__p2l_entry_lookup(&p2l_entry, fs, &rev_file,
+                                              revision, offset, iterpool));
+
+          if (p2l_entry == NULL)
+            return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT,
+                                     NULL,
+                                     _("p2l index entry not found for "
+                                       "PHYS %s returned by "
+                                       "l2p index for LOG r%ld:i%ld"),
+                                     apr_off_t_toa(pool, offset),
+                                     revision, (long)k);
+
+          if (   p2l_entry->item.number != k
+              || p2l_entry->item.revision != revision)
+            return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT,
+                                     NULL,
+                                     _("p2l index info LOG r%ld:i%ld"
+                                       " does not match "
+                                       "l2p index for LOG r%ld:i%ld"),
+                                     p2l_entry->item.revision,
+                                     (long)p2l_entry->item.number,
+                                     revision, (long)k);
+        }
+
+      if (cancel_func)
+        SVN_ERR(cancel_func(cancel_baton));
+    }
+
+  svn_pool_destroy(iterpool);
+
+  return SVN_NO_ERROR;
+}
+
+/* Verify that for all phys-to-log index entries for revisions START to
+ * START + COUNT-1 in FS there is a consistent entry in the log-to-phys
+ * index.  If given, invoke CANCEL_FUNC with CANCEL_BATON at regular
+ * intervals. Use POOL for allocations.
+ *
+ * Please note that we can only check on pack / rev file granularity and
+ * must only be called for a single rev / pack file.
+ */
+static svn_error_t *
+compare_p2l_to_l2p_index(svn_fs_t *fs,
+                         svn_revnum_t start,
+                         svn_revnum_t count,
+                         svn_cancel_func_t cancel_func,
+                         void *cancel_baton,
+                         apr_pool_t *pool)
+{
+  apr_pool_t *iterpool = svn_pool_create(pool);
+  apr_off_t max_offset;
+  apr_off_t offset = 0;
+
+  /* common file access structure */
+  svn_fs_fs__revision_file_t rev_file;
+  svn_fs_fs__init_revision_file(&rev_file, fs, start, pool);
+
+  /* get the size of the rev / pack file as covered by the P2L index */
+  SVN_ERR(svn_fs_fs__p2l_get_max_offset(&max_offset, fs, &rev_file, start,
+                                        pool));
+
+  /* for all offsets in the file, get the P2L index entries and check
+     them against the L2P index */
+  for (offset = 0; offset < max_offset; )
+    {
+      apr_array_header_t *entries;
+      svn_fs_fs__p2l_entry_t *last_entry;
+      int i;
+
+      svn_pool_clear(iterpool);
+
+      /* get all entries for the current block */
+      SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, fs, &rev_file, start,
+                                          offset, iterpool));
+      if (entries->nelts == 0)
+        return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_CORRUPTION,
+                                 NULL,
+                                 _("p2l does not cover offset %s"
+                                   " for revision %ld"),
+                                  apr_off_t_toa(pool, offset), start);
+
+      /* process all entries (and later continue with the next block) */
+      last_entry
+        = &APR_ARRAY_IDX(entries, entries->nelts-1, svn_fs_fs__p2l_entry_t);
+      offset = last_entry->offset + last_entry->size;
+
+      for (i = 0; i < entries->nelts; ++i)
+        {
+          svn_fs_fs__p2l_entry_t *entry
+            = &APR_ARRAY_IDX(entries, i, svn_fs_fs__p2l_entry_t);
+
+          /* check all sub-items for consist entries in the L2P index */
+          if (entry->type != SVN_FS_FS__ITEM_TYPE_UNUSED)
+            {
+              apr_off_t l2p_offset;
+              SVN_ERR(svn_fs_fs__item_offset(&l2p_offset, fs, &rev_file,
+                                             entry->item.revision, NULL,
+                                             entry->item.number, iterpool));
+
+              if (l2p_offset != entry->offset)
+                return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT,
+                                         NULL,
+                                         _("l2p index entry PHYS %s"
+                                           "does not match p2l index value "
+                                           "LOG r%ld:i%ld for PHYS %s"),
+                                         apr_off_t_toa(pool, l2p_offset),
+                                         entry->item.revision,
+                                         (long)entry->item.number,
+                                         apr_off_t_toa(pool, entry->offset));
+            }
+        }
+
+      if (cancel_func)
+        SVN_ERR(cancel_func(cancel_baton));
+    }
+
+  svn_pool_destroy(iterpool);
+
+  return SVN_NO_ERROR;
+}
+
+/* Items smaller than this can be read at once into a buffer and directly
+ * be checksummed.  Larger items require stream processing.
+ * Must be a multiple of 8. */
+#define STREAM_THRESHOLD 4096
+
+/* Verify that the next SIZE bytes read from FILE are NUL.
+ * SIZE must not exceed STREAM_THRESHOLD.  Use POOL for allocations.
+ */
+static svn_error_t *
+expect_buffer_nul(apr_file_t *file,
+                  apr_off_t size,
+                  apr_pool_t *pool)
+{
+  union
+  {
+    unsigned char buffer[STREAM_THRESHOLD];
+    apr_uint64_t chunks[STREAM_THRESHOLD / sizeof(apr_uint64_t)];
+  } data;
+
+  apr_size_t i;
+  SVN_ERR_ASSERT(size <= STREAM_THRESHOLD);
+
+  /* read the whole data block; error out on failure */
+  data.chunks[(size - 1)/ sizeof(apr_uint64_t)] = 0;
+  SVN_ERR(svn_io_file_read_full2(file, data.buffer, size, NULL, NULL, pool));
+
+  /* chunky check */
+  for (i = 0; i < size / sizeof(apr_uint64_t); ++i)
+    if (data.chunks[i] != 0)
+      break;
+
+  /* byte-wise check upon mismatch or at the end of the block */
+  for (i *= sizeof(apr_uint64_t); i < size; ++i)
+    if (data.buffer[i] != 0)
+      {
+        const char *file_name;
+        apr_off_t offset;
+        
+        SVN_ERR(svn_io_file_name_get(&file_name, file, pool));
+        SVN_ERR(svn_fs_fs__get_file_offset(&offset, file, pool));
+        offset -= size - i;
+
+        return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                                 _("Empty section in file %s contains "
+                                   "non-NUL data at offset %s"),
+                                 file_name, apr_off_t_toa(pool, offset));
+      }
+
+  return SVN_NO_ERROR;
+}
+
+/* Verify that the next SIZE bytes read from FILE are NUL.
+ * Use POOL for allocations.
+ */
+static svn_error_t *
+read_all_nul(apr_file_t *file,
+             apr_off_t size,
+             apr_pool_t *pool)
+{
+  for (; size >= STREAM_THRESHOLD; size -= STREAM_THRESHOLD)
+    SVN_ERR(expect_buffer_nul(file, STREAM_THRESHOLD, pool));
+
+  if (size)
+    SVN_ERR(expect_buffer_nul(file, size, pool));
+
+  return SVN_NO_ERROR;
+}
+
+/* Compare the ACTUAL checksum with the one expected by ENTRY.
+ * Return an error in case of mismatch.  Use the name of FILE
+ * in error message.  Allocate data in POOL.
+ */
+static svn_error_t *
+expected_checksum(apr_file_t *file,
+                  svn_fs_fs__p2l_entry_t *entry,
+                  apr_uint32_t actual,
+                  apr_pool_t *pool)
+{
+  if (actual != entry->fnv1_checksum)
+    {
+      const char *file_name;
+
+      SVN_ERR(svn_io_file_name_get(&file_name, file, pool));
+      return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                               _("Checksum mismatch item at offset %s of "
+                                 "length %s bytes in file %s"),
+                               apr_off_t_toa(pool, entry->offset),
+                               apr_off_t_toa(pool, entry->size), file_name);
+    }
+
+  return SVN_NO_ERROR;
+}
+
+/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read
+ * from FILE will match ENTRY's expected checksum.  SIZE must not
+ * exceed STREAM_THRESHOLD.  Use POOL for allocations.
+ */
+static svn_error_t *
+expected_buffered_checksum(apr_file_t *file,
+                           svn_fs_fs__p2l_entry_t *entry,
+                           apr_pool_t *pool)
+{
+  unsigned char buffer[STREAM_THRESHOLD];
+  SVN_ERR_ASSERT(entry->size <= STREAM_THRESHOLD);
+
+  SVN_ERR(svn_io_file_read_full2(file, buffer, (apr_size_t)entry->size,
+                                 NULL, NULL, pool));
+  SVN_ERR(expected_checksum(file, entry,
+                            svn__fnv1a_32x4(buffer, (apr_size_t)entry->size),
+                            pool));
+
+  return SVN_NO_ERROR;
+}
+
+/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read from
+ * FILE will match ENTRY's expected checksum.  Use POOL for allocations.
+ */
+static svn_error_t *
+expected_streamed_checksum(apr_file_t *file,
+                           svn_fs_fs__p2l_entry_t *entry,
+                           apr_pool_t *pool)
+{
+  unsigned char buffer[STREAM_THRESHOLD];
+  svn_checksum_t *checksum;
+  svn_checksum_ctx_t *context
+    = svn_checksum_ctx_create(svn_checksum_fnv1a_32x4, pool);
+  apr_off_t size = entry->size;
+
+  while (size > 0)
+    {
+      apr_size_t to_read = size > sizeof(buffer)
+                         ? sizeof(buffer)
+                         : (apr_size_t)size;
+      SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL,
+                                     pool));
+      SVN_ERR(svn_checksum_update(context, buffer, to_read));
+      size -= to_read;
+    }
+
+  SVN_ERR(svn_checksum_final(&checksum, context, pool));
+  SVN_ERR(expected_checksum(file, entry,
+                            *(apr_uint32_t *)checksum->digest,
+                            pool));
+
+  return SVN_NO_ERROR;
+}
+
+/* Verify that for all phys-to-log index entries for revisions START to
+ * START + COUNT-1 in FS match the actual pack / rev file contents.
+ * If given, invoke CANCEL_FUNC with CANCEL_BATON at regular intervals.
+ * Use POOL for allocations.
+ *
+ * Please note that we can only check on pack / rev file granularity and
+ * must only be called for a single rev / pack file.
+ */
+static svn_error_t *
+compare_p2l_to_rev(svn_fs_t *fs,
+                   svn_revnum_t start,
+                   svn_revnum_t count,
+                   svn_cancel_func_t cancel_func,
+                   void *cancel_baton,
+                   apr_pool_t *pool)
+{
+  fs_fs_data_t *ffd = fs->fsap_data;
+  apr_pool_t *iterpool = svn_pool_create(pool);
+  apr_off_t max_offset;
+  apr_off_t offset = 0;
+  svn_fs_fs__revision_file_t *rev_file;
+
+  /* open the pack / rev file that is covered by the p2l index */
+  SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start, pool));
+
+  /* check file size vs. range covered by index */
+  SVN_ERR(svn_io_file_seek(rev_file->file, APR_END, &offset, pool));
+  SVN_ERR(svn_fs_fs__p2l_get_max_offset(&max_offset, fs, rev_file, start,
+                                        pool));
+
+  if (offset != max_offset)
+    return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT, NULL,
+                             _("File size of %s for revision r%ld does "
+                               "not match p2l index size of %s"),
+                             apr_off_t_toa(pool, offset), start,
+                             apr_off_t_toa(pool, max_offset));
+
+  SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, NULL, 0,
+                                   pool));
+
+  /* for all offsets in the file, get the P2L index entries and check
+     them against the L2P index */
+  for (offset = 0; offset < max_offset; )
+    {
+      apr_array_header_t *entries;
+      int i;
+
+      svn_pool_clear(iterpool);
+
+      /* get all entries for the current block */
+      SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, fs, rev_file, start,
+                                          offset, iterpool));
+
+      /* process all entries (and later continue with the next block) */
+      for (i = 0; i < entries->nelts; ++i)
+        {
+          svn_fs_fs__p2l_entry_t *entry
+            = &APR_ARRAY_IDX(entries, i, svn_fs_fs__p2l_entry_t);
+
+          /* skip bits we previously checked */
+          if (i == 0 && entry->offset < offset)
+            continue;
+
+          /* skip zero-sized entries */
+          if (entry->size == 0)
+            continue;
+
+          /* p2l index must cover all rev / pack file offsets exactly once */
+          if (entry->offset != offset)
+            return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT,
+                                     NULL,
+                                     _("p2l index entry for revision r%ld"
+                                       " is non-contiguous between offsets "
+                                       " %s and %s"),
+                                     start,
+                                     apr_off_t_toa(pool, offset),
+                                     apr_off_t_toa(pool, entry->offset));
+
+          /* empty sections must contain NUL bytes only */
+          if (entry->type == SVN_FS_FS__ITEM_TYPE_UNUSED)
+            {
+              /* skip filler entry at the end of the p2l index */
+              if (entry->offset != max_offset)
+                SVN_ERR(read_all_nul(rev_file->file, entry->size, pool));
+            }
+          else if (entry->fnv1_checksum)
+            {
+              if (entry->size < STREAM_THRESHOLD)
+                SVN_ERR(expected_buffered_checksum(rev_file->file, entry,
+                                                   pool));
+              else
+                SVN_ERR(expected_streamed_checksum(rev_file->file, entry,
+                                                   pool));
+            }
+
+          /* advance offset */
+          offset += entry->size;
+        }
+
+      if (cancel_func)
+        SVN_ERR(cancel_func(cancel_baton));
+    }
+
+  svn_pool_destroy(iterpool);
+
+  return SVN_NO_ERROR;
+}
+
+static svn_revnum_t
+packed_base_rev(svn_fs_t *fs, svn_revnum_t rev)
+{
+  fs_fs_data_t *ffd = fs->fsap_data;
+
+  return rev < ffd->min_unpacked_rev
+       ? rev - (rev % ffd->max_files_per_dir)
+       : rev;
+}
+
+static svn_revnum_t
+pack_size(svn_fs_t *fs, svn_revnum_t rev)
+{
+  fs_fs_data_t *ffd = fs->fsap_data;
+
+  return rev < ffd->min_unpacked_rev ? ffd->max_files_per_dir : 1;
+}
+
+/* Verify that the log-to-phys indexes and phys-to-log indexes are
+ * consistent with each other.  The function signature is similar to
+ * svn_fs_fs__verify.
+ *
+ * The values of START and END have already been auto-selected and
+ * verified.  You may call this for format7 or higher repos.
+ */
+static svn_error_t *
+verify_index_consistency(svn_fs_t *fs,
+                         svn_revnum_t start,
+                         svn_revnum_t end,
+                         svn_fs_progress_notify_func_t notify_func,
+                         void *notify_baton,
+                         svn_cancel_func_t cancel_func,
+                         void *cancel_baton,
+                         apr_pool_t *pool)
+{
+  fs_fs_data_t *ffd = fs->fsap_data;
+  svn_revnum_t revision, next_revision;
+  apr_pool_t *iterpool = svn_pool_create(pool);
+
+  for (revision = start; revision <= end; revision = next_revision)
+    {
+      svn_error_t *err = SVN_NO_ERROR;
+
+      svn_revnum_t count = pack_size(fs, revision);
+      svn_revnum_t pack_start = packed_base_rev(fs, revision);
+      svn_revnum_t pack_end = pack_start + count;
+
+      svn_pool_clear(iterpool);
+
+      if (notify_func && (pack_start % ffd->max_files_per_dir == 0))
+        notify_func(pack_start, notify_baton, iterpool);
+
+      /* two-way index check */
+      err = compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start,
+                                     cancel_func, cancel_baton, iterpool);
+      if (!err)
+        err = compare_p2l_to_l2p_index(fs, pack_start, pack_end - pack_start,
+                                       cancel_func, cancel_baton, iterpool);
+
+      /* verify in-index checksums and types vs. actual rev / pack files */
+      if (!err)
+        err = compare_p2l_to_rev(fs, pack_start, pack_end - pack_start,
+                                 cancel_func, cancel_baton, iterpool);
+
+      /* concurrent packing is one of the reasons why verification may fail.
+         Make sure, we operate on up-to-date information. */
+      if (err)
+        SVN_ERR(svn_fs_fs__read_min_unpacked_rev(&ffd->min_unpacked_rev,
+                                                 fs, pool));
+
+      /* retry the whole shard if it got packed in the meantime */
+      if (err && count != pack_size(fs, revision))
+        {
+          svn_error_clear(err);
+
+          /* We could simply assign revision here but the code below is
+             more intuitive to maintainers. */
+          next_revision = packed_base_rev(fs, revision);
+        }
+      else
+        {
+          SVN_ERR(err);
+          next_revision = pack_end;
+        }
+    }
+
+  svn_pool_destroy(iterpool);
+
+  return SVN_NO_ERROR;
+}
+
 svn_error_t *
 svn_fs_fs__verify(svn_fs_t *fs,
                   svn_revnum_t start,
@@ -175,6 +691,14 @@ svn_fs_fs__verify(svn_fs_t *fs,
   SVN_ERR(svn_fs_fs__ensure_revision_exists(start, fs, pool));
   SVN_ERR(svn_fs_fs__ensure_revision_exists(end, fs, pool));
 
+  /* log/phys index consistency.  We need to check them first to make
+     sure we can access the rev / pack files in format7. */
+  if (svn_fs_fs__use_log_addressing(fs, end))
+    SVN_ERR(verify_index_consistency(fs,
+                                     MAX(start, ffd->min_log_addressing_rev),
+                                     end, notify_func, notify_baton,
+                                     cancel_func, cancel_baton, pool));
+
   /* rep cache consistency */
   if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT)
     SVN_ERR(verify_rep_cache(fs, start, end, notify_func, notify_baton,

Modified: subversion/branches/fsfs-improvements/subversion/tests/cmdline/svnadmin_tests.py
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-improvements/subversion/tests/cmdline/svnadmin_tests.py?rev=1546928&r1=1546927&r2=1546928&view=diff
==============================================================================
--- subversion/branches/fsfs-improvements/subversion/tests/cmdline/svnadmin_tests.py (original)
+++ subversion/branches/fsfs-improvements/subversion/tests/cmdline/svnadmin_tests.py Mon Dec  2 08:48:00 2013
@@ -245,18 +245,77 @@ def load_dumpstream(sbox, dump, *varargs
   return load_and_verify_dumpstream(sbox, None, None, None, False, dump,
                                     *varargs)
 
-def set_changed_path_list(filename, changes):
-  """ Replace the changed paths list in the file given by FILENAME
+def read_l2p(sbox, revision, item):
+  """ For the format 7+ repository in SBOX, return the physical offset
+      of ITEM in REVISION.  This code supports only small, nonpacked revs. """
+
+  filename = fsfs_file(sbox.repo_dir, 'revs', str(revision) + ".l2p")
+
+  fp = open(filename, 'rb')
+  contents = fp.read()
+  length = len(contents)
+  fp.close()
+
+  # decode numbers
+  numbers = []
+  value = 0
+  shift = 0
+  for c in contents:
+    char = ord(c)
+    value += (char & 127) << shift
+    if char < 128:
+      numbers.append(value)
+      shift = 0
+      value = 0
+    else:
+      shift += 7
+
+  # decode offsets
+  numbers[7] = -1
+  for i in range(8, len(numbers)):
+    if numbers[i] & 1 == 1:
+      numbers[i] = - (numbers[i] + 1) / 2
+    else:
+      numbers[i] = numbers[i] / 2
+    numbers[i] += numbers[i-1]
+
+  # we support only small, unpacked rev files
+  if numbers[1] < len(numbers) or numbers[3] != 1 :
+    raise svntest.Failure("More than 1 page in %s" % filename)
+  if numbers[2] != 1:
+    raise svntest.Failure("More than 1 rev in %s" % filename)
+
+  return numbers[item + 7]
+
+def repo_format(sbox):
+  """ Return the repository format number for SBOX."""
+
+  format_file = open(os.path.join(sbox.repo_dir, "db", "format"))
+  format = int(format_file.read()[:1])
+  format_file.close()
+
+  return format
+
+def set_changed_path_list(sbox, revision, changes):
+  """ Replace the changed paths list in the revision file REVISION in SBOX
       with the text CHANGES."""
 
   # read full file
-  fp = open(filename, 'r+b')
+  fp = open(fsfs_file(sbox.repo_dir, 'revs', str(revision)), 'r+b')
   contents = fp.read()
 
-  # replace the changed paths list
-  length = len(contents)
-  header = contents[contents.rfind('\n', length - 64, length - 1):]
-  body_len = long(header.split(' ')[1])
+  if repo_format(sbox) < 7:
+    # replace the changed paths list
+    length = len(contents)
+    header = contents[contents.rfind('\n', length - 64, length - 1):]
+    body_len = long(header.split(' ')[1])
+
+  else:
+    # we will invalidate the l2p index but that's ok for the
+    # kind of tests we run here. The p2l index remains valid
+    # because the offset of the last item does not change
+    body_len = read_l2p(sbox, revision, 1)
+    header = '\n'
 
   contents = contents[:body_len] + changes + header
 
@@ -272,7 +331,7 @@ def set_changed_path_list(filename, chan
 
 #----------------------------------------------------------------------
 
-def test_create(sbox):
+def test_create(sbox, minor_version=None):
   "'svnadmin create'"
 
 
@@ -282,7 +341,7 @@ def test_create(sbox):
   svntest.main.safe_rmtree(repo_dir, 1)
   svntest.main.safe_rmtree(wc_dir)
 
-  svntest.main.create_repos(repo_dir)
+  svntest.main.create_repos(repo_dir, minor_version)
 
   svntest.actions.run_and_verify_svn("Creating rev 0 checkout",
                                      ["Checked out revision 0.\n"], [],
@@ -581,7 +640,9 @@ def verify_windows_paths_in_repos(sbox):
 
   # unfortunately, some backends needs to do more checks than other
   # resulting in different progress output
-  if svntest.main.is_fs_type_fsx():
+  if svntest.main.is_fs_type_fsx() or \
+    (svntest.main.is_fs_type_fsfs() and \
+        svntest.main.options.server_minor_version >= 9):
     svntest.verify.compare_and_display_lines(
       "Error while running 'svnadmin verify'.",
       'STDERR', ["* Verifying metadata at revision 0 ...\n",
@@ -634,7 +695,9 @@ def verify_incremental_fsfs(sbox):
   """svnadmin verify detects corruption dump can't"""
 
   # setup a repo with a directory 'c:hi'
-  sbox.build(create_wc = False)
+  # use physical addressing as this is hard to provoke with logical addressing
+  sbox.build(create_wc = False,
+             minor_version = min(svntest.main.options.server_minor_version,8))
   repo_url = sbox.repo_url
   E_url = sbox.repo_url + '/A/B/E'
 
@@ -1458,7 +1521,11 @@ def verify_non_utf8_paths(sbox):
   "svnadmin verify with non-UTF-8 paths"
 
   dumpfile = clean_dumpfile()
-  test_create(sbox)
+
+  # Corruption only possible in physically addressed revisions created
+  # with pre-1.6 servers.
+  test_create(sbox,
+              minor_version = min(svntest.main.options.server_minor_version,8))
 
   # Load the dumpstream
   load_and_verify_dumpstream(sbox, [], [], dumpfile_revisions, False,
@@ -1476,11 +1543,14 @@ def verify_non_utf8_paths(sbox):
       # replace 'A' with a latin1 character -- the new path is not valid UTF-8
       fp_new.write("\xE6\n")
     elif line == "text: 1 279 32 0 d63ecce65d8c428b86f4f8b0920921fe\n":
-      # fix up the representation checksum
+      # phys, PLAIN directories: fix up the representation checksum
       fp_new.write("text: 1 279 32 0 b50b1d5ed64075b5f632f3b8c30cd6b2\n")
     elif line == "text: 1 292 44 32 a6be7b4cf075fd39e6a99eb69a31232b\n":
-      # fix up the representation checksum
+      # phys, deltified directories: fix up the representation checksum
       fp_new.write("text: 1 292 44 32 f2e93e73272cac0f18fccf16f224eb93\n")
+    elif line == "text: 1 6 31 0 90f306aa9bfd72f456072076a2bd94f7\n":
+      # log addressing: fix up the representation checksum
+      fp_new.write("text: 1 6 31 0 db2d4a0bad5dff0aea9a288dec02f1fb\n")
     elif line == "cpath: /A\n":
       # also fix up the 'created path' field
       fp_new.write("cpath: /\xE6\n")
@@ -1654,9 +1724,13 @@ def hotcopy_incremental_packed(sbox):
   backup_dir, backup_url = sbox.add_repo_path('backup')
   os.mkdir(backup_dir)
   cwd = os.getcwd()
+
   # Configure two files per shard to trigger packing
   format_file = open(os.path.join(sbox.repo_dir, 'db', 'format'), 'wb')
-  format_file.write("6\nlayout sharded 2\n")
+  if svntest.main.options.server_minor_version >= 9:
+    format_file.write("7\nlayout sharded 2\naddressing logical 0\n")
+  else:
+    format_file.write("6\nlayout sharded 2\n")
   format_file.close()
 
   # Pack revisions 0 and 1.
@@ -1890,20 +1964,26 @@ def verify_keep_going(sbox):
                                                         "--keep-going",
                                                         sbox.repo_dir)
 
-  exp_out = svntest.verify.RegexListOutput([".*Verifying repository metadata",
-                                           ".*Verified revision 0.",
-                                           ".*Verified revision 1.",
-                                           ".*Error verifying revision 2.",
-                                           ".*Error verifying revision 3.",
-                                           ".*",
-                                           ".*Summary.*",
-                                           ".*r2: E160004:.*",
-                                           ".*r3: E160004:.*",
-                                           ".*r3: E160004:.*"])
-
-  exp_err = svntest.verify.RegexListOutput(["svnadmin: E160004:.*",
-                                           "svnadmin: E165011:.*"], False)
-
+  if svntest.main.is_fs_log_addressing():
+    exp_out = svntest.verify.RegexListOutput([".*Verifying metadata at revision 0",
+                                             ".*Verified revision 0.",
+                                             ".*Verified revision 1.",
+                                             ".*Verified revision 2.",
+                                             ".*Verified revision 3."])
+    exp_err = svntest.verify.RegexListOutput(["svnadmin: E165011:.*"], False)
+  else:
+    exp_out = svntest.verify.RegexListOutput([".*Verifying repository metadata",
+                                              ".*Verified revision 0.",
+                                              ".*Verified revision 1.",
+                                              ".*Error verifying revision 2.",
+                                              ".*Error verifying revision 3.",
+                                              ".*",
+                                              ".*Summary.*",
+                                              ".*r2: E160004:.*",
+                                              ".*r3: E160004:.*",
+                                              ".*r3: E160004:.*"])
+    exp_err = svntest.verify.RegexListOutput(["svnadmin: E160004:.*",
+                                              "svnadmin: E165011:.*"], False)
 
   if svntest.verify.verify_outputs("Unexpected error while running 'svnadmin verify'.",
                                    output, errput, exp_out, exp_err):
@@ -1912,10 +1992,13 @@ def verify_keep_going(sbox):
   exit_code, output, errput = svntest.main.run_svnadmin("verify",
                                                         sbox.repo_dir)
 
-  exp_out = svntest.verify.RegexListOutput([".*Verifying repository metadata",
-                                           ".*Verified revision 0.",
-                                           ".*Verified revision 1.",
-                                           ".*Error verifying revision 2."])
+  if (svntest.main.options.server_minor_version < 9):
+    exp_out = svntest.verify.RegexListOutput([".*Verifying repository metadata",
+                                             ".*Verified revision 0.",
+                                             ".*Verified revision 1.",
+                                             ".*Error verifying revision 2."])
+  else:
+    exp_out = svntest.verify.RegexListOutput([".*Verifying metadata at revision 0"])
 
   if svntest.verify.verify_outputs("Unexpected error while running 'svnadmin verify'.",
                                    output, errput, exp_out, exp_err):
@@ -1947,43 +2030,44 @@ def verify_invalid_path_changes(sbox):
   # "carried over" but that all corrupts we get detected independently
 
   # add existing node
-  set_changed_path_list(fsfs_file(sbox.repo_dir, 'revs', '2'),
+  set_changed_path_list(sbox, 2,
                         "_0.0.t1-1 add-dir false false /A\n\n")
 
   # add into non-existent parent
-  set_changed_path_list(fsfs_file(sbox.repo_dir, 'revs', '4'),
+  set_changed_path_list(sbox, 4,
                         "_0.0.t3-2 add-dir false false /C/X\n\n")
 
   # del non-existent node
-  set_changed_path_list(fsfs_file(sbox.repo_dir, 'revs', '6'),
+  set_changed_path_list(sbox, 6,
                         "_0.0.t5-2 delete-dir false false /C\n\n")
 
   # del existent node of the wrong kind
+  #
   # THIS WILL NOT BE DETECTED
   # since dump mechanism and file don't care about the types of deleted nodes
-  set_changed_path_list(fsfs_file(sbox.repo_dir, 'revs', '8'),
+  set_changed_path_list(sbox, 8,
                         "_0.0.t7-2 delete-file false false /B3\n\n")
 
   # copy from non-existent node
-  set_changed_path_list(fsfs_file(sbox.repo_dir, 'revs', '10'),
+  set_changed_path_list(sbox, 10,
                         "_0.0.t9-2 add-dir false false /B10\n"
                         "6 /B8\n")
 
   # copy from existing node of the wrong kind
-  set_changed_path_list(fsfs_file(sbox.repo_dir, 'revs', '12'),
+  set_changed_path_list(sbox, 12,
                         "_0.0.t11-2 add-file false false /B12\n"
                         "9 /B8\n")
 
   # modify non-existent node
-  set_changed_path_list(fsfs_file(sbox.repo_dir, 'revs', '14'),
+  set_changed_path_list(sbox, 14,
                         "_0.0.t13-2 modify-file false false /A/D/H/foo\n\n")
 
   # modify existent node of the wrong kind
-  set_changed_path_list(fsfs_file(sbox.repo_dir, 'revs', '16'),
+  set_changed_path_list(sbox, 16,
                         "_0.0.t15-2 modify-file false false /B12\n\n")
 
   # replace non-existent node
-  set_changed_path_list(fsfs_file(sbox.repo_dir, 'revs', '18'),
+  set_changed_path_list(sbox, 18,
                         "_0.0.t17-2 replace-file false false /A/D/H/foo\n\n")
 
   # find corruptions
@@ -1991,7 +2075,12 @@ def verify_invalid_path_changes(sbox):
                                                         "--keep-going",
                                                         sbox.repo_dir)
 
-  exp_out = svntest.verify.RegexListOutput([".*Verifying repository metadata",
+  if repo_format(sbox) < 7:
+    first_line = ".*Verifying repository metadata"
+  else:
+    first_line = ".*Verifying metadata at revision 0"
+
+  exp_out = svntest.verify.RegexListOutput([first_line,
                                            ".*Verified revision 0.",
                                            ".*Verified revision 1.",
                                            ".*Error verifying revision 2.",
@@ -2042,13 +2131,17 @@ def verify_invalid_path_changes(sbox):
   exit_code, output, errput = svntest.main.run_svnadmin("verify",
                                                         sbox.repo_dir)
 
-  exp_out = svntest.verify.RegexListOutput([".*Verifying repository metadata",
-                                           ".*Verified revision 0.",
-                                           ".*Verified revision 1.",
-                                           ".*Error verifying revision 2."])
-
-  exp_err = svntest.verify.RegexListOutput(["svnadmin: E160020:.*",
-                                            "svnadmin: E165011:.*"], False)
+  if svntest.main.is_fs_log_addressing():
+    exp_out = svntest.verify.RegexListOutput([first_line])
+    exp_err = svntest.verify.RegexListOutput(["svnadmin: E160058:.*",
+                                              "svnadmin: E165011:.*"], False)
+  else:
+    exp_out = svntest.verify.RegexListOutput([".*Verifying repository metadata",
+                                              ".*Verified revision 0.",
+                                              ".*Verified revision 1.",
+                                              ".*Error verifying revision 2."])
+    exp_err = svntest.verify.RegexListOutput(["svnadmin: E160020:.*",
+                                              "svnadmin: E165011:.*"], False)
 
   if svntest.verify.verify_outputs("Unexpected error while running 'svnadmin verify'.",
                                    output, errput, exp_out, exp_err):

Modified: subversion/branches/fsfs-improvements/subversion/tests/cmdline/svntest/main.py
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-improvements/subversion/tests/cmdline/svntest/main.py?rev=1546928&r1=1546927&r2=1546928&view=diff
==============================================================================
--- subversion/branches/fsfs-improvements/subversion/tests/cmdline/svntest/main.py (original)
+++ subversion/branches/fsfs-improvements/subversion/tests/cmdline/svntest/main.py Mon Dec  2 08:48:00 2013
@@ -1337,7 +1337,8 @@ def is_fs_type_bdb():
   return options.fs_type == 'bdb'
 
 def is_fs_log_addressing():
-  return is_fs_type_fsx()
+  return is_fs_type_fsx() or \
+        (is_fs_type_fsfs() and options.server_minor_version >= 9)
 
 def is_os_windows():
   return os.name == 'nt'

Propchange: subversion/branches/fsfs-improvements/subversion/tests/libsvn_fs_fs/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Mon Dec  2 08:48:00 2013
@@ -2,6 +2,7 @@ Debug
 Release
 .libs
 test-repo-*
+upgrade_*
 fs-pack-test
 fs-fs-pack-test
 test-get-set-revprop-packed-fs



Mime
View raw message