subversion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From v...@apache.org
Subject svn commit: r1425415 [4/13] - in /subversion/branches/javahl-ra: ./ build/generator/templates/ notes/ notes/directory-index/ subversion/bindings/swig/python/svn/ subversion/bindings/swig/ruby/test/ subversion/include/ subversion/include/private/ subver...
Date Sun, 23 Dec 2012 06:03:17 GMT
Modified: subversion/branches/javahl-ra/subversion/libsvn_fs_fs/fs_fs.c
URL: http://svn.apache.org/viewvc/subversion/branches/javahl-ra/subversion/libsvn_fs_fs/fs_fs.c?rev=1425415&r1=1425414&r2=1425415&view=diff
==============================================================================
--- subversion/branches/javahl-ra/subversion/libsvn_fs_fs/fs_fs.c (original)
+++ subversion/branches/javahl-ra/subversion/libsvn_fs_fs/fs_fs.c Sun Dec 23 06:03:14 2012
@@ -62,6 +62,7 @@
 #include "private/svn_string_private.h"
 #include "private/svn_fs_util.h"
 #include "private/svn_subr_private.h"
+#include "private/svn_delta_private.h"
 #include "../libsvn_fs/fs-loader.h"
 
 #include "svn_private_config.h"
@@ -192,13 +193,12 @@ is_packed_rev(svn_fs_t *fs, svn_revnum_t
 static svn_boolean_t
 is_packed_revprop(svn_fs_t *fs, svn_revnum_t rev)
 {
-#if 0
   fs_fs_data_t *ffd = fs->fsap_data;
 
-  return (rev < ffd->min_unpacked_revprop);
-#else
-  return FALSE;
-#endif
+  /* rev 0 will not be packed */
+  return (rev < ffd->min_unpacked_rev)
+      && (rev != 0)
+      && (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT);
 }
 
 static const char *
@@ -253,7 +253,8 @@ path_rev_packed(svn_fs_t *fs, svn_revnum
   assert(is_packed_rev(fs, rev));
 
   return svn_dirent_join_many(pool, fs->path, PATH_REVS_DIR,
-                              apr_psprintf(pool, "%ld.pack",
+                              apr_psprintf(pool,
+                                           "%ld" PATH_EXT_PACKED_SHARD,
                                            rev / ffd->max_files_per_dir),
                               kind, NULL);
 }
@@ -303,7 +304,7 @@ svn_fs_fs__path_rev_absolute(const char 
     }
   else
     {
-      *path = path_rev_packed(fs, rev, "pack", pool);
+      *path = path_rev_packed(fs, rev, PATH_PACKED, pool);
     }
 
   return SVN_NO_ERROR;
@@ -322,6 +323,18 @@ path_revprops_shard(svn_fs_t *fs, svn_re
 }
 
 static const char *
+path_revprops_pack_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool)
+{
+  fs_fs_data_t *ffd = fs->fsap_data;
+
+  assert(ffd->max_files_per_dir);
+  return svn_dirent_join_many(pool, fs->path, PATH_REVPROPS_DIR,
+                              apr_psprintf(pool, "%ld" PATH_EXT_PACKED_SHARD,
+                                           rev / ffd->max_files_per_dir),
+                              NULL);
+}
+
+static const char *
 path_revprops(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool)
 {
   fs_fs_data_t *ffd = fs->fsap_data;
@@ -1089,15 +1102,17 @@ svn_fs_fs__fs_supports_mergeinfo(svn_fs_
   return ffd->format >= SVN_FS_FS__MIN_MERGEINFO_FORMAT;
 }
 
+/* Read the configuration information of the file system at FS_PATH
+ * and set the respective values in FFD.  Use POOL for allocations.
+ */
 static svn_error_t *
-read_config(svn_fs_t *fs,
+read_config(fs_fs_data_t *ffd,
+            const char *fs_path,
             apr_pool_t *pool)
 {
-  fs_fs_data_t *ffd = fs->fsap_data;
-
   SVN_ERR(svn_config_read2(&ffd->config,
-                           svn_dirent_join(fs->path, PATH_CONFIG, pool),
-                           FALSE, FALSE, fs->pool));
+                           svn_dirent_join(fs_path, PATH_CONFIG, pool),
+                           FALSE, FALSE, pool));
 
   /* Initialize ffd->rep_sharing_allowed. */
   if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT)
@@ -1107,7 +1122,7 @@ read_config(svn_fs_t *fs,
   else
     ffd->rep_sharing_allowed = FALSE;
 
-  /* Initialize ffd->deltify_directories. */
+  /* Initialize deltification settings in ffd. */
   if (ffd->format >= SVN_FS_FS__MIN_DELTIFICATION_FORMAT)
     {
       SVN_ERR(svn_config_get_bool(ffd->config, &ffd->deltify_directories,
@@ -1135,6 +1150,28 @@ read_config(svn_fs_t *fs,
       ffd->max_linear_deltification = SVN_FS_FS_MAX_LINEAR_DELTIFICATION;
     }
 
+  /* Initialize revprop packing settings in ffd. */
+  if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
+    {
+      SVN_ERR(svn_config_get_bool(ffd->config, &ffd->compress_packed_revprops,
+                                  CONFIG_SECTION_PACKED_REVPROPS,
+                                  CONFIG_OPTION_COMPRESS_PACKED_REVPROPS,
+                                  FALSE));
+      SVN_ERR(svn_config_get_int64(ffd->config, &ffd->revprop_pack_size,
+                                   CONFIG_SECTION_PACKED_REVPROPS,
+                                   CONFIG_OPTION_REVPROP_PACK_SIZE,
+                                   ffd->compress_packed_revprops
+                                       ? 0x100
+                                       : 0x40));
+
+      ffd->revprop_pack_size *= 1024;
+    }
+  else
+    {
+      ffd->revprop_pack_size = 0x10000;
+      ffd->compress_packed_revprops = FALSE;
+    }
+
   return SVN_NO_ERROR;
 }
 
@@ -1204,7 +1241,7 @@ write_config(svn_fs_t *fs,
 "### In rarely read repositories, the I/O overhead may be significant as"    NL
 "### cache hit rates will most likely be low"                                NL
 "### directory deltification is disabled by default."                        NL
-"# " CONFIG_OPTION_ENABLE_DIR_DELTIFICATION " = true"                        NL
+"# " CONFIG_OPTION_ENABLE_DIR_DELTIFICATION " = false"                       NL
 "###"                                                                        NL
 "### The following parameter enables deltification for properties on files"  NL
 "### and directories.  Overall, this is a minor tuning option but can save"  NL
@@ -1212,7 +1249,7 @@ write_config(svn_fs_t *fs,
 "### properties.  You should not activate this if rep-sharing has been"      NL
 "### disabled."                                                              NL
 "### property deltification is disabled by default."                         NL
-"# " CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION " = true"                      NL
+"# " CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION " = false"                     NL
 "###"                                                                        NL
 "### During commit, the server may need to walk the whole change history of" NL
 "### of a given node to find a suitable deltification base.  This linear"    NL
@@ -1244,6 +1281,31 @@ write_config(svn_fs_t *fs,
 "### exclusive use of skip-deltas (as in pre-1.8)."                          NL
 "### For 1.8, the default value is 16; earlier versions use 1."              NL
 "# " CONFIG_OPTION_MAX_LINEAR_DELTIFICATION " = 16"                          NL
+""                                                                           NL
+"[" CONFIG_SECTION_PACKED_REVPROPS "]"                                       NL
+"### This parameter controls the size (in kBytes) of packed revprop files."  NL
+"### Revprops of consecutive revisions will be concatenated into a single"   NL
+"### file up to but not exceeding the threshold given here.  However, each"  NL
+"### pack file may be much smaller and revprops of a single revision may be" NL
+"### much larger than the limit set here.  The threshold will be applied"    NL
+"### before optional compression takes place."                               NL
+"### Large values will reduce disk space usage at the expense of increased"  NL
+"### latency and CPU usage reading and changing individual revprops.  They"  NL
+"### become an advantage when revprop caching has been enabled because a"    NL
+"### lot of data can be read in one go.  Values smaller than 4 kByte will"   NL
+"### not improve latency any further and quickly render revprop packing"     NL
+"### ineffective."                                                           NL
+"### revprop-pack-size is 64 kBytes by default for non-compressed revprop"   NL
+"### pack files and 256 kBytes when compression has been enabled."           NL
+"# " CONFIG_OPTION_REVPROP_PACK_SIZE " = 64"                                 NL
+"###"                                                                        NL
+"### To save disk space, packed revprop files may be compressed.  Standard"  NL
+"### revprops tend to allow for very effective compression.  Reading and"    NL
+"### even more so writing, become significantly more CPU intensive.  With"   NL
+"### revprop caching enabled, the overhead can be offset by reduced I/O"     NL
+"### unless you often modify revprops after packing."                        NL
+"### Compressing packed revprops is disabled by default."                    NL
+"# " CONFIG_OPTION_COMPRESS_PACKED_REVPROPS " = false"                       NL
 ;
 #undef NL
   return svn_io_file_create(svn_dirent_join(fs->path, PATH_CONFIG, pool),
@@ -1316,7 +1378,7 @@ svn_fs_fs__open(svn_fs_t *fs, const char
     SVN_ERR(update_min_unpacked_rev(fs, pool));
 
   /* Read the configuration file. */
-  SVN_ERR(read_config(fs, pool));
+  SVN_ERR(read_config(ffd, fs->path, pool));
 
   return get_youngest(&(ffd->youngest_rev_cache), path, pool);
 }
@@ -1336,6 +1398,85 @@ create_file_ignore_eexist(const char *fi
   return svn_error_trace(err);
 }
 
+/* forward declarations */
+
+static svn_error_t *
+pack_revprops_shard(const char *pack_file_dir,
+                    const char *shard_path,
+                    apr_int64_t shard,
+                    int max_files_per_dir,
+                    apr_off_t max_pack_size,
+                    int compression_level,
+                    svn_cancel_func_t cancel_func,
+                    void *cancel_baton,
+                    apr_pool_t *scratch_pool);
+
+static svn_error_t *
+delete_revprops_shard(const char *shard_path,
+                      apr_int64_t shard,
+                      int max_files_per_dir,
+                      svn_cancel_func_t cancel_func,
+                      void *cancel_baton,
+                      apr_pool_t *scratch_pool);
+
+/* In the filesystem FS, pack all revprop shards up to min_unpacked_rev.
+ * Use SCRATCH_POOL for temporary allocations.
+ */
+static svn_error_t *
+upgrade_pack_revprops(svn_fs_t *fs,
+                      apr_pool_t *scratch_pool)
+{
+  fs_fs_data_t *ffd = fs->fsap_data;
+  const char *revprops_shard_path;
+  const char *revprops_pack_file_dir;
+  apr_int64_t shard;
+  apr_int64_t first_unpacked_shard
+    =  ffd->min_unpacked_rev / ffd->max_files_per_dir;
+
+  apr_pool_t *iterpool = svn_pool_create(scratch_pool);
+  const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR,
+                                              scratch_pool);
+  int compression_level = ffd->compress_packed_revprops
+                           ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
+                           : SVN_DELTA_COMPRESSION_LEVEL_NONE;
+
+  /* first, pack all revprops shards to match the packed revision shards */
+  for (shard = 0; shard < first_unpacked_shard; ++shard)
+    {
+      revprops_pack_file_dir = svn_dirent_join(revsprops_dir,
+                   apr_psprintf(iterpool,
+                                "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
+                                shard),
+                   iterpool);
+      revprops_shard_path = svn_dirent_join(revsprops_dir,
+                       apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
+                       iterpool);
+
+      SVN_ERR(pack_revprops_shard(revprops_pack_file_dir, revprops_shard_path,
+                                  shard, ffd->max_files_per_dir,
+                                  (int)(0.9 * ffd->revprop_pack_size),
+                                  compression_level,
+                                  NULL, NULL, iterpool));
+      svn_pool_clear(iterpool);
+    }
+
+  /* delete the non-packed revprops shards afterwards */
+  for (shard = 0; shard < first_unpacked_shard; ++shard)
+    {
+      revprops_shard_path = svn_dirent_join(revsprops_dir,
+                       apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
+                       iterpool);
+      SVN_ERR(delete_revprops_shard(revprops_shard_path,
+                                    shard, ffd->max_files_per_dir,
+                                    NULL, NULL, iterpool));
+      svn_pool_clear(iterpool);
+    }
+
+  svn_pool_destroy(iterpool);
+
+  return SVN_NO_ERROR;
+}
+
 static svn_error_t *
 upgrade_body(void *baton, apr_pool_t *pool)
 {
@@ -1394,6 +1535,12 @@ upgrade_body(void *baton, apr_pool_t *po
   if (format < SVN_FS_FS__MIN_PACKED_FORMAT)
     SVN_ERR(svn_io_file_create(path_min_unpacked_rev(fs, pool), "0\n", pool));
 
+  /* If the file system supports revision packing but not revprop packing,
+     pack the revprops up to the point that revision data has been packed. */
+  if (   format >= SVN_FS_FS__MIN_PACKED_FORMAT
+      && format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
+    SVN_ERR(upgrade_pack_revprops(fs, pool));
+
   /* Bump the format file. */
   return write_format(format_path, SVN_FS_FS__FORMAT_NUMBER, max_files_per_dir,
                       TRUE, pool);
@@ -1407,7 +1554,7 @@ svn_fs_fs__upgrade(svn_fs_t *fs, apr_poo
 }
 
 
-/* SVN_ERR-like macros for dealing with recoverable errors on mutable files
+/* Functions for dealing with recoverable errors on mutable files
  *
  * Revprops, current, and txn-current files are mutable; that is, they
  * change as part of normal fsfs operation, in constrat to revs files, or
@@ -1440,98 +1587,82 @@ svn_fs_fs__upgrade(svn_fs_t *fs, apr_poo
  *
  ** Solution
  *
- * Wrap opens and reads of such files with RETRY_RECOVERABLE and
- * closes with IGNORE_RECOVERABLE.  Call these macros within a loop of
- * RECOVERABLE_RETRY_COUNT iterations (though, realistically, the
- * second try will succeed).  Make sure you put a break statement
- * after the close, at the end of your loop.  Immediately after your
- * loop, return err if err.
+ * Try open and read of such files in try_stringbuf_from_file().  Call
+ * this function within a loop of RECOVERABLE_RETRY_COUNT iterations
+ * (though, realistically, the second try will succeed).
+ */
+
+#define RECOVERABLE_RETRY_COUNT 10
+
+/* Read the file at PATH and return its content in *CONTENT. *CONTENT will
+ * not be modified unless the whole file was read successfully.
  *
- * You must initialize err to SVN_NO_ERROR and filehandle to NULL, as
- * these macros do not.
+ * ESTALE, EIO and ENOENT will not cause this function to return an error
+ * unless LAST_ATTEMPT has been set.  If MISSING is not NULL, indicate
+ * missing files (ENOENT) there.
+ *
+ * Use POOL for allocations.
  */
+static svn_error_t *
+try_stringbuf_from_file(svn_stringbuf_t **content,
+                        svn_boolean_t *missing,
+                        const char *path,
+                        svn_boolean_t last_attempt,
+                        apr_pool_t *pool)
+{
+  svn_error_t *err = svn_stringbuf_from_file2(content, path, pool);
+  if (missing)
+    *missing = FALSE;
+
+  if (err)
+    {
+      *content = NULL;
 
+      if (APR_STATUS_IS_ENOENT(err->apr_err))
+        {
+          if (!last_attempt)
+            {
+              svn_error_clear(err);
+              if (missing)
+                *missing = TRUE;
+              return SVN_NO_ERROR;
+            }
+        }
 #ifdef ESTALE
-/* Do not use do-while due to the embedded 'continue'.  */
-#define RETRY_RECOVERABLE(err, filehandle, expr)                \
-  if (1) {                                                      \
-    svn_error_clear(err);                                       \
-    err = (expr);                                               \
-    if (err)                                                    \
-      {                                                         \
-        apr_status_t _e = APR_TO_OS_ERROR(err->apr_err);        \
-        if ((_e == ESTALE) || (_e == EIO) || (_e == ENOENT)) {  \
-          if (NULL != filehandle)                               \
-            (void)apr_file_close(filehandle);                   \
-          continue;                                             \
-        }                                                       \
-        return svn_error_trace(err);                           \
-      }                                                         \
-  } else
-#define IGNORE_RECOVERABLE(err, expr)                           \
-  if (1) {                                                      \
-    svn_error_clear(err);                                       \
-    err = (expr);                                               \
-    if (err)                                                    \
-      {                                                         \
-        apr_status_t _e = APR_TO_OS_ERROR(err->apr_err);        \
-        if ((_e != ESTALE) && (_e != EIO))                      \
-          return svn_error_trace(err);                         \
-      }                                                         \
-  } else
-#define RECOVERABLE_RETRY_TEST \
-  i < 10
-#define RECOVERABLE_RETRY_NEXT \
-  i++
-#else
-#define RETRY_RECOVERABLE(err, filehandle, expr)  SVN_ERR(expr)
-#define IGNORE_RECOVERABLE(err, expr) SVN_ERR(expr)
-#define RECOVERABLE_RETRY_TEST
-#define RECOVERABLE_RETRY_NEXT
+      else if (APR_TO_OS_ERROR(err->apr_err) == ESTALE
+                || APR_TO_OS_ERROR(err->apr_err) == EIO)
+        {
+          if (!last_attempt)
+            {
+              svn_error_clear(err);
+              return SVN_NO_ERROR;
+            }
+        }
 #endif
+    }
 
-/* Long enough to hold: "<svn_revnum_t> <node id> <copy id>\0"
- * 19 bytes for svn_revnum_t (room for 32 or 64 bit values)
- * + 2 spaces
- * + 26 bytes for each id (these are actually unbounded, so we just
- *   have to pick something; 2^64 is 13 bytes in base-36)
- * + 1 terminating null
- */
-#define CURRENT_BUF_LEN 48
+  return svn_error_trace(err);
+}
 
 /* Read the 'current' file FNAME and store the contents in *BUF.
    Allocations are performed in POOL. */
 static svn_error_t *
-read_current(const char *fname, char **buf, apr_pool_t *pool)
+read_content(svn_stringbuf_t **content, const char *fname, apr_pool_t *pool)
 {
-  apr_file_t *revision_file = NULL;
-  apr_size_t len;
   int i;
-  svn_error_t *err = SVN_NO_ERROR;
-  apr_pool_t *iterpool;
+  *content = NULL;
 
-  *buf = apr_palloc(pool, CURRENT_BUF_LEN);
-  iterpool = svn_pool_create(pool);
-  for (i = 0; RECOVERABLE_RETRY_TEST; RECOVERABLE_RETRY_NEXT)
-    {
-      svn_pool_clear(iterpool);
-
-      RETRY_RECOVERABLE(err, revision_file,
-                        svn_io_file_open(&revision_file, fname,
-                                         APR_READ | APR_BUFFERED,
-                                         APR_OS_DEFAULT, iterpool));
-
-      len = CURRENT_BUF_LEN;
-      RETRY_RECOVERABLE(err, revision_file,
-                        svn_io_read_length_line(revision_file,
-                                                *buf, &len, iterpool));
-      IGNORE_RECOVERABLE(err, svn_io_file_close(revision_file, iterpool));
+  for (i = 0; !*content && (i < RECOVERABLE_RETRY_COUNT); ++i)
+    SVN_ERR(try_stringbuf_from_file(content, NULL,
+                                    fname, i + 1 < RECOVERABLE_RETRY_COUNT,
+                                    pool));
 
-      break;
-    }
-  svn_pool_destroy(iterpool);
+  if (!*content)
+    return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                             _("Can't read '%s'"),
+                             svn_dirent_local_style(fname, pool));
 
-  return svn_error_trace(err);
+  return SVN_NO_ERROR;
 }
 
 /* Find the youngest revision in a repository at path FS_PATH and
@@ -1542,12 +1673,11 @@ get_youngest(svn_revnum_t *youngest_p,
              const char *fs_path,
              apr_pool_t *pool)
 {
-  char *buf;
-
-  SVN_ERR(read_current(svn_dirent_join(fs_path, PATH_CURRENT, pool),
-                       &buf, pool));
+  svn_stringbuf_t *buf;
+  SVN_ERR(read_content(&buf, svn_dirent_join(fs_path, PATH_CURRENT, pool),
+                       pool));
 
-  *youngest_p = SVN_STR_TO_REV(buf);
+  *youngest_p = SVN_STR_TO_REV(buf->data);
 
   return SVN_NO_ERROR;
 }
@@ -1733,6 +1863,41 @@ open_pack_or_rev_file(apr_file_t **file,
   return svn_error_trace(err);
 }
 
+/* Reads a line from STREAM and converts it to a 64 bit integer to be
+ * returned in *RESULT.  If we encounter eof, set *HIT_EOF and leave
+ * *RESULT unchanged.  If HIT_EOF is NULL, EOF causes an "corrupt FS"
+ * error return.
+ * SCRATCH_POOL is used for temporary allocations.
+ */
+static svn_error_t *
+read_number_from_stream(apr_int64_t *result,
+                        svn_boolean_t *hit_eof,
+                        svn_stream_t *stream,
+                        apr_pool_t *scratch_pool)
+{
+  svn_stringbuf_t *sb;
+  svn_boolean_t eof;
+  svn_error_t *err;
+
+  SVN_ERR(svn_stream_readline(stream, &sb, "\n", &eof, scratch_pool));
+  if (hit_eof)
+    *hit_eof = eof;
+  else
+    if (eof)
+      return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Unexpected EOF"));
+
+  if (!eof)
+    {
+      err = svn_cstring_atoi64(result, sb->data);
+      if (err)
+        return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
+                                 _("Number '%s' invalid or too large"),
+                                 sb->data);
+    }
+
+  return SVN_NO_ERROR;
+}
+
 /* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file.
    Use POOL for temporary allocations. */
 static svn_error_t *
@@ -1766,7 +1931,8 @@ get_packed_offset(apr_off_t *rev_offset,
 
   /* Open the manifest file. */
   SVN_ERR(svn_stream_open_readonly(&manifest_stream,
-                                   path_rev_packed(fs, rev, "manifest", pool),
+                                   path_rev_packed(fs, rev, PATH_MANIFEST,
+                                                   pool),
                                    pool, pool));
 
   /* While we're here, let's just read the entire manifest file into an array,
@@ -1775,21 +1941,14 @@ get_packed_offset(apr_off_t *rev_offset,
   manifest = apr_array_make(pool, ffd->max_files_per_dir, sizeof(apr_off_t));
   while (1)
     {
-      svn_stringbuf_t *sb;
       svn_boolean_t eof;
       apr_int64_t val;
-      svn_error_t *err;
 
       svn_pool_clear(iterpool);
-      SVN_ERR(svn_stream_readline(manifest_stream, &sb, "\n", &eof, iterpool));
+      SVN_ERR(read_number_from_stream(&val, &eof, manifest_stream, iterpool));
       if (eof)
         break;
 
-      err = svn_cstring_atoi64(&val, sb->data);
-      if (err)
-        return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
-                                 _("Manifest offset '%s' too large"),
-                                 sb->data);
       APR_ARRAY_PUSH(manifest, apr_off_t) = (apr_off_t)val;
     }
   svn_pool_destroy(iterpool);
@@ -3123,74 +3282,383 @@ end_revprop_change(svn_fs_t *fs, apr_poo
   return write_revprop_generation_file(fs, current, pool);
 }
 
-/* Set the revision property list of revision REV in filesystem FS to
-   PROPLIST.  Use POOL for temporary allocations. */
+/* Container for all data required to access the packed revprop file
+ * for a given REVISION.  This structure will be filled incrementally
+ * by read_pack_revprops() its sub-routines.
+ */
+typedef struct packed_revprops_t
+{
+  /* revision number to read (not necessarily the first in the pack) */
+  svn_revnum_t revision;
+
+  /* current revprop generation. Used when populating the revprop cache */
+  apr_int64_t generation;
+
+  /* the actual revision properties */
+  apr_hash_t *properties;
+
+  /* their size when serialized to a single string
+   * (as found in PACKED_REVPROPS) */
+  apr_size_t serialized_size;
+
+
+  /* name of the pack file (without folder path) */
+  const char *filename;
+
+  /* packed shard folder path */
+  const char *folder;
+
+  /* sum of values in SIZES */
+  apr_size_t total_size;
+
+  /* first revision in the pack */
+  svn_revnum_t start_revision;
+
+  /* size of the revprops in PACKED_REVPROPS */
+  apr_array_header_t *sizes;
+
+  /* offset of the revprops in PACKED_REVPROPS */
+  apr_array_header_t *offsets;
+
+
+  /* concatenation of the serialized representation of all revprops 
+   * in the pack, i.e. the pack content without header and compression */
+  svn_stringbuf_t *packed_revprops;
+
+  /* content of the manifest.
+   * Maps long(rev - START_REVISION) to const char* pack file name */
+  apr_array_header_t *manifest;
+} packed_revprops_t;
+
+/* Parse the serialized revprops in CONTENT and return them in *PROPERTIES.
+ * Also, put them into the revprop cache, if activated, for future use.
+ * Three more parameters are being used to update the revprop cache: FS is
+ * our file system, the revprops belong to REVISION and the global revprop
+ * GENERATION is used as well.
+ * 
+ * The returned hash will be allocated in POOL, SCRATCH_POOL is being used
+ * for temporary allocations.
+ */
 static svn_error_t *
-set_revision_proplist(svn_fs_t *fs,
-                      svn_revnum_t rev,
-                      apr_hash_t *proplist,
-                      apr_pool_t *pool)
+parse_revprop(apr_hash_t **properties,
+              svn_fs_t *fs,
+              svn_revnum_t revision,
+              apr_int64_t generation,
+              svn_string_t *content,
+              apr_pool_t *pool,
+              apr_pool_t *scratch_pool)
 {
-  SVN_ERR(ensure_revision_exists(fs, rev, pool));
+  svn_stream_t *stream = svn_stream_from_string(content, scratch_pool);
+  *properties = apr_hash_make(pool);
 
-  /* if (1); null condition for easier merging to revprop-packing */
+  SVN_ERR(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool));
+  if (has_revprop_cache(fs, pool))
     {
-      const char *final_path = path_revprops(fs, rev, pool);
-      const char *tmp_path;
-      const char *perms_reference;
-      svn_stream_t *stream;
-      svn_node_kind_t kind = svn_node_none;
-
-      /* test whether revprops already exist for this revision */
-      if (has_revprop_cache(fs, pool))
-        SVN_ERR(svn_io_check_path(final_path, &kind, pool));
-
-      /* ### do we have a directory sitting around already? we really shouldn't
-         ### have to get the dirname here. */
-      SVN_ERR(svn_stream_open_unique(&stream, &tmp_path,
-                                     svn_dirent_dirname(final_path, pool),
-                                     svn_io_file_del_none, pool, pool));
-      SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
-      SVN_ERR(svn_stream_close(stream));
+      const char *key;
+      fs_fs_data_t *ffd = fs->fsap_data;
 
-      /* We use the rev file of this revision as the perms reference,
-         because when setting revprops for the first time, the revprop
-         file won't exist and therefore can't serve as its own reference.
-         (Whereas the rev file should already exist at this point.) */
-      SVN_ERR(svn_fs_fs__path_rev_absolute(&perms_reference, fs, rev, pool));
+      key = svn_fs_fs__combine_two_numbers(revision, generation,
+                                           scratch_pool);
+      SVN_ERR(svn_cache__set(ffd->revprop_cache, key, *properties,
+                             scratch_pool));
+    }
 
-      /* Now, we may actually be replacing revprops. Make sure that all other
-         threads and processes will know about this. */
-      if (kind != svn_node_none)
-        SVN_ERR(begin_revprop_change(fs, pool));
+  return SVN_NO_ERROR;
+}
 
-      SVN_ERR(move_into_place(tmp_path, final_path, perms_reference, pool));
+/* Read the non-packed revprops for revision REV in FS, put them into the
+ * revprop cache if activated and return them in *PROPERTIES.  GENERATION
+ * is the current revprop generation.
+ *
+ * If the data could not be read due to an otherwise recoverable error,
+ * leave *PROPERTIES unchanged. No error will be returned in that case.
+ *
+ * Allocations will be done in POOL.
+ */
+static svn_error_t *
+read_non_packed_revprop(apr_hash_t **properties,
+                        svn_fs_t *fs,
+                        svn_revnum_t rev,
+                        apr_int64_t generation,
+                        apr_pool_t *pool)
+{
+  svn_stringbuf_t *content = NULL;
+  apr_pool_t *iterpool = svn_pool_create(pool);
+  svn_boolean_t missing = FALSE;
+  int i;
 
-      /* Indicate that the update (if relevant) has been completed. */
-      if (kind != svn_node_none)
-        SVN_ERR(end_revprop_change(fs, pool));
+  for (i = 0; i < RECOVERABLE_RETRY_COUNT && !missing && !content; ++i)
+    {
+      svn_pool_clear(iterpool);
+      SVN_ERR(try_stringbuf_from_file(&content,
+                                      &missing,
+                                      path_revprops(fs, rev, iterpool),
+                                      i + 1 < RECOVERABLE_RETRY_COUNT,
+                                      iterpool));
     }
 
+  if (content)
+    SVN_ERR(parse_revprop(properties, fs, rev, generation,
+                          svn_stringbuf__morph_into_string(content),
+                          pool, iterpool));
+
+  svn_pool_clear(iterpool);
+
   return SVN_NO_ERROR;
 }
 
+/* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST
+ * members. Use POOL for allocating results and SCRATCH_POOL for temporaries.
+ */
 static svn_error_t *
-revision_proplist(apr_hash_t **proplist_p,
+get_revprop_packname(svn_fs_t *fs,
+                     packed_revprops_t *revprops,
+                     apr_pool_t *pool,
+                     apr_pool_t *scratch_pool)
+{
+  fs_fs_data_t *ffd = fs->fsap_data;
+  svn_stringbuf_t *content = NULL;
+  const char *manifest_file_path;
+  int idx;
+
+  /* read content of the the manifest file */
+  revprops->folder = path_revprops_pack_shard(fs, revprops->revision, pool);
+  manifest_file_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool);
+
+  SVN_ERR(read_content(&content, manifest_file_path, pool));
+
+  /* parse the manifest. Every line is a file name */
+  revprops->manifest = apr_array_make(pool, ffd->max_files_per_dir,
+                                      sizeof(const char*));
+  while (content->data)
+    {
+      APR_ARRAY_PUSH(revprops->manifest, const char*) = content->data;
+      content->data = strchr(content->data, '\n');
+      if (content->data)
+        {
+          *content->data = 0;
+          content->data++;
+        }
+    }
+
+  /* Index for our revision. Rev 0 is excluded from the first shard. */
+  idx = (int)(revprops->revision % ffd->max_files_per_dir);
+  if (revprops->revision < ffd->max_files_per_dir)
+    --idx;
+
+  if (revprops->manifest->nelts <= idx)
+    return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                             _("Packed revprop manifest for rev %ld too "
+                               "small"), revprops->revision);
+
+  /* Now get the file name */
+  revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*);
+
+  return SVN_NO_ERROR;
+}
+
+/* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS,
+ * fill the START_REVISION, SIZES, OFFSETS members. Also, make
+ * PACKED_REVPROPS point to the first serialized revprop.
+ *
+ * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as
+ * well as the SERIALIZED_SIZE member.  If revprop caching has been
+ * enabled, parse all revprops in the pack and cache them.
+ */
+static svn_error_t *
+parse_packed_revprops(svn_fs_t *fs,
+                      packed_revprops_t *revprops,
+                      apr_pool_t *pool,
+                      apr_pool_t *scratch_pool)
+{
+  svn_stream_t *stream;
+  apr_int64_t first_rev, count, i;
+  apr_off_t offset;
+  const char *header_end;
+  apr_pool_t *iterpool = svn_pool_create(scratch_pool);
+
+  /* decompress (even if the data is only "stored", there is still a
+   * length header to remove) */
+  svn_string_t *compressed
+      = svn_stringbuf__morph_into_string(revprops->packed_revprops);
+  svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(pool);
+  SVN_ERR(svn__decompress(compressed, uncompressed, 0x1000000));
+
+  /* read first revision number and number of revisions in the pack */
+  stream = svn_stream_from_stringbuf(uncompressed, scratch_pool);
+  SVN_ERR(read_number_from_stream(&first_rev, NULL, stream, iterpool));
+  SVN_ERR(read_number_from_stream(&count, NULL, stream, iterpool));
+
+  /* make PACKED_REVPROPS point to the first char after the header.
+   * This is where the serialized revprops are. */
+  header_end = strstr(uncompressed->data, "\n\n");
+  if (header_end == NULL)
+    return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
+                            _("Header end not found"));
+
+  offset = header_end - uncompressed->data + 2;
+
+  revprops->packed_revprops = svn_stringbuf_create_empty(pool);
+  revprops->packed_revprops->data = uncompressed->data + offset;
+  revprops->packed_revprops->len = uncompressed->len - offset;
+  revprops->packed_revprops->blocksize = uncompressed->blocksize - offset;
+
+  /* STREAM still points to the first entry in the sizes list.
+   * Init / construct REVPROPS members. */
+  revprops->start_revision = (svn_revnum_t)first_rev;
+  revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset));
+  revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset));
+
+  /* Now parse, revision by revision, the size and content of each
+   * revisions' revprops. */
+  for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i)
+    {
+      apr_int64_t size;
+      svn_string_t serialized;
+      apr_hash_t *properties;
+      svn_revnum_t revision = (svn_revnum_t)(first_rev + i);
+
+      /* read & check the serialized size */
+      SVN_ERR(read_number_from_stream(&size, NULL, stream, iterpool));
+      if (size + offset > revprops->packed_revprops->len)
+        return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
+                        _("Packed revprop size exceeds pack file size"));
+
+      /* Parse this revprops list, if necessary */
+      serialized.data = revprops->packed_revprops->data + offset;
+      serialized.len = (apr_size_t)size;
+
+      if (revision == revprops->revision)
+        {
+          SVN_ERR(parse_revprop(&revprops->properties, fs, revision,
+                                revprops->generation, &serialized,
+                                pool, iterpool));
+          revprops->serialized_size = serialized.len;
+        }
+      else
+        {
+          /* If revprop caching is enabled, parse any revprops.
+           * They will get cached as a side-effect of this. */
+          if (has_revprop_cache(fs, pool))
+            SVN_ERR(parse_revprop(&properties, fs, revision,
+                                  revprops->generation, &serialized,
+                                  iterpool, iterpool));
+        }
+
+      /* fill REVPROPS data structures */
+      APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len;
+      APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset;
+      revprops->total_size += serialized.len;
+
+      offset += serialized.len;
+
+      svn_pool_clear(iterpool);
+    }
+
+  return SVN_NO_ERROR;
+}
+
+/* In filesystem FS, read the packed revprops for revision REV into
+ * *REVPROPS.  Use GENERATION to populate the revprop cache, if enabled.
+ * Allocate data in POOL.
+ */
+static svn_error_t *
+read_pack_revprop(packed_revprops_t **revprops,
                   svn_fs_t *fs,
                   svn_revnum_t rev,
+                  apr_int64_t generation,
                   apr_pool_t *pool)
 {
-  apr_hash_t *proplist;
+  apr_pool_t *iterpool = svn_pool_create(pool);
+  svn_boolean_t missing = FALSE;
+  svn_error_t *err;
+  packed_revprops_t *result;
+  int i;
+
+  /* someone insisted that REV is packed. Double-check if necessary */
+  if (!is_packed_revprop(fs, rev))
+     SVN_ERR(update_min_unpacked_rev(fs, iterpool));
+
+  if (!is_packed_revprop(fs, rev))
+    return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
+                              _("No such packed revision %ld"), rev);
+
+  /* initialize the result data structure */
+  result = apr_pcalloc(pool, sizeof(*result));
+  result->revision = rev;
+  result->generation = generation;
+
+  /* try to read the packed revprops. This may require retries if we have
+   * concurrent writers. */
+  for (i = 0; i < RECOVERABLE_RETRY_COUNT && !result->packed_revprops; ++i)
+    {
+      const char *file_path;
+
+      /* there might have been concurrent writes.
+       * Re-read the manifest and the pack file.
+       */
+      SVN_ERR(get_revprop_packname(fs, result, pool, iterpool));
+      file_path  = svn_dirent_join(result->folder,
+                                   result->filename,
+                                   iterpool);
+      SVN_ERR(try_stringbuf_from_file(&result->packed_revprops,
+                                      &missing,
+                                      file_path,
+                                      i + 1 < RECOVERABLE_RETRY_COUNT,
+                                      pool));
+
+      /* If we could not find the file, there was a write.
+       * So, we should refresh our revprop generation info as well such
+       * that others may find data we will put into the cache.  They would
+       * consider it outdated, otherwise.
+       */
+      if (missing && has_revprop_cache(fs, pool))
+        SVN_ERR(read_revprop_generation(&result->generation, fs, pool));
+
+      svn_pool_clear(iterpool);
+    }
+
+  /* the file content should be available now */
+  if (!result->packed_revprops)
+    return svn_error_createf(SVN_ERR_FS_PACKED_REPPROP_READ_FAILURE, NULL,
+                  _("Failed to read revprop pack file for rev %ld"), rev);
+
+  /* parse it. RESULT will be complete afterwards. */
+  err = parse_packed_revprops(fs, result, pool, iterpool);
+  svn_pool_destroy(iterpool);
+  if (err)
+    return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
+                  _("Revprop pack file for rev %ld is corrupt"), rev);
+
+  *revprops = result;
+
+  return SVN_NO_ERROR;
+}
+
+/* Read the revprops for revision REV in FS and return them in *PROPERTIES_P.
+ *
+ * Allocations will be done in POOL.
+ */
+static svn_error_t *
+get_revision_proplist(apr_hash_t **proplist_p,
+                      svn_fs_t *fs,
+                      svn_revnum_t rev,
+                      apr_pool_t *pool)
+{
   fs_fs_data_t *ffd = fs->fsap_data;
-  const char *key;
+  apr_int64_t generation = 0;
+
+  /* not found, yet */
+  *proplist_p = NULL;
 
+  /* should they be available at all? */
   SVN_ERR(ensure_revision_exists(fs, rev, pool));
 
   /* Try cache lookup first. */
   if (has_revprop_cache(fs, pool))
     {
-      apr_int64_t generation;
       svn_boolean_t is_cached;
+      const char *key;
 
       SVN_ERR(read_revprop_generation(&generation, fs, pool));
 
@@ -3201,65 +3669,481 @@ revision_proplist(apr_hash_t **proplist_
         return SVN_NO_ERROR;
     }
 
-  /* if (1); null condition for easier merging to revprop-packing */
+  /* if REV had not been packed when we began, try reading it from the
+   * non-packed shard.  If that fails, we will fall through to packed
+   * shard reads. */
+  if (!is_packed_revprop(fs, rev))
+    SVN_ERR(read_non_packed_revprop(proplist_p, fs, rev, generation,
+                                    pool));
+
+  /* if revprop packing is available and we have not read the revprops, yet,
+   * try reading them from a packed shard.  If that fails, REV is most
+   * likely invalid (or its revprops highly contested). */
+  if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p)
+    {
+      packed_revprops_t *packed_revprops;
+      SVN_ERR(read_pack_revprop(&packed_revprops, fs, rev, generation, pool));
+      *proplist_p = packed_revprops->properties;
+    }
+
+  /* The revprops should have been there. Did we get them? */
+  if (!*proplist_p)
+    return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
+                             _("Could not read revprops for revision %ld"),
+                             rev);
+
+  return SVN_NO_ERROR;
+}
+
+/* Serialize the revision property list PROPLIST of revision REV in
+ * filesystem FS to a non-packed file.  Return the name of that temporary
+ * file in *TMP_PATH and the file path that it must be moved to in
+ * *FINAL_PATH.
+ * 
+ * Use POOL for allocations.
+ */
+static svn_error_t *
+write_non_packed_revprop(const char **final_path,
+                         const char **tmp_path,
+                         svn_fs_t *fs,
+                         svn_revnum_t rev,
+                         apr_hash_t *proplist,
+                         apr_pool_t *pool)
+{
+  svn_stream_t *stream;
+  *final_path = path_revprops(fs, rev, pool);
+
+  /* ### do we have a directory sitting around already? we really shouldn't
+     ### have to get the dirname here. */
+  SVN_ERR(svn_stream_open_unique(&stream, tmp_path,
+                                 svn_dirent_dirname(*final_path, pool),
+                                 svn_io_file_del_none, pool, pool));
+  SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
+  SVN_ERR(svn_stream_close(stream));
+
+  return SVN_NO_ERROR;
+}
+
+/* After writing the new revprop file(s), call this function to move the
+ * file at TMP_PATH to FINAL_PATH and give it the permissions from
+ * PERMS_REFERENCE.
+ *
+ * If indicated in BUMP_GENERATION, increase FS' revprop generation.
+ * Finally, delete all the temporary files given in FILES_TO_DELETE.
+ * The latter may be NULL.
+ * 
+ * Use POOL for temporary allocations.
+ */
+static svn_error_t *
+switch_to_new_revprop(svn_fs_t *fs,
+                      const char *final_path,
+                      const char *tmp_path,
+                      const char *perms_reference,
+                      apr_array_header_t *files_to_delete,
+                      svn_boolean_t bump_generation,
+                      apr_pool_t *pool)
+{
+  /* Now, we may actually be replacing revprops. Make sure that all other
+     threads and processes will know about this. */
+  if (bump_generation)
+    SVN_ERR(begin_revprop_change(fs, pool));
+
+  SVN_ERR(move_into_place(tmp_path, final_path, perms_reference, pool));
+
+  /* Indicate that the update (if relevant) has been completed. */
+  if (bump_generation)
+    SVN_ERR(end_revprop_change(fs, pool));
+
+  /* Clean up temporary files, if necessary. */
+  if (files_to_delete)
+    {
+      apr_pool_t *iterpool = svn_pool_create(pool);
+      int i;
+      
+      for (i = 0; i < files_to_delete->nelts; ++i)
+        {
+          const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*);
+          SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
+          svn_pool_clear(iterpool);
+        }
+
+      svn_pool_destroy(iterpool);
+    }
+  return SVN_NO_ERROR;
+}
+
+/* Write a pack file header to STREAM that starts at revision START_REVISION
+ * and contains the indexes [START,END) of SIZES.
+ */
+static svn_error_t *
+serialize_revprops_header(svn_stream_t *stream,
+                          svn_revnum_t start_revision,
+                          apr_array_header_t *sizes,
+                          int start,
+                          int end,
+                          apr_pool_t *pool)
+{
+  apr_pool_t *iterpool = svn_pool_create(pool);
+  int i;
+
+  SVN_ERR_ASSERT(start < end);
+
+  /* start revision and entry count */
+  SVN_ERR(svn_stream_printf(stream, pool, "%ld\n", start_revision));
+  SVN_ERR(svn_stream_printf(stream, pool, "%d\n", end - start));
+
+  /* the sizes array */
+  for (i = start; i < end; ++i)
+    {
+      apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t);
+      SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n",
+                                size));
+    }
+
+  /* the double newline char indicates the end of the header */
+  SVN_ERR(svn_stream_printf(stream, iterpool, "\n"));
+
+  svn_pool_clear(iterpool);
+  return SVN_NO_ERROR;
+}
+
+/* Writes the a pack file to FILE_STREAM.  It copies the serialized data
+ * from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX.
+ * 
+ * The data for the latter is taken from NEW_SERIALIZED.  Note, that
+ * CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is
+ * taken in that case but only a subset of the old data will be copied.
+ *
+ * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size.
+ * POOL is used for temporary allocations.
+ */
+static svn_error_t *
+repack_revprops(svn_fs_t *fs,
+                packed_revprops_t *revprops,
+                int start,
+                int end,
+                int changed_index,
+                svn_stringbuf_t *new_serialized,
+                apr_off_t new_total_size,
+                svn_stream_t *file_stream,
+                apr_pool_t *pool)
+{
+  fs_fs_data_t *ffd = fs->fsap_data;
+  svn_stream_t *stream;
+  int i;
+
+  /* create data empty buffers and the stream object */
+  svn_stringbuf_t *uncompressed
+    = svn_stringbuf_create_ensure((apr_size_t)new_total_size, pool);
+  svn_stringbuf_t *compressed
+    = svn_stringbuf_create_empty(pool);
+  stream = svn_stream_from_stringbuf(uncompressed, pool);
+
+  /* write the header*/
+  SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start,
+                                    revprops->sizes, start, end, pool));
+
+  /* append the serialized revprops */
+  for (i = start; i < end; ++i)
+    if (i == changed_index)
+      {
+        SVN_ERR(svn_stream_write(stream,
+                                 new_serialized->data,
+                                 &new_serialized->len));
+      }
+    else
+      {
+        apr_size_t size 
+            = (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t);
+        apr_size_t offset 
+            = (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t);
+
+        SVN_ERR(svn_stream_write(stream,
+                                 revprops->packed_revprops->data + offset,
+                                 &size));
+      }
+
+  /* flush the stream buffer (if any) to our underlying data buffer */
+  SVN_ERR(svn_stream_close(stream));
+
+  /* compress / store the data */
+  SVN_ERR(svn__compress(svn_stringbuf__morph_into_string(uncompressed),
+                        compressed,
+                        ffd->compress_packed_revprops
+                          ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
+                          : SVN_DELTA_COMPRESSION_LEVEL_NONE));
+
+  /* finally, write the content to the target stream and close it */
+  SVN_ERR(svn_stream_write(file_stream, compressed->data, &compressed->len));
+  SVN_ERR(svn_stream_close(file_stream));
+
+  return SVN_NO_ERROR;
+}
+
+/* Allocate a new pack file name for the revisions at index [START,END)
+ * of REVPROPS->MANIFEST.  Add the name of old file to FILES_TO_DELETE,
+ * auto-create that array if necessary.  Return an open file stream to
+ * the new file in *STREAM allocated in POOL.
+ */
+static svn_error_t *
+repack_stream_open(svn_stream_t **stream,
+                   svn_fs_t *fs,
+                   packed_revprops_t *revprops,
+                   int start,
+                   int end,
+                   apr_array_header_t **files_to_delete,
+                   apr_pool_t *pool)
+{
+  apr_int64_t tag;
+  const char *tag_string;
+  svn_string_t *new_filename;
+  int i;
+  apr_file_t *file;
+
+  /* get the old (= current) file name and enlist it for later deletion */
+  const char *old_filename
+    = APR_ARRAY_IDX(revprops->manifest, start, const char*);
+
+  if (*files_to_delete == NULL)
+    *files_to_delete = apr_array_make(pool, 3, sizeof(const char*));
+
+  APR_ARRAY_PUSH(*files_to_delete, const char*)
+    = svn_dirent_join(revprops->folder, old_filename, pool);
+
+  /* increase the tag part, i.e. the counter after the dot */
+  tag_string = strchr(old_filename, '.');
+  if (tag_string == NULL)
+    return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                             _("Packed file '%s' misses a tag"),
+                             old_filename);
+    
+  SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1));
+  new_filename = svn_string_createf(pool, "%ld.%" APR_INT64_T_FMT,
+                                    revprops->start_revision + start,
+                                    ++tag);
+
+  /* update the manifest to point to the new file */
+  for (i = start; i < end; ++i)
+    APR_ARRAY_IDX(revprops->manifest, i, const char*) = new_filename->data;
+
+  /* create a file stream for the new file */
+  SVN_ERR(svn_io_file_open(&file, svn_dirent_join(revprops->folder,
+                                                  new_filename->data,
+                                                  pool),
+                           APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool));
+  *stream = svn_stream_from_aprfile2(file, FALSE, pool);
+
+  return SVN_NO_ERROR;
+}
+
+/* For revision REV in filesystem FS, set the revision properties to
+ * PROPLIST.  Return a new file in *TMP_PATH that the caller shall move
+ * to *FINAL_PATH to make the change visible.  Files to be deleted will
+ * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated.
+ * Use POOL for allocations.
+ */
+static svn_error_t *
+write_packed_revprop(const char **final_path,
+                     const char **tmp_path,
+                     apr_array_header_t **files_to_delete,
+                     svn_fs_t *fs,
+                     svn_revnum_t rev,
+                     apr_hash_t *proplist,
+                     apr_pool_t *pool)
+{
+  fs_fs_data_t *ffd = fs->fsap_data;
+  packed_revprops_t *revprops;
+  apr_int64_t generation = 0;
+  svn_stream_t *stream;
+  svn_stringbuf_t *serialized;
+  apr_off_t new_total_size;
+  int changed_index;
+
+  /* read the current revprop generation. This value will not change
+   * while we hold the global write lock to this FS. */
+  if (has_revprop_cache(fs, pool))
+    SVN_ERR(read_revprop_generation(&generation, fs, pool));
+
+  /* read contents of the current pack file */
+  SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, pool));
+
+  /* serialize the new revprops */
+  serialized = svn_stringbuf_create_empty(pool);
+  stream = svn_stream_from_stringbuf(serialized, pool);
+  SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
+  SVN_ERR(svn_stream_close(stream));
+
+  /* calculate the size of the new data */
+  changed_index = (int)(rev - revprops->start_revision);
+  new_total_size = revprops->total_size - revprops->serialized_size
+                 + serialized->len
+                 + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE;
+
+  APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len;
+
+  /* can we put the new data into the same pack as the before? */
+  if (   new_total_size < ffd->revprop_pack_size
+      || revprops->sizes->nelts == 1)
+    {
+      /* simply replace the old pack file with new content as we do it
+       * in the non-packed case */
+
+      *final_path = svn_dirent_join(revprops->folder, revprops->filename,
+                                    pool);
+      SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder,
+                                     svn_io_file_del_none, pool, pool));
+      SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts,
+                              changed_index, serialized, new_total_size,
+                              stream, pool));
+    }
+  else
     {
-      apr_file_t *revprop_file = NULL;
-      svn_error_t *err = SVN_NO_ERROR;
-      int i;
-      apr_pool_t *iterpool;
+      /* split the pack file into two of roughly equal size */
+      int right_count, left_count, i;
+          
+      int left = 0;
+      int right = revprops->sizes->nelts - 1;
+      apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE;
+      apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE;
+
+      /* let left and right side grow such that their size difference
+       * is minimal after each step. */
+      while (left <= right)
+        if (  left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t)
+            < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t))
+          {
+            left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t)
+                      + SVN_INT64_BUFFER_SIZE;
+            ++left;
+          }
+        else
+          {
+            right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)
+                        + SVN_INT64_BUFFER_SIZE;
+            --right;
+          }
+
+       /* since the items need much less than SVN_INT64_BUFFER_SIZE
+        * bytes to represent their length, the split may not be optimal */
+      left_count = left;
+      right_count = revprops->sizes->nelts - left;
+
+      /* if new_size is large, one side may exceed the pack size limit.
+       * In that case, split before and after the modified revprop.*/
+      if (   left_size > ffd->revprop_pack_size
+          || right_size > ffd->revprop_pack_size)
+        {
+          left_count = changed_index;
+          right_count = revprops->sizes->nelts - left_count - 1;
+        }
+
+      /* write the new, split files */
+      if (left_count)
+        {
+          SVN_ERR(repack_stream_open(&stream, fs, revprops, 0,
+                                     left_count, files_to_delete, pool));
+          SVN_ERR(repack_revprops(fs, revprops, 0, left_count,
+                                  changed_index, serialized, new_total_size,
+                                  stream, pool));
+        }
 
-      proplist = apr_hash_make(pool);
-      iterpool = svn_pool_create(pool);
-      for (i = 0; RECOVERABLE_RETRY_TEST; RECOVERABLE_RETRY_NEXT)
+      if (left_count + right_count < revprops->sizes->nelts)
         {
-          svn_pool_clear(iterpool);
-
-          /* Clear err here rather than after finding a recoverable error so
-           * we can return that error on the last iteration of the loop. */
-          svn_error_clear(err);
-          err = svn_io_file_open(&revprop_file, path_revprops(fs, rev,
-                                                              iterpool),
-                                 APR_READ | APR_BUFFERED, APR_OS_DEFAULT,
-                                 iterpool);
-          if (err)
-            {
-              if (APR_STATUS_IS_ENOENT(err->apr_err))
-                {
-                  svn_error_clear(err);
-                  return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
-                                           _("No such revision %ld"), rev);
-                }
-#ifdef ESTALE
-              else if (APR_TO_OS_ERROR(err->apr_err) == ESTALE
-                       || APR_TO_OS_ERROR(err->apr_err) == EIO
-                       || APR_TO_OS_ERROR(err->apr_err) == ENOENT)
-                continue;
-#endif
-              return svn_error_trace(err);
-            }
-
-          SVN_ERR(svn_hash__clear(proplist, iterpool));
-          RETRY_RECOVERABLE(err, revprop_file,
-                            svn_hash_read2(proplist,
-                                           svn_stream_from_aprfile2(
-                                                revprop_file, TRUE, iterpool),
-                                           SVN_HASH_TERMINATOR, pool));
+          SVN_ERR(repack_stream_open(&stream, fs, revprops, changed_index,
+                                     changed_index + 1, files_to_delete,
+                                     pool));
+          SVN_ERR(repack_revprops(fs, revprops, changed_index,
+                                  changed_index + 1,
+                                  changed_index, serialized, new_total_size,
+                                  stream, pool));
+        }
+
+      if (right_count)
+        {
+          SVN_ERR(repack_stream_open(&stream, fs, revprops,
+                                     revprops->sizes->nelts - right_count,
+                                     revprops->sizes->nelts,
+                                     files_to_delete, pool));
+          SVN_ERR(repack_revprops(fs, revprops,
+                                  revprops->sizes->nelts - right_count,
+                                  revprops->sizes->nelts, changed_index,
+                                  serialized, new_total_size, stream,
+                                  pool));
+        }
 
-          IGNORE_RECOVERABLE(err, svn_io_file_close(revprop_file, iterpool));
+      /* write the new manifest */
+      *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool);
+      SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder,
+                                     svn_io_file_del_none, pool, pool));
 
-          break;
+      for (i = 0; i < revprops->manifest->nelts; ++i)
+        {
+          const char *filename = APR_ARRAY_IDX(revprops->manifest, i,
+                                               const char*);
+          SVN_ERR(svn_stream_printf(stream, pool, "%s\n", filename));
         }
 
-      if (err)
-        return svn_error_trace(err);
-      svn_pool_destroy(iterpool);
+      SVN_ERR(svn_stream_close(stream));
     }
+  
+  return SVN_NO_ERROR;
+}
+
+/* Set the revision property list of revision REV in filesystem FS to
+   PROPLIST.  Use POOL for temporary allocations. */
+static svn_error_t *
+set_revision_proplist(svn_fs_t *fs,
+                      svn_revnum_t rev,
+                      apr_hash_t *proplist,
+                      apr_pool_t *pool)
+{
+  svn_boolean_t is_packed;
+  svn_boolean_t bump_generation = FALSE;
+  const char *final_path;
+  const char *tmp_path;
+  const char *perms_reference;
+  apr_array_header_t *files_to_delete = NULL;
+
+  SVN_ERR(ensure_revision_exists(fs, rev, pool));
 
-  /* Cache the result, if caching has been activated. */
+  /* this info will not change while we hold the global FS write lock */
+  is_packed = is_packed_revprop(fs, rev);
+  
+  /* Test whether revprops already exist for this revision.
+   * Only then will we need to bump the revprop generation. */
   if (has_revprop_cache(fs, pool))
-    SVN_ERR(svn_cache__set(ffd->revprop_cache, key, proplist, pool));
+    {
+      if (is_packed)
+        {
+          bump_generation = TRUE;
+        }
+      else
+        {
+          svn_node_kind_t kind;
+          SVN_ERR(svn_io_check_path(path_revprops(fs, rev, pool), &kind,
+                                    pool));
+          bump_generation = kind != svn_node_none;
+        }
+    }
 
-  *proplist_p = proplist;
+  /* Serialize the new revprop data */
+  if (is_packed)
+    SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete,
+                                 fs, rev, proplist, pool));
+  else
+    SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path,
+                                     fs, rev, proplist, pool));
+
+  /* We use the rev file of this revision as the perms reference,
+   * because when setting revprops for the first time, the revprop
+   * file won't exist and therefore can't serve as its own reference.
+   * (Whereas the rev file should already exist at this point.)
+   */
+  SVN_ERR(svn_fs_fs__path_rev_absolute(&perms_reference, fs, rev, pool));
+
+  /* Now, switch to the new revprop data. */
+  SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference,
+                                files_to_delete, bump_generation, pool));
 
   return SVN_NO_ERROR;
 }
@@ -3270,7 +4154,7 @@ svn_fs_fs__revision_proplist(apr_hash_t 
                              svn_revnum_t rev,
                              apr_pool_t *pool)
 {
-  SVN_ERR(revision_proplist(proplist_p, fs, rev, pool));
+  SVN_ERR(get_revision_proplist(proplist_p, fs, rev, pool));
 
   return SVN_NO_ERROR;
 }
@@ -5034,40 +5918,17 @@ get_and_increment_txn_key_body(void *bat
 {
   struct get_and_increment_txn_key_baton *cb = baton;
   const char *txn_current_filename = path_txn_current(cb->fs, pool);
-  apr_file_t *txn_current_file = NULL;
   const char *tmp_filename;
   char next_txn_id[MAX_KEY_SIZE+3];
-  svn_error_t *err = SVN_NO_ERROR;
-  apr_pool_t *iterpool;
   apr_size_t len;
-  int i;
-
-  cb->txn_id = apr_palloc(cb->pool, MAX_KEY_SIZE);
-
-  iterpool = svn_pool_create(pool);
-  for (i = 0; RECOVERABLE_RETRY_TEST; RECOVERABLE_RETRY_NEXT)
-    {
-      svn_pool_clear(iterpool);
-
-      RETRY_RECOVERABLE(err, txn_current_file,
-                        svn_io_file_open(&txn_current_file,
-                                         txn_current_filename,
-                                         APR_READ | APR_BUFFERED,
-                                         APR_OS_DEFAULT, iterpool));
-      len = MAX_KEY_SIZE;
-      RETRY_RECOVERABLE(err, txn_current_file,
-                        svn_io_read_length_line(txn_current_file,
-                                                cb->txn_id,
-                                                &len,
-                                                iterpool));
-      IGNORE_RECOVERABLE(err, svn_io_file_close(txn_current_file,
-                                                iterpool));
 
-      break;
-    }
-  SVN_ERR(err);
+  svn_stringbuf_t *buf;
+  SVN_ERR(read_content(&buf, txn_current_filename, cb->pool));
 
-  svn_pool_destroy(iterpool);
+  /* remove trailing newlines */
+  svn_stringbuf_strip_whitespace(buf);
+  cb->txn_id = buf->data;
+  len = buf->len;
 
   /* Increment the key and add a trailing \n to the string so the
      txn-current file has a newline in it. */
@@ -6155,8 +7016,10 @@ get_next_revision_ids(const char **node_
 {
   char *buf;
   char *str;
+  svn_stringbuf_t *content;
 
-  SVN_ERR(read_current(svn_fs_fs__path_current(fs, pool), &buf, pool));
+  SVN_ERR(read_content(&content, svn_fs_fs__path_current(fs, pool), pool));
+  buf = content->data;
 
   str = svn_cstring_tokenize(" ", &buf);
   if (! str)
@@ -6975,7 +7838,7 @@ commit_body(void *baton, apr_pool_t *poo
      fails because the shard already existed for some reason. */
   if (ffd->max_files_per_dir && new_rev % ffd->max_files_per_dir == 0)
     {
-      /* if (1); null condition for easier merging to revprop-packing */
+      /* Create the revs shard. */
         {
           const char *new_dir = path_rev_shard(cb->fs, new_rev, pool);
           svn_error_t *err = svn_io_dir_make(new_dir, APR_OS_DEFAULT, pool);
@@ -7205,6 +8068,9 @@ svn_fs_fs__create(svn_fs_t *fs,
       else if (apr_hash_get(fs->config, SVN_FS_CONFIG_PRE_1_6_COMPATIBLE,
                                         APR_HASH_KEY_STRING))
         format = 3;
+      else if (apr_hash_get(fs->config, SVN_FS_CONFIG_PRE_1_8_COMPATIBLE,
+                                        APR_HASH_KEY_STRING))
+        format = 4;
     }
   ffd->format = format;
 
@@ -7253,7 +8119,7 @@ svn_fs_fs__create(svn_fs_t *fs,
 
   SVN_ERR(write_config(fs, pool));
 
-  SVN_ERR(read_config(fs, pool));
+  SVN_ERR(read_config(ffd, fs->path, pool));
 
   /* Create the min unpacked rev file. */
   if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
@@ -7520,6 +8386,77 @@ recover_find_max_ids(svn_fs_t *fs, svn_r
   return SVN_NO_ERROR;
 }
 
+/* Return TRUE, if for REVISION in FS, we can find the revprop pack file.
+ * Use POOL for temporary allocations.
+ * Set *MISSING, if the reason is a missing manifest or pack file. 
+ */
+static svn_boolean_t
+packed_revprop_available(svn_boolean_t *missing,
+                         svn_fs_t *fs,
+                         svn_revnum_t revision,
+                         apr_pool_t *pool)
+{
+  fs_fs_data_t *ffd = fs->fsap_data;
+  svn_stringbuf_t *content = NULL;
+
+  /* try to read the manifest file */
+  const char *folder = path_revprops_pack_shard(fs, revision, pool);
+  const char *manifest_path = svn_dirent_join(folder, PATH_MANIFEST, pool);
+
+  svn_error_t *err = try_stringbuf_from_file(&content,
+                                             missing,
+                                             manifest_path,
+                                             FALSE,
+                                             pool);
+
+  /* if the manifest cannot be read, consider the pack files inaccessible
+   * even if the file itself exists. */
+  if (err)
+    {
+      svn_error_clear(err);
+      return FALSE;
+    }
+
+  if (*missing)
+    return FALSE;
+
+  /* parse manifest content until we find the entry for REVISION.
+   * Revision 0 is never packed. */
+  revision = revision < ffd->max_files_per_dir
+           ? revision - 1
+           : revision % ffd->max_files_per_dir;
+  while (content->data)
+    {
+      char *next = strchr(content->data, '\n');
+      if (next)
+        {
+          *next = 0;
+          ++next;
+        }
+
+      if (revision-- == 0)
+        {
+          /* the respective pack file must exist (and be a file) */
+          svn_node_kind_t kind;
+          err = svn_io_check_path(svn_dirent_join(folder, content->data,
+                                                  pool),
+                                  &kind, pool);
+          if (err)
+            {
+              svn_error_clear(err);
+              return FALSE;
+            }
+
+          *missing = kind == svn_node_none;
+          return kind == svn_node_file;
+        }
+
+      content->data = next;
+    }
+
+  return FALSE;
+}
+
 /* Baton used for recover_body below. */
 struct recover_baton {
   svn_fs_t *fs;
@@ -7633,13 +8570,24 @@ recover_body(void *baton, apr_pool_t *po
                             &youngest_revprops_kind, pool));
   if (youngest_revprops_kind == svn_node_none)
     {
-      /* if (1); null condition for easier merging to revprop-packing */
-        {
-          return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
-                                   _("Revision %ld has a revs file but no "
-                                     "revprops file"),
-                                   max_rev);
-        }
+      svn_boolean_t missing = TRUE;
+      if (!packed_revprop_available(&missing, fs, max_rev, pool))
+	{
+          if (missing)
+            {
+              return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                                      _("Revision %ld has a revs file but no "
+                                        "revprops file"),
+                                      max_rev);
+            }
+          else
+            {
+              return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+                                      _("Revision %ld has a revs file but the "
+                                        "revprops file is inaccessible"),
+                                      max_rev);
+            }
+          }
     }
   else if (youngest_revprops_kind != svn_node_file)
     {
@@ -8180,43 +9128,31 @@ write_revnum_file(const char *fs_path,
   return SVN_NO_ERROR;
 }
 
-/* Pack a single shard SHARD in REVS_DIR, using POOL for allocations.
-   CANCEL_FUNC and CANCEL_BATON are what you think they are.
-
-   If for some reason we detect a partial packing already performed, we
-   remove the pack file and start again. */
+/* Pack the revision SHARD containing exactly MAX_FILES_PER_DIR revisions
+ * from SHARD_PATH into the PACK_FILE_DIR, using POOL for allocations.
+ * CANCEL_FUNC and CANCEL_BATON are what you think they are.
+ *
+ * If for some reason we detect a partial packing already performed, we
+ * remove the pack file and start again.
+ */
 static svn_error_t *
-pack_shard(const char *revs_dir,
-           const char *fs_path,
-           apr_int64_t shard,
-           int max_files_per_dir,
-           svn_fs_pack_notify_t notify_func,
-           void *notify_baton,
-           svn_cancel_func_t cancel_func,
-           void *cancel_baton,
-           apr_pool_t *pool)
+pack_rev_shard(const char *pack_file_dir,
+               const char *shard_path,
+               apr_int64_t shard,
+               int max_files_per_dir,
+               svn_cancel_func_t cancel_func,
+               void *cancel_baton,
+               apr_pool_t *pool)
 {
-  const char *pack_file_path, *manifest_file_path, *shard_path;
-  const char *pack_file_dir;
+  const char *pack_file_path, *manifest_file_path;
   svn_stream_t *pack_stream, *manifest_stream;
   svn_revnum_t start_rev, end_rev, rev;
   apr_off_t next_offset;
   apr_pool_t *iterpool;
 
   /* Some useful paths. */
-  pack_file_dir = svn_dirent_join(revs_dir,
-                        apr_psprintf(pool, "%" APR_INT64_T_FMT ".pack", shard),
-                        pool);
-  pack_file_path = svn_dirent_join(pack_file_dir, "pack", pool);
-  manifest_file_path = svn_dirent_join(pack_file_dir, "manifest", pool);
-  shard_path = svn_dirent_join(revs_dir,
-                               apr_psprintf(pool, "%" APR_INT64_T_FMT, shard),
-                               pool);
-
-  /* Notify caller we're starting to pack this shard. */
-  if (notify_func)
-    SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start,
-                        pool));
+  pack_file_path = svn_dirent_join(pack_file_dir, PATH_PACKED, pool);
+  manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, pool);
 
   /* Remove any existing pack file for this shard, since it is incomplete. */
   SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton,
@@ -8262,21 +9198,338 @@ pack_shard(const char *revs_dir,
 
   SVN_ERR(svn_stream_close(manifest_stream));
   SVN_ERR(svn_stream_close(pack_stream));
-  SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, pool));
-  SVN_ERR(svn_io_set_file_read_only(pack_file_path, FALSE, pool));
-  SVN_ERR(svn_io_set_file_read_only(manifest_file_path, FALSE, pool));
+  SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool));
+  SVN_ERR(svn_io_set_file_read_only(pack_file_path, FALSE, iterpool));
+  SVN_ERR(svn_io_set_file_read_only(manifest_file_path, FALSE, iterpool));
+
+  svn_pool_destroy(iterpool);
+
+  return SVN_NO_ERROR;
+}
+
+/* Copy revprop files for revisions [START_REV, END_REV) from SHARD_PATH
+ * to the pack file at PACK_FILE_NAME in PACK_FILE_DIR.
+ *
+ * The file sizes have already been determined and written to SIZES.
+ * Please note that this function will be executed while the filesystem
+ * has been locked and that revprops files will therefore not be modified
+ * while the pack is in progress.
+ *
+ * COMPRESSION_LEVEL defines how well the resulting pack file shall be
+ * compressed or whether is shall be compressed at all.  TOTAL_SIZE is
+ * a hint on which initial buffer size we should use to hold the pack file
+ * content.
+ *
+ * CANCEL_FUNC and CANCEL_BATON are used as usual. Temporary allocations
+ * are done in SCRATCH_POOL.
+ */
+static svn_error_t *
+copy_revprops(const char *pack_file_dir,
+              const char *pack_filename,
+              const char *shard_path,
+              svn_revnum_t start_rev,
+              svn_revnum_t end_rev,
+              apr_array_header_t *sizes,
+              apr_size_t total_size,
+              int compression_level,
+              svn_cancel_func_t cancel_func,
+              void *cancel_baton,
+              apr_pool_t *scratch_pool)
+{
+  svn_stream_t *pack_stream;
+  apr_file_t *pack_file;
+  svn_revnum_t rev;
+  apr_pool_t *iterpool = svn_pool_create(scratch_pool);
+  svn_stream_t *stream;
+
+  /* create empty data buffer and a write stream on top of it */
+  svn_stringbuf_t *uncompressed
+    = svn_stringbuf_create_ensure(total_size, scratch_pool);
+  svn_stringbuf_t *compressed
+    = svn_stringbuf_create_empty(scratch_pool);
+  pack_stream = svn_stream_from_stringbuf(uncompressed, scratch_pool);
+
+  /* write the pack file header */
+  SVN_ERR(serialize_revprops_header(pack_stream, start_rev, sizes, 0,
+                                    sizes->nelts, iterpool));
+
+  /* Some useful paths. */
+  SVN_ERR(svn_io_file_open(&pack_file, svn_dirent_join(pack_file_dir,
+                                                       pack_filename,
+                                                       scratch_pool),
+                           APR_WRITE | APR_CREATE, APR_OS_DEFAULT,
+                           scratch_pool));
+
+  /* Iterate over the revisions in this shard, squashing them together. */
+  for (rev = start_rev; rev <= end_rev; rev++)
+    {
+      const char *path;
+
+      svn_pool_clear(iterpool);
+
+      /* Construct the file name. */
+      path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
+                             iterpool);
+
+      /* Copy all the bits from the non-packed revprop file to the end of
+       * the pack file. */
+      SVN_ERR(svn_stream_open_readonly(&stream, path, iterpool, iterpool));
+      SVN_ERR(svn_stream_copy3(stream, pack_stream,
+                               cancel_func, cancel_baton, iterpool));
+    }
+
+  /* flush stream buffers to content buffer */
+  SVN_ERR(svn_stream_close(pack_stream));
+
+  /* compress the content (or just store it for COMPRESSION_LEVEL 0) */
+  SVN_ERR(svn__compress(svn_stringbuf__morph_into_string(uncompressed),
+                        compressed, compression_level));
+
+  /* write the pack file content to disk */
+  stream = svn_stream_from_aprfile2(pack_file, FALSE, scratch_pool);
+  SVN_ERR(svn_stream_write(stream, compressed->data, &compressed->len));
+  SVN_ERR(svn_stream_close(stream));
+
+  svn_pool_destroy(iterpool);
+
+  return SVN_NO_ERROR;
+}
+
+/* For the revprop SHARD at SHARD_PATH with exactly MAX_FILES_PER_DIR
+ * revprop files in it, create a packed shared at PACK_FILE_DIR.
+ *
+ * COMPRESSION_LEVEL defines how well the resulting pack file shall be
+ * compressed or whether is shall be compressed at all.  Individual pack
+ * file containing more than one revision will be limited to a size of
+ * MAX_PACK_SIZE bytes before compression.
+ *
+ * CANCEL_FUNC and CANCEL_BATON are used in the usual way.  Temporary
+ * allocations are done in SCRATCH_POOL.
+ */
+static svn_error_t *
+pack_revprops_shard(const char *pack_file_dir,
+                    const char *shard_path,
+                    apr_int64_t shard,
+                    int max_files_per_dir,
+                    apr_off_t max_pack_size,
+                    int compression_level,
+                    svn_cancel_func_t cancel_func,
+                    void *cancel_baton,
+                    apr_pool_t *scratch_pool)
+{
+  const char *manifest_file_path, *pack_filename = NULL;
+  svn_stream_t *manifest_stream;
+  svn_revnum_t start_rev, end_rev, rev;
+  apr_off_t total_size;
+  apr_pool_t *iterpool = svn_pool_create(scratch_pool);
+  apr_array_header_t *sizes;
+
+  /* Some useful paths. */
+  manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST,
+                                       scratch_pool);
+
+  /* Remove any existing pack file for this shard, since it is incomplete. */
+  SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton,
+                             scratch_pool));
+
+  /* Create the new directory and manifest file stream. */
+  SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool));
+  SVN_ERR(svn_stream_open_writable(&manifest_stream, manifest_file_path,
+                                   scratch_pool, scratch_pool));
+
+  /* revisions to handle. Special case: revision 0 */
+  start_rev = (svn_revnum_t) (shard * max_files_per_dir);
+  end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1);
+  if (start_rev == 0)
+    ++start_rev;
+
+  /* initialize the revprop size info */
+  sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_off_t));
+  total_size = 2 * SVN_INT64_BUFFER_SIZE;
+
+  /* Iterate over the revisions in this shard, determine their size and
+   * squashing them together into pack files. */
+  for (rev = start_rev; rev <= end_rev; rev++)
+    {
+      apr_finfo_t finfo;
+      const char *path;
+
+      svn_pool_clear(iterpool);
+
+      /* Get the size of the file. */
+      path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
+                             iterpool);
+      SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool));
+
+      /* if we already have started a pack file and this revprop cannot be
+       * appended to it, write the previous pack file. */
+      if (sizes->nelts != 0 &&
+          total_size + SVN_INT64_BUFFER_SIZE + finfo.size > max_pack_size)
+        {
+          SVN_ERR(copy_revprops(pack_file_dir, pack_filename, shard_path,
+                                start_rev, rev-1, sizes, total_size,
+                                compression_level, cancel_func, cancel_baton,
+                                iterpool));
+
+          /* next pack file starts empty again */
+          apr_array_clear(sizes);
+          total_size = 2 * SVN_INT64_BUFFER_SIZE;
+          start_rev = rev;
+        }
+
+      /* Update the manifest. Allocate a file name for the current pack
+       * file if it is a new one */
+      if (sizes->nelts == 0)
+        pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev);
+
+      SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%s\n",
+                                pack_filename));
+
+      /* add to list of files to put into the current pack file */
+      APR_ARRAY_PUSH(sizes, apr_off_t) = finfo.size;
+      total_size += SVN_INT64_BUFFER_SIZE + finfo.size;
+    }
+
+  /* write the last pack file */
+  if (sizes->nelts != 0)
+    SVN_ERR(copy_revprops(pack_file_dir, pack_filename, shard_path,
+                          start_rev, rev-1, sizes, total_size,
+                          compression_level, cancel_func, cancel_baton,
+                          iterpool));
+
+  /* flush the manifest file and update permissions */
+  SVN_ERR(svn_stream_close(manifest_stream));
+  SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool));
+
+  svn_pool_destroy(iterpool);
+
+  return SVN_NO_ERROR;
+}
+
+/* Delete the non-packed revprop SHARD at SHARD_PATH with exactly
+ * MAX_FILES_PER_DIR revprop files in it.  If this is shard 0, keep the
+ * revprop file for revision 0.
+ *
+ * CANCEL_FUNC and CANCEL_BATON are used in the usual way.  Temporary
+ * allocations are done in SCRATCH_POOL.
+ */
+static svn_error_t *
+delete_revprops_shard(const char *shard_path,
+                      apr_int64_t shard,
+                      int max_files_per_dir,
+                      svn_cancel_func_t cancel_func,
+                      void *cancel_baton,
+                      apr_pool_t *scratch_pool)
+{
+  if (shard == 0)
+    {
+      apr_pool_t *iterpool = svn_pool_create(scratch_pool);
+      int i;
+
+      /* delete all files except the one for revision 0 */
+      for (i = 1; i < max_files_per_dir; ++i)
+        {
+          const char *path = svn_dirent_join(shard_path,
+                                       apr_psprintf(iterpool, "%d", i),
+                                       iterpool);
+          if (cancel_func)
+            SVN_ERR((*cancel_func)(cancel_baton));
+
+          SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
+          svn_pool_clear(iterpool);
+        }
+
+      svn_pool_destroy(iterpool);
+    }
+  else
+    SVN_ERR(svn_io_remove_dir2(shard_path, TRUE,
+                               cancel_func, cancel_baton, scratch_pool));
+
+  return SVN_NO_ERROR;
+}
+
+/* In the file system at FS_PATH, pack the SHARD in REVS_DIR and
+ * REVPROPS_DIR containing exactly MAX_FILES_PER_DIR revisions, using POOL
+ * for allocations.  REVPROPS_DIR will be NULL if revprop packing is not
+ * supported.  COMPRESSION_LEVEL and MAX_PACK_SIZE will be ignored in that
+ * case.
+ * 
+ * CANCEL_FUNC and CANCEL_BATON are what you think they are; similarly
+ * NOTIFY_FUNC and NOTIFY_BATON.
+ *
+ * If for some reason we detect a partial packing already performed, we
+ * remove the pack file and start again.
+ */
+static svn_error_t *
+pack_shard(const char *revs_dir,
+           const char *revsprops_dir,
+           const char *fs_path,
+           apr_int64_t shard,
+           int max_files_per_dir,
+           apr_off_t max_pack_size,
+           int compression_level,
+           svn_fs_pack_notify_t notify_func,
+           void *notify_baton,
+           svn_cancel_func_t cancel_func,
+           void *cancel_baton,
+           apr_pool_t *pool)
+{
+  const char *rev_shard_path, *rev_pack_file_dir;
+  const char *revprops_shard_path, *revprops_pack_file_dir;
+
+  /* Notify caller we're starting to pack this shard. */
+  if (notify_func)
+    SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start,
+                        pool));
+
+  /* Some useful paths. */
+  rev_pack_file_dir = svn_dirent_join(revs_dir,
+                  apr_psprintf(pool,
+                               "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
+                               shard),
+                  pool);
+  rev_shard_path = svn_dirent_join(revs_dir,
+                           apr_psprintf(pool, "%" APR_INT64_T_FMT, shard),
+                           pool);
+
+  /* pack the revision content */
+  SVN_ERR(pack_rev_shard(rev_pack_file_dir, rev_shard_path,
+                         shard, max_files_per_dir,
+                         cancel_func, cancel_baton, pool));
+
+  /* if enabled, pack the revprops in an equivalent way */
+  if (revsprops_dir)
+    {
+      revprops_pack_file_dir = svn_dirent_join(revsprops_dir,
+                   apr_psprintf(pool,
+                                "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
+                                shard),
+                   pool);
+      revprops_shard_path = svn_dirent_join(revsprops_dir,
+                           apr_psprintf(pool, "%" APR_INT64_T_FMT, shard),
+                           pool);
+
+      SVN_ERR(pack_revprops_shard(revprops_pack_file_dir, revprops_shard_path,
+                                  shard, max_files_per_dir,
+                                  (int)(0.9 * max_pack_size),
+                                  compression_level,
+                                  cancel_func, cancel_baton, pool));
+    }
 
   /* Update the min-unpacked-rev file to reflect our newly packed shard.
    * (This doesn't update ffd->min_unpacked_rev.  That will be updated by
    * update_min_unpacked_rev() when necessary.) */
   SVN_ERR(write_revnum_file(fs_path, PATH_MIN_UNPACKED_REV,
                             (svn_revnum_t)((shard + 1) * max_files_per_dir),
-                            iterpool));
-  svn_pool_destroy(iterpool);
+                            pool));
 
-  /* Finally, remove the existing shard directory. */
-  SVN_ERR(svn_io_remove_dir2(shard_path, TRUE, cancel_func, cancel_baton,
-                             pool));
+  /* Finally, remove the existing shard directories. */
+  SVN_ERR(svn_io_remove_dir2(rev_shard_path, TRUE,
+                             cancel_func, cancel_baton, pool));
+  if (revsprops_dir)
+    SVN_ERR(delete_revprops_shard(revprops_shard_path,
+                                  shard, max_files_per_dir,
+                                  cancel_func, cancel_baton, pool));
 
   /* Notify caller we're starting to pack this shard. */
   if (notify_func)
@@ -8307,8 +9560,7 @@ struct pack_baton
      extension, on not having to use a retry when calling
      svn_fs_fs__path_rev_absolute() and friends).  If you add a call
      to this function, consider whether you have to call
-     update_min_unpacked_rev() and update_min_unpacked_revprop()
-     afterwards.
+     update_min_unpacked_rev().
      See this thread: http://thread.gmane.org/1291206765.3782.3309.camel@edith
  */
 static svn_error_t *
@@ -8316,51 +9568,63 @@ pack_body(void *baton,
           apr_pool_t *pool)
 {
   struct pack_baton *pb = baton;
-  int format, max_files_per_dir;
+  fs_fs_data_t ffd = {0};
   apr_int64_t completed_shards;
   apr_int64_t i;
   svn_revnum_t youngest;
   apr_pool_t *iterpool;
-  const char *data_path;
-  svn_revnum_t min_unpacked_rev;
+  const char *rev_data_path;
+  const char *revprops_data_path = NULL;
 
-  SVN_ERR(read_format(&format, &max_files_per_dir, path_format(pb->fs, pool),
-                      pool));
-  SVN_ERR(check_format(format));
+  /* read repository settings */
+  SVN_ERR(read_format(&ffd.format, &ffd.max_files_per_dir,
+                      path_format(pb->fs, pool), pool));
+  SVN_ERR(check_format(ffd.format));
+  SVN_ERR(read_config(&ffd, pb->fs->path, pool));
 
   /* If the repository isn't a new enough format, we don't support packing.
      Return a friendly error to that effect. */
-  if (format < SVN_FS_FS__MIN_PACKED_FORMAT)
+  if (ffd.format < SVN_FS_FS__MIN_PACKED_FORMAT)
     return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
       _("FSFS format (%d) too old to pack; please upgrade the filesystem."),
-      format);
+      ffd.format);
 
   /* If we aren't using sharding, we can't do any packing, so quit. */
-  if (!max_files_per_dir)
+  if (!ffd.max_files_per_dir)
     return SVN_NO_ERROR;
 
-  SVN_ERR(read_min_unpacked_rev(&min_unpacked_rev,
+  SVN_ERR(read_min_unpacked_rev(&ffd.min_unpacked_rev,
                                 path_min_unpacked_rev(pb->fs, pool),
                                 pool));
 
   SVN_ERR(get_youngest(&youngest, pb->fs->path, pool));
-  completed_shards = (youngest + 1) / max_files_per_dir;
+  completed_shards = (youngest + 1) / ffd.max_files_per_dir;
 
   /* See if we've already completed all possible shards thus far. */
-  if (min_unpacked_rev == (completed_shards * max_files_per_dir))
+  if (ffd.min_unpacked_rev == (completed_shards * ffd.max_files_per_dir))
     return SVN_NO_ERROR;
 
-  data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, pool);
+  rev_data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, pool);
+  if (ffd.format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
+    revprops_data_path = svn_dirent_join(pb->fs->path, PATH_REVPROPS_DIR,
+                                         pool);
 
   iterpool = svn_pool_create(pool);
-  for (i = min_unpacked_rev / max_files_per_dir; i < completed_shards; i++)
+  for (i = ffd.min_unpacked_rev / ffd.max_files_per_dir;
+       i < completed_shards;
+       i++)
     {
       svn_pool_clear(iterpool);
 
       if (pb->cancel_func)
         SVN_ERR(pb->cancel_func(pb->cancel_baton));
 
-      SVN_ERR(pack_shard(data_path, pb->fs->path, i, max_files_per_dir,
+      SVN_ERR(pack_shard(rev_data_path, revprops_data_path,
+                         pb->fs->path, i, ffd.max_files_per_dir,
+                         ffd.revprop_pack_size,
+                         ffd.compress_packed_revprops
+                           ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
+                           : SVN_DELTA_COMPRESSION_LEVEL_NONE,
                          pb->notify_func, pb->notify_baton,
                          pb->cancel_func, pb->cancel_baton, iterpool));
     }
@@ -8720,7 +9984,7 @@ hotcopy_copy_packed_shard(svn_revnum_t *
   /* Copy the packed shard. */
   src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, scratch_pool);
   dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, scratch_pool);
-  packed_shard = apr_psprintf(scratch_pool, "%ld.pack",
+  packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD,
                               rev / max_files_per_dir);
   src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard,
                                             scratch_pool);

Modified: subversion/branches/javahl-ra/subversion/libsvn_fs_fs/rep-cache.c
URL: http://svn.apache.org/viewvc/subversion/branches/javahl-ra/subversion/libsvn_fs_fs/rep-cache.c?rev=1425415&r1=1425414&r2=1425415&view=diff
==============================================================================
--- subversion/branches/javahl-ra/subversion/libsvn_fs_fs/rep-cache.c (original)
+++ subversion/branches/javahl-ra/subversion/libsvn_fs_fs/rep-cache.c Sun Dec 23 06:03:14 2012
@@ -158,11 +158,11 @@ svn_fs_fs__walk_rep_reference(svn_fs_t *
                                         STMT_GET_MAX_REV));
       SVN_ERR(svn_sqlite__step(&have_row, stmt2));
       max = svn_sqlite__column_revnum(stmt2, 0);
-      SVN_ERR(svn_fs_fs__revision_exists(max, fs, iterpool));
+      if (SVN_IS_VALID_REVNUM(max))  /* The rep-cache could be empty. */
+        SVN_ERR(svn_fs_fs__revision_exists(max, fs, iterpool));
       SVN_ERR(svn_sqlite__reset(stmt2));
     }
 
-  /* Get the statement. (There are no arguments to bind.) */
   SVN_ERR(svn_sqlite__get_statement(&stmt, ffd->rep_cache_db,
                                     STMT_GET_REPS_FOR_RANGE));
   SVN_ERR(svn_sqlite__bindf(stmt, "rr",



Mime
View raw message