subversion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From stef...@apache.org
Subject svn commit: r1711384 - in /subversion/trunk/subversion/libsvn_fs_x: pack.c revprops.c revprops.h
Date Thu, 29 Oct 2015 23:34:13 GMT
Author: stefan2
Date: Thu Oct 29 23:34:13 2015
New Revision: 1711384

URL: http://svn.apache.org/viewvc?rev=1711384&view=rev
Log:
Speed up FSX pack by using a common fsync batch for rev props and rev data.

* subversion/libsvn_fs_x/revprops.h
  (svn_fs_x__pack_revprops_shard): Make the BATCH an argument.

* subversion/libsvn_fs_x/revprops.c
  (svn_fs_x__pack_revprops_shard): No longer create & run the BATCH locally.

* subversion/libsvn_fs_x/pack.c
  (pack_log_addressed): Ditto.
  (pack_rev_shard): Take a BATCH and pass it through.  Also, ensure the
                    directory addition gets sync'ed properly.
  (pack_shard): Create the fsync batch locally, let the pack steps fill it
                and run it here. 

Modified:
    subversion/trunk/subversion/libsvn_fs_x/pack.c
    subversion/trunk/subversion/libsvn_fs_x/revprops.c
    subversion/trunk/subversion/libsvn_fs_x/revprops.h

Modified: subversion/trunk/subversion/libsvn_fs_x/pack.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/pack.c?rev=1711384&r1=1711383&r2=1711384&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/pack.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/pack.c Thu Oct 29 23:34:13 2015
@@ -1913,6 +1913,7 @@ append_revision(pack_context_t *context,
  * SHARD_DIR into the PACK_FILE_DIR, using SCRATCH_POOL for temporary
  * allocations.  Limit the extra memory consumption to MAX_MEM bytes.
  * CANCEL_FUNC and CANCEL_BATON are what you think they are.
+ * Schedule necessary fsync calls in BATCH.
  */
 static svn_error_t *
 pack_log_addressed(svn_fs_t *fs,
@@ -1920,6 +1921,7 @@ pack_log_addressed(svn_fs_t *fs,
                    const char *shard_dir,
                    svn_revnum_t shard_rev,
                    apr_size_t max_mem,
+                   svn_fs_x__batch_fsync_t *batch,
                    svn_cancel_func_t cancel_func,
                    void *cancel_baton,
                    apr_pool_t *scratch_pool)
@@ -1943,10 +1945,6 @@ pack_log_addressed(svn_fs_t *fs,
   int i;
   apr_size_t item_count = 0;
   apr_pool_t *iterpool = svn_pool_create(scratch_pool);
-  svn_fs_x__batch_fsync_t *batch;
-
-  /* Perform all fsyncs through this instance. */
-  SVN_ERR(svn_fs_x__batch_fsync_create(&batch, scratch_pool));
 
   /* set up a pack context */
   SVN_ERR(initialize_pack_context(&context, fs, pack_file_dir, shard_dir,
@@ -1998,9 +1996,6 @@ pack_log_addressed(svn_fs_t *fs,
   /* last phase: finalize indexes and clean up */
   SVN_ERR(reset_pack_context(&context, iterpool));
   SVN_ERR(close_pack_context(&context, iterpool));
-
-  /* Ensure that packed file is written to disk.*/
-  SVN_ERR(svn_fs_x__batch_fsync_run(batch, iterpool));
   svn_pool_destroy(iterpool);
 
   return SVN_NO_ERROR;
@@ -2010,7 +2005,7 @@ pack_log_addressed(svn_fs_t *fs,
  * MAX_FILES_PER_DIR revisions from SHARD_PATH into the PACK_FILE_DIR,
  * using SCRATCH_POOL for temporary allocations.  Try to limit the amount of
  * temporary memory needed to MAX_MEM bytes.  CANCEL_FUNC and CANCEL_BATON
- * are what you think they are.
+ * are what you think they are.  Schedule necessary fsync calls in BATCH.
  *
  * If for some reason we detect a partial packing already performed, we
  * remove the pack file and start again.
@@ -2024,6 +2019,7 @@ pack_rev_shard(svn_fs_t *fs,
                apr_int64_t shard,
                int max_files_per_dir,
                apr_size_t max_mem,
+               svn_fs_x__batch_fsync_t *batch,
                svn_cancel_func_t cancel_func,
                void *cancel_baton,
                apr_pool_t *scratch_pool)
@@ -2040,10 +2036,11 @@ pack_rev_shard(svn_fs_t *fs,
 
   /* Create the new directory and pack file. */
   SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool));
+  SVN_ERR(svn_fs_x__batch_fsync_new_path(batch, pack_file_dir, scratch_pool));
 
   /* Index information files */
   SVN_ERR(pack_log_addressed(fs, pack_file_dir, shard_path, shard_rev,
-                              max_mem, cancel_func, cancel_baton,
+                             max_mem, batch, cancel_func, cancel_baton,
                              scratch_pool));
 
   SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, scratch_pool));
@@ -2077,12 +2074,16 @@ pack_shard(const char *dir,
 {
   svn_fs_x__data_t *ffd = fs->fsap_data;
   const char *shard_path, *pack_file_dir;
+  svn_fs_x__batch_fsync_t *batch;
 
   /* Notify caller we're starting to pack this shard. */
   if (notify_func)
     SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start,
                         scratch_pool));
 
+  /* Perform all fsyncs through this instance. */
+  SVN_ERR(svn_fs_x__batch_fsync_create(&batch, scratch_pool));
+
   /* Some useful paths. */
   pack_file_dir = svn_dirent_join(dir,
                   apr_psprintf(scratch_pool,
@@ -2095,7 +2096,7 @@ pack_shard(const char *dir,
 
   /* pack the revision content */
   SVN_ERR(pack_rev_shard(fs, pack_file_dir, shard_path,
-                         shard, max_files_per_dir, DEFAULT_MAX_MEM,
+                         shard, max_files_per_dir, DEFAULT_MAX_MEM, batch,
                          cancel_func, cancel_baton, scratch_pool));
 
   /* pack the revprops in an equivalent way */
@@ -2104,7 +2105,7 @@ pack_shard(const char *dir,
                                         shard_path,
                                         shard, max_files_per_dir,
                                         (int)(0.9 * max_pack_size),
-                                        compression_level,
+                                        compression_level, batch,
                                         cancel_func, cancel_baton,
                                         scratch_pool));
 
@@ -2114,6 +2115,9 @@ pack_shard(const char *dir,
                           scratch_pool));
   ffd->min_unpacked_rev = (svn_revnum_t)((shard + 1) * max_files_per_dir);
 
+  /* Ensure that packed file is written to disk.*/
+  SVN_ERR(svn_fs_x__batch_fsync_run(batch, scratch_pool));
+
   /* Finally, remove the existing shard directories. */
   SVN_ERR(svn_io_remove_dir2(shard_path, TRUE,
                              cancel_func, cancel_baton, scratch_pool));

Modified: subversion/trunk/subversion/libsvn_fs_x/revprops.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/revprops.c?rev=1711384&r1=1711383&r2=1711384&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/revprops.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/revprops.c Thu Oct 29 23:34:13 2015
@@ -27,7 +27,6 @@
 #include "svn_hash.h"
 #include "svn_dirent_uri.h"
 
-#include "batch_fsync.h"
 #include "fs_x.h"
 #include "revprops.h"
 #include "util.h"
@@ -1567,6 +1566,7 @@ svn_fs_x__pack_revprops_shard(svn_fs_t *
                               int max_files_per_dir,
                               apr_off_t max_pack_size,
                               int compression_level,
+                              svn_fs_x__batch_fsync_t *batch,
                               svn_cancel_func_t cancel_func,
                               void *cancel_baton,
                               apr_pool_t *scratch_pool)
@@ -1578,15 +1578,11 @@ svn_fs_x__pack_revprops_shard(svn_fs_t *
   apr_off_t total_size;
   apr_pool_t *iterpool = svn_pool_create(scratch_pool);
   apr_array_header_t *sizes;
-  svn_fs_x__batch_fsync_t *batch;
 
   /* Some useful paths. */
   manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST,
                                        scratch_pool);
 
-  /* Perform all fsyncs through this instance. */
-  SVN_ERR(svn_fs_x__batch_fsync_create(&batch, scratch_pool));
-
   /* Create the manifest file stream. */
   SVN_ERR(svn_fs_x__batch_fsync_open_file(&manifest_file, batch,
                                           manifest_file_path, scratch_pool));
@@ -1668,8 +1664,6 @@ svn_fs_x__pack_revprops_shard(svn_fs_t *
   /* flush all data to disk and update permissions */
   SVN_ERR(svn_stream_close(manifest_stream));
   SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool));
-  SVN_ERR(svn_fs_x__batch_fsync_run(batch, scratch_pool));
-
   svn_pool_destroy(iterpool);
 
   return SVN_NO_ERROR;

Modified: subversion/trunk/subversion/libsvn_fs_x/revprops.h
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/revprops.h?rev=1711384&r1=1711383&r2=1711384&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/revprops.h (original)
+++ subversion/trunk/subversion/libsvn_fs_x/revprops.h Thu Oct 29 23:34:13 2015
@@ -25,6 +25,8 @@
 
 #include "svn_fs.h"
 
+#include "batch_fsync.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif /* __cplusplus */
@@ -85,7 +87,7 @@ svn_fs_x__packed_revprop_available(svn_b
 
 /* For the revprop SHARD at SHARD_PATH with exactly MAX_FILES_PER_DIR
  * revprop files in it, create a packed shared at PACK_FILE_DIR in
- * filesystem FS.
+ * filesystem FS.  Schedule necessary fsync calls in BATCH.
  *
  * COMPRESSION_LEVEL defines how well the resulting pack file shall be
  * compressed or whether is shall be compressed at all.  Individual pack
@@ -103,6 +105,7 @@ svn_fs_x__pack_revprops_shard(svn_fs_t *
                               int max_files_per_dir,
                               apr_off_t max_pack_size,
                               int compression_level,
+                              svn_fs_x__batch_fsync_t *batch,
                               svn_cancel_func_t cancel_func,
                               void *cancel_baton,
                               apr_pool_t *scratch_pool);



Mime
View raw message