subversion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From svn-r...@apache.org
Subject svn commit: r1701345 - in /subversion/branches/1.8.x: ./ STATUS subversion/ subversion/libsvn_fs_fs/fs_fs.c
Date Sat, 05 Sep 2015 04:00:26 GMT
Author: svn-role
Date: Sat Sep  5 04:00:25 2015
New Revision: 1701345

URL: http://svn.apache.org/r1701345
Log:
Merge the 1.8.x-memory-fragmentation branch:

 * ^/subversion/branches/1.8.x-memory-fragmentation
   Reduce FSFS memory allocation in Apache with unbounded MaxFreeMem.
   Justification:
     A user reported that their worker processes ran OOM with Apache 2.2
     in default configuration when fulltext caching was enabled. Although
     people should set MaxFreeMem to something other than 0, we should
     play nice with out-of-the-box setups.
     The patch itself is relatively low-risk (changes initial buffer size).
   Votes:
     +1: stefan2, brane, rhuijben

Modified:
    subversion/branches/1.8.x/   (props changed)
    subversion/branches/1.8.x/STATUS
    subversion/branches/1.8.x/subversion/   (props changed)
    subversion/branches/1.8.x/subversion/libsvn_fs_fs/fs_fs.c

Propchange: subversion/branches/1.8.x/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Sep  5 04:00:25 2015
@@ -13,6 +13,7 @@
 /subversion/branches/1.8.x-issue4551:1654795-1660018
 /subversion/branches/1.8.x-javahl-exception-crash:1586424-1588151
 /subversion/branches/1.8.x-libsvnjavahl-version:1483910-1485054
+/subversion/branches/1.8.x-memory-fragmentation:1669948-1701344
 /subversion/branches/1.8.x-openssl-dirs:1535137-1540436
 /subversion/branches/1.8.x-r1477876:1477981-1487716
 /subversion/branches/1.8.x-r1481625:1481637-1482135

Modified: subversion/branches/1.8.x/STATUS
URL: http://svn.apache.org/viewvc/subversion/branches/1.8.x/STATUS?rev=1701345&r1=1701344&r2=1701345&view=diff
==============================================================================
--- subversion/branches/1.8.x/STATUS (original)
+++ subversion/branches/1.8.x/STATUS Sat Sep  5 04:00:25 2015
@@ -99,14 +99,3 @@ Veto-blocked changes:
 
 Approved changes:
 =================
-
- * ^/subversion/branches/1.8.x-memory-fragmentation
-   Reduce FSFS memory allocation in Apache with unbounded MaxFreeMem.
-   Justification:
-     A user reported that their worker processes ran OOM with Apache 2.2
-     in default configuration when fulltext caching was enabled. Although
-     people should set MaxFreeMem to something other than 0, we should
-     play nice with out-of-the-box setups.
-     The patch itself is relatively low-risk (changes initial buffer size).
-   Votes:
-     +1: stefan2, brane, rhuijben

Propchange: subversion/branches/1.8.x/subversion/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Sep  5 04:00:25 2015
@@ -12,6 +12,7 @@
 /subversion/branches/1.8.x-issue4551/subversion:1654795-1660018
 /subversion/branches/1.8.x-javahl-exception-crash/subversion:1586424-1588151
 /subversion/branches/1.8.x-libsvnjavahl-version/subversion:1483910-1485054
+/subversion/branches/1.8.x-memory-fragmentation/subversion:1669948-1701344
 /subversion/branches/1.8.x-openssl-dirs/subversion:1535137-1540436
 /subversion/branches/1.8.x-r1477876/subversion:1477981-1487716
 /subversion/branches/1.8.x-r1481625/subversion:1481637-1482135

Modified: subversion/branches/1.8.x/subversion/libsvn_fs_fs/fs_fs.c
URL: http://svn.apache.org/viewvc/subversion/branches/1.8.x/subversion/libsvn_fs_fs/fs_fs.c?rev=1701345&r1=1701344&r2=1701345&view=diff
==============================================================================
--- subversion/branches/1.8.x/subversion/libsvn_fs_fs/fs_fs.c (original)
+++ subversion/branches/1.8.x/subversion/libsvn_fs_fs/fs_fs.c Sat Sep  5 04:00:25 2015
@@ -4910,6 +4910,49 @@ build_rep_list(apr_array_header_t **list
     }
 }
 
+/* Determine the optimal size of a string buf that shall receive a
+ * (full-) text of NEEDED bytes.
+ *
+ * The critical point is that those buffers may be very large and
+ * can cause memory fragmentation.  We apply simple heuristics to
+ * make fragmentation less likely.
+ */
+static apr_size_t
+optimimal_allocation_size(apr_size_t needed)
+{
+  /* For all allocations, assume some overhead that is shared between
+   * OS memory managemnt, APR memory management and svn_stringbuf_t. */
+  const apr_size_t overhead = 0x400;
+  apr_size_t optimal;
+
+  /* If an allocation size if safe for other ephemeral buffers, it should
+   * be safe for ours. */
+  if (needed <= SVN__STREAM_CHUNK_SIZE)
+    return needed;
+
+  /* Paranoia edge case:
+   * Skip our heuristics if they created arithmetical overflow.
+   * Beware to make this test work for NEEDED = APR_SIZE_MAX as well! */
+  if (needed >= APR_SIZE_MAX / 2 - overhead)
+    return needed;
+
+  /* As per definition SVN__STREAM_CHUNK_SIZE is a power of two.
+   * Since we know NEEDED to be larger than that, use it as the
+   * starting point.
+   *
+   * Heuristics: Allocate a power-of-two number of bytes that fit
+   *             NEEDED plus some OVERHEAD.  The APR allocator
+   *             will round it up to the next full page size.
+   */
+  optimal = SVN__STREAM_CHUNK_SIZE;
+  while (optimal - overhead < needed)
+    optimal *= 2;
+
+  /* This is above or equal to NEEDED. */
+  return optimal - overhead;
+}
+
+
 
 /* Create a rep_read_baton structure for node revision NODEREV in
    filesystem FS and store it in *RB_P.  If FULLTEXT_CACHE_KEY is not
@@ -4945,7 +4988,7 @@ rep_read_get_baton(struct rep_read_baton
 
   if (SVN_IS_VALID_REVNUM(fulltext_cache_key.revision))
     b->current_fulltext = svn_stringbuf_create_ensure
-                            ((apr_size_t)b->len,
+                            (optimimal_allocation_size((apr_size_t)b->len),
                              b->filehandle_pool);
   else
     b->current_fulltext = NULL;



Mime
View raw message