lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jpou...@apache.org
Subject [6/9] lucene-solr:branch_7x: LUCENE-8392: Avoir corner cases when maxMergeAtOnce is greater than segmentsPerTier.
Date Tue, 10 Jul 2018 07:28:19 GMT
LUCENE-8392: Avoir corner cases when maxMergeAtOnce is greater than segmentsPerTier.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/43481f26
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/43481f26
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/43481f26

Branch: refs/heads/branch_7x
Commit: 43481f26db0afe9d9ad39013e0ff6ae76308d643
Parents: 0dc6ef9
Author: Adrien Grand <jpountz@gmail.com>
Authored: Tue Jul 10 08:42:59 2018 +0200
Committer: Adrien Grand <jpountz@gmail.com>
Committed: Tue Jul 10 09:18:02 2018 +0200

----------------------------------------------------------------------
 .../apache/lucene/index/TieredMergePolicy.java  | 22 ++++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/43481f26/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java
index 8ac4977..34d7af6 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java
@@ -90,7 +90,9 @@ public class TieredMergePolicy extends MergePolicy {
    *  the index, then we disable compound file for it.
    *  @see MergePolicy#setNoCFSRatio */
   public static final double DEFAULT_NO_CFS_RATIO = 0.1;
-  
+
+  // User-specified maxMergeAtOnce. In practice we always take the min of its
+  // value and segsPerTier to avoid suboptimal merging.
   private int maxMergeAtOnce = 10;
   private long maxMergedSegmentBytes = 5*1024*1024*1024L;
   private int maxMergeAtOnceExplicit = 30;
@@ -255,10 +257,6 @@ public class TieredMergePolicy extends MergePolicy {
   /** Sets the allowed number of segments per tier.  Smaller
    *  values mean more merging but fewer segments.
    *
-   *  <p><b>NOTE</b>: this value should be {@code >=} the {@link
-   *  #setMaxMergeAtOnce} otherwise you'll force too much
-   *  merging to occur.</p>
-   *
    *  <p>Default is 10.0.</p> */
   public TieredMergePolicy setSegmentsPerTier(double v) {
     if (v < 2.0) {
@@ -397,6 +395,7 @@ public class TieredMergePolicy extends MergePolicy {
       }
     }
 
+    final int mergeFactor = (int) Math.min(maxMergeAtOnce, segsPerTier);
     // Compute max allowed segments in the index
     long levelSize = Math.max(minSegmentBytes, floorSegmentBytes);
     long bytesLeft = totIndexBytes;
@@ -409,20 +408,20 @@ public class TieredMergePolicy extends MergePolicy {
       }
       allowedSegCount += segsPerTier;
       bytesLeft -= segsPerTier * levelSize;
-      levelSize *= maxMergeAtOnce;
+      levelSize *= mergeFactor;
     }
 
     if (verbose(mergeContext) && tooBigCount > 0) {
       message("  allowedSegmentCount=" + allowedSegCount + " vs count=" + infos.size() +
           " (eligible count=" + sortedInfos.size() + ") tooBigCount= " + tooBigCount, mergeContext);
     }
-    return doFindMerges(sortedInfos, maxMergedSegmentBytes, maxMergeAtOnce, (int) allowedSegCount,
MERGE_TYPE.NATURAL,
+    return doFindMerges(sortedInfos, maxMergedSegmentBytes, mergeFactor, (int) allowedSegCount,
MERGE_TYPE.NATURAL,
         mergeContext, mergingBytes >= maxMergedSegmentBytes);
   }
 
   private MergeSpecification doFindMerges(List<SegmentSizeAndDocs> sortedEligibleInfos,
                                           final long maxMergedSegmentBytes,
-                                          final int maxMergeAtOnce, final int allowedSegCount,
+                                          final int mergeFactor, final int allowedSegCount,
                                           final MERGE_TYPE mergeType,
                                           MergeContext mergeContext,
                                           boolean maxMergeIsRunning) throws IOException {
@@ -486,7 +485,7 @@ public class TieredMergePolicy extends MergePolicy {
       long bestMergeBytes = 0;
 
       // Consider all merge starts.
-      int lim = sortedEligible.size() - maxMergeAtOnce; // assume the usual case of background
merging.
+      int lim = sortedEligible.size() - mergeFactor; // assume the usual case of background
merging.
 
       if (mergeType != MERGE_TYPE.NATURAL) { // The unusual case of forceMerge or expungeDeletes.
         // The incoming eligible list will have only segments with > forceMergeDeletesPctAllowed
in the case of
@@ -502,7 +501,7 @@ public class TieredMergePolicy extends MergePolicy {
         final List<SegmentCommitInfo> candidate = new ArrayList<>();
         boolean hitTooLarge = false;
         long bytesThisMerge = 0;
-        for (int idx = startIdx; idx < sortedEligible.size() && candidate.size()
< maxMergeAtOnce && bytesThisMerge < maxMergedSegmentBytes; idx++) {
+        for (int idx = startIdx; idx < sortedEligible.size() && candidate.size()
< mergeFactor && bytesThisMerge < maxMergedSegmentBytes; idx++) {
           final SegmentSizeAndDocs segSizeDocs = sortedEligible.get(idx);
           final long segBytes = segSizeDocs.sizeInBytes;
 
@@ -601,7 +600,8 @@ public class TieredMergePolicy extends MergePolicy {
       // matter in this case because this merge will not
       // "cascade" and so it cannot lead to N^2 merge cost
       // over time:
-      skew = 1.0/maxMergeAtOnce;
+      final int mergeFactor = (int) Math.min(maxMergeAtOnce, segsPerTier);
+      skew = 1.0/mergeFactor;
     } else {
       skew = ((double) floorSize(segmentsSizes.get(candidate.get(0)).sizeInBytes)) / totAfterMergeBytesFloored;
     }


Mime
View raw message