From commits-return-51292-apmail-lucene-commits-archive=www.apache.org@lucene.apache.org Wed Oct 8 10:46:08 2014 Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 489DA17FBE for ; Wed, 8 Oct 2014 10:46:08 +0000 (UTC) Received: (qmail 45962 invoked by uid 500); 8 Oct 2014 10:46:08 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 45953 invoked by uid 99); 8 Oct 2014 10:46:08 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 08 Oct 2014 10:46:08 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 08 Oct 2014 10:46:06 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 53F3823889D5; Wed, 8 Oct 2014 10:45:46 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1630068 - in /lucene/dev/trunk: ./ lucene/ lucene/test-framework/ lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Date: Wed, 08 Oct 2014 10:45:46 -0000 To: commits@lucene.apache.org From: mikemccand@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20141008104546.53F3823889D5@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: mikemccand Date: Wed Oct 8 10:45:45 2014 New Revision: 1630068 URL: http://svn.apache.org/r1630068 Log: avoid pathological merging Modified: lucene/dev/trunk/ (props changed) lucene/dev/trunk/lucene/ (props changed) lucene/dev/trunk/lucene/test-framework/ (props changed) lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1630068&r1=1630067&r2=1630068&view=diff ============================================================================== --- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original) +++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Wed Oct 8 10:45:45 2014 @@ -61,10 +61,51 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.*; -import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.AlcoholicMergePolicy; +import org.apache.lucene.index.AssertingDirectoryReader; +import org.apache.lucene.index.AssertingLeafReader; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.CompositeReader; +import org.apache.lucene.index.ConcurrentMergeScheduler; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldFilterLeafReader; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader.ReaderClosedListener; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.LiveIndexWriterConfig; +import org.apache.lucene.index.LogByteSizeMergePolicy; +import org.apache.lucene.index.LogDocMergePolicy; +import org.apache.lucene.index.LogMergePolicy; +import org.apache.lucene.index.MergePolicy; +import org.apache.lucene.index.MergeScheduler; +import org.apache.lucene.index.MockRandomMergePolicy; +import org.apache.lucene.index.MultiDocValues; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.ParallelCompositeReader; +import org.apache.lucene.index.ParallelLeafReader; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.SerialMergeScheduler; +import org.apache.lucene.index.SimpleMergedSegmentWarmer; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.StorableField; +import org.apache.lucene.index.StoredDocument; +import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum.SeekStatus; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.search.AssertingIndexSearcher; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; @@ -95,7 +136,6 @@ import org.junit.Test; import org.junit.rules.RuleChain; import org.junit.rules.TestRule; import org.junit.runner.RunWith; - import com.carrotsearch.randomizedtesting.JUnit4MethodProvider; import com.carrotsearch.randomizedtesting.LifecycleScope; import com.carrotsearch.randomizedtesting.MixWithSuiteName; @@ -707,8 +747,9 @@ public abstract class LuceneTestCase ext */ public static SegmentReader getOnlySegmentReader(DirectoryReader reader) { List subReaders = reader.leaves(); - if (subReaders.size() != 1) + if (subReaders.size() != 1) { throw new IllegalArgumentException(reader + " has " + subReaders.size() + " segments instead of exactly one"); + } final LeafReader r = subReaders.get(0).reader(); assertTrue(r instanceof SegmentReader); return (SegmentReader) r; @@ -862,6 +903,8 @@ public abstract class LuceneTestCase ext c.setMergePolicy(newMergePolicy(r)); + avoidPathologicalMerging(c); + if (rarely(r)) { c.setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.getInfoStream())); } @@ -870,6 +913,53 @@ public abstract class LuceneTestCase ext return c; } + private static void avoidPathologicalMerging(IndexWriterConfig iwc) { + // Don't allow "tiny" flushed segments with "big" merge + // floor: this leads to pathological O(N^2) merge costs: + long estFlushSizeBytes = Long.MAX_VALUE; + if (iwc.getMaxBufferedDocs() != IndexWriterConfig.DISABLE_AUTO_FLUSH) { + // Gross estimation of 1 KB segment bytes for each doc indexed: + estFlushSizeBytes = Math.min(estFlushSizeBytes, iwc.getMaxBufferedDocs() * 1024); + } + if (iwc.getRAMBufferSizeMB() != IndexWriterConfig.DISABLE_AUTO_FLUSH) { + estFlushSizeBytes = Math.min(estFlushSizeBytes, (long) (iwc.getRAMBufferSizeMB() * 1024 * 1024)); + } + assert estFlushSizeBytes > 0; + + MergePolicy mp = iwc.getMergePolicy(); + if (mp instanceof TieredMergePolicy) { + TieredMergePolicy tmp = (TieredMergePolicy) mp; + long floorSegBytes = (long) (tmp.getFloorSegmentMB() * 1024 * 1024); + if (floorSegBytes / estFlushSizeBytes > 10) { + double newValue = estFlushSizeBytes * 10.0 / 1024 / 1024; + if (VERBOSE) { + System.out.println("NOTE: LuceneTestCase: changing TieredMergePolicy.floorSegmentMB from " + tmp.getFloorSegmentMB() + " to " + newValue + " to avoid pathological merging"); + } + tmp.setFloorSegmentMB(newValue); + } + } else if (mp instanceof LogByteSizeMergePolicy) { + LogByteSizeMergePolicy lmp = (LogByteSizeMergePolicy) mp; + if ((lmp.getMinMergeMB()*1024*1024) / estFlushSizeBytes > 10) { + double newValue = estFlushSizeBytes * 10.0 / 1024 / 1024; + if (VERBOSE) { + System.out.println("NOTE: LuceneTestCase: changing LogByteSizeMergePolicy.minMergeMB from " + lmp.getMinMergeMB() + " to " + newValue + " to avoid pathological merging"); + } + lmp.setMinMergeMB(newValue); + } + } else if (mp instanceof LogDocMergePolicy) { + LogDocMergePolicy lmp = (LogDocMergePolicy) mp; + assert estFlushSizeBytes / 1024 < Integer.MAX_VALUE/10; + int estFlushDocs = Math.max(1, (int) (estFlushSizeBytes / 1024)); + if (lmp.getMinMergeDocs() / estFlushDocs > 10) { + int newValue = estFlushDocs * 10; + if (VERBOSE) { + System.out.println("NOTE: LuceneTestCase: changing LogDocMergePolicy.minMergeDocs from " + lmp.getMinMergeDocs() + " to " + newValue + " to avoid pathological merging"); + } + lmp.setMinMergeDocs(newValue); + } + } + } + public static MergePolicy newMergePolicy(Random r) { if (rarely(r)) { return new MockRandomMergePolicy(r);