lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hoss...@apache.org
Subject svn commit: r1494696 - in /lucene/dev/branches/branch_4x: ./ dev-tools/ lucene/ lucene/analysis/ lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/ lucene/benchmark/ lucene/classification/ lucene/classification/src/ lucene/code...
Date Wed, 19 Jun 2013 17:05:05 GMT
Author: hossman
Date: Wed Jun 19 17:05:04 2013
New Revision: 1494696

URL: http://svn.apache.org/r1494696
Log:
SOLR-4934: Fix handling of <mergePolicy> init arg useCompoundFile needed after changes
in LUCENE-5038 (merge r1494348)

Added:
    lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-defaults.xml
      - copied unchanged from r1494348, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-defaults.xml
    lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-legacy.xml
      - copied unchanged from r1494348, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-legacy.xml
Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/dev-tools/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/BUILD.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/LICENSE.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/MIGRATE.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/README.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/SYSTEM_REQUIREMENTS.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/analysis/   (props changed)
    lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/backwards/   (props changed)
    lucene/dev/branches/branch_4x/lucene/benchmark/   (props changed)
    lucene/dev/branches/branch_4x/lucene/build.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/classification/   (props changed)
    lucene/dev/branches/branch_4x/lucene/classification/build.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/classification/ivy.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/classification/src/   (props changed)
    lucene/dev/branches/branch_4x/lucene/codecs/   (props changed)
    lucene/dev/branches/branch_4x/lucene/common-build.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSort.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/demo/   (props changed)
    lucene/dev/branches/branch_4x/lucene/facet/   (props changed)
    lucene/dev/branches/branch_4x/lucene/grouping/   (props changed)
    lucene/dev/branches/branch_4x/lucene/highlighter/   (props changed)
    lucene/dev/branches/branch_4x/lucene/ivy-settings.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/join/   (props changed)
    lucene/dev/branches/branch_4x/lucene/licenses/   (props changed)
    lucene/dev/branches/branch_4x/lucene/memory/   (props changed)
    lucene/dev/branches/branch_4x/lucene/misc/   (props changed)
    lucene/dev/branches/branch_4x/lucene/module-build.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/queries/   (props changed)
    lucene/dev/branches/branch_4x/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/queryparser/   (props changed)
    lucene/dev/branches/branch_4x/lucene/replicator/   (props changed)
    lucene/dev/branches/branch_4x/lucene/sandbox/   (props changed)
    lucene/dev/branches/branch_4x/lucene/site/   (props changed)
    lucene/dev/branches/branch_4x/lucene/spatial/   (props changed)
    lucene/dev/branches/branch_4x/lucene/suggest/   (props changed)
    lucene/dev/branches/branch_4x/lucene/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/lucene/tools/   (props changed)
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/solr/LICENSE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/README.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/SYSTEM_REQUIREMENTS.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/build.xml   (props changed)
    lucene/dev/branches/branch_4x/solr/cloud-dev/   (props changed)
    lucene/dev/branches/branch_4x/solr/common-build.xml   (props changed)
    lucene/dev/branches/branch_4x/solr/contrib/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
    lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy.xml
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestConfig.java
  (props changed)
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java
    lucene/dev/branches/branch_4x/solr/example/   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/scripts/   (props changed)
    lucene/dev/branches/branch_4x/solr/site/   (props changed)
    lucene/dev/branches/branch_4x/solr/solrj/   (props changed)
    lucene/dev/branches/branch_4x/solr/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/solr/webapp/   (props changed)

Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1494696&r1=1494695&r2=1494696&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Wed Jun 19 17:05:04 2013
@@ -33,6 +33,16 @@ Apache ZooKeeper 3.4.5
 Upgrading from Solr 4.3.0
 ----------------------
 
+* TieredMergePolicy and the various subtypes of LogMergePolicy no longer have 
+  an explicit "setUseCompoundFile" method.  Instead this behavior is now driven
+  entirely by the value of "setNoCFSRatio".  If you have explicitly configured 
+  one of these classes using <mergePolicy> and include an init arg like this...
+     <bool name="useCompoundFile">true</bool>
+  ...this will now be ignored and a warning will be logged.  Users with an 
+  explicitly declared <mergePolicy> are encouraged to review the current 
+  javadocs for their MergePolicy subclass and review their configured options 
+  carefully.  See SOLR-4934 and LUCENE-5038 for more information.
+
 * SOLR-4778: The signature of LogWatcher.registerListener has changed, from
   (ListenerConfig, CoreContainer) to (ListenerConfig).  Users implementing their
   own LogWatcher classes will need to change their code accordingly.
@@ -138,7 +148,9 @@ Bug Fixes
   defined in the original. Benign since the default values are saved, but still incorrect.
   (Erick Erickson, thanks Shawn Heisey for helping test!)
 
-=======
+* SOLR-4934: Fix handling of <mergePolicy> init arg "useCompoundFile" needed 
+  after changes in LUCENE-5038 (hossman)
+
 Optimizations
 ----------------------
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java?rev=1494696&r1=1494695&r2=1494696&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
Wed Jun 19 17:05:04 2013
@@ -221,6 +221,9 @@ public class SolrIndexConfig {
 
       if (mergeFactor != -1)
         logMergePolicy.setMergeFactor(mergeFactor);
+
+      fixUseCFInitArg(LogMergePolicy.class);
+
     } else if (policy instanceof TieredMergePolicy) {
       TieredMergePolicy tieredMergePolicy = (TieredMergePolicy) policy;
       
@@ -230,8 +233,11 @@ public class SolrIndexConfig {
         tieredMergePolicy.setMaxMergeAtOnce(mergeFactor);
         tieredMergePolicy.setSegmentsPerTier(mergeFactor);
       }
-    } else {
-      log.warn("Use of compound file format or mergefactor cannot be configured if merge
policy is not an instance of LogMergePolicy or TieredMergePolicy. The configured policy's
defaults will be used.");
+
+      fixUseCFInitArg(TieredMergePolicy.class);
+
+    } else if (useCompoundFile && (mergeFactor != -1)) {
+      log.warn("Use of <useCompoundFile> or <mergeFactor> cannot be configured
if merge policy is not an instance of LogMergePolicy or TieredMergePolicy. The configured
policy's defaults will be used.");
     }
 
     if (mergePolicyInfo != null)
@@ -249,4 +255,20 @@ public class SolrIndexConfig {
 
     return scheduler;
   }
+
+  /**
+   * Lucene 4.4 removed the setUseCompoundFile(boolean) method from the two 
+   * conrete MergePolicies provided with Lucene/Solr.  In the event that users 
+   * have a value explicitly configured for this setter in their mergePolicy 
+   * init args, we remove it for them and warn about it.
+   */
+  private void fixUseCFInitArg(Class c) {
+
+    if (null == mergePolicyInfo || null == mergePolicyInfo.initArgs) return;
+
+    Object useCFSArg = mergePolicyInfo.initArgs.remove("useCompoundFile");
+    if (null != useCFSArg) {
+      log.warn("Ignoring 'useCompoundFile' specified as an init arg for the <mergePolicy>
since it is no longer supported by " + c.getSimpleName());
+    }
+  }
 }

Modified: lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy.xml?rev=1494696&r1=1494695&r2=1494696&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy.xml
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy.xml
Wed Jun 19 17:05:04 2013
@@ -26,7 +26,13 @@
     <mergePolicy class="org.apache.lucene.index.TieredMergePolicy">
       <int name="maxMergeAtOnceExplicit">19</int>
       <int name="segmentsPerTier">9</int>
-      <double name="noCFSRatio">1.0</double>
+      <double name="noCFSRatio">0.6</double>
+
+      <!-- Setter for this was removed in Lucene 4.4, but we should
+           ignore it with a warning if users have it in their configs
+      -->
+      <bool name="useCompoundFile">true</bool>
+
     </mergePolicy>
   </indexConfig>
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java?rev=1494696&r1=1494695&r2=1494696&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java
Wed Jun 19 17:05:04 2013
@@ -21,31 +21,64 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.index.MergePolicy;
 import org.apache.lucene.index.TieredMergePolicy;
 import org.apache.solr.SolrTestCaseJ4;
-import org.junit.BeforeClass;
+import org.junit.After;
 
 public class TestMergePolicyConfig extends SolrTestCaseJ4 {
 
-  @BeforeClass
-  public static void beforeClass() throws Exception {
-    initCore("solrconfig-mergepolicy.xml","schema.xml");
+  @After
+  public void after() throws Exception {
+    deleteCore();
+  }
+
+  public void testDefaultMergePolicyConfig() throws Exception {
+    initCore("solrconfig-mergepolicy-defaults.xml","schema-minimal.xml");
+    IndexWriterConfig iwc = solrConfig.indexConfig.toIndexWriterConfig(h.getCore().getLatestSchema());
+
+    TieredMergePolicy tieredMP = assertAndCast(TieredMergePolicy.class,
+                                               iwc.getMergePolicy());
+    assertEquals(0.0D, tieredMP.getNoCFSRatio(), 0.0D);
+
+  }
+
+  public void testLegacyMergePolicyConfig() throws Exception {
+    initCore("solrconfig-mergepolicy-legacy.xml","schema-minimal.xml");
+    IndexWriterConfig iwc = solrConfig.indexConfig.toIndexWriterConfig(h.getCore().getLatestSchema());
+    TieredMergePolicy tieredMP = assertAndCast(TieredMergePolicy.class,
+                                               iwc.getMergePolicy());
+
+    assertEquals(7, tieredMP.getMaxMergeAtOnce());
+    assertEquals(7.0D, tieredMP.getSegmentsPerTier(), 0.0D);
+    assertEquals(1.0D, tieredMP.getNoCFSRatio(), 0.0D);
   }
   
   public void testTieredMergePolicyConfig() throws Exception {
+    initCore("solrconfig-mergepolicy.xml","schema-minimal.xml");
     IndexWriterConfig iwc = solrConfig.indexConfig.toIndexWriterConfig(h.getCore().getLatestSchema());
-    MergePolicy mp = iwc.getMergePolicy();
-    assertTrue(mp instanceof TieredMergePolicy);
-    TieredMergePolicy tieredMP = (TieredMergePolicy) mp;
+    TieredMergePolicy tieredMP = assertAndCast(TieredMergePolicy.class,
+                                               iwc.getMergePolicy());
 
-    // mp-specific setter
-    assertEquals(19, tieredMP.getMaxMergeAtOnceExplicit());
-    
+    // set by legacy <mergeFactor> setting
     assertEquals(7, tieredMP.getMaxMergeAtOnce());
     
-    // make sure we overrode segmentsPerTier (split from maxMergeAtOnce out of mergeFactor)
+    // mp-specific setters
+    assertEquals(19, tieredMP.getMaxMergeAtOnceExplicit());
+    assertEquals(0.6D, tieredMP.getNoCFSRatio(), 0.001);
+    // make sure we overrode segmentsPerTier 
+    // (split from maxMergeAtOnce out of mergeFactor)
     assertEquals(9D, tieredMP.getSegmentsPerTier(), 0.001);
     
-    // make sure we overrode noCFSRatio (useless because we disabled useCompoundFile,
-    // but just to make sure it works)
-    assertEquals(1.0D, tieredMP.getNoCFSRatio(), 0.001);
   }
+
+  /**
+   * Given a Type and an object asserts that the object is non-null and an 
+   * instance of the specified Type.  The object is then cast to that type and 
+   * returned.
+   */
+  public static <T> T assertAndCast(Class<? extends T> clazz, Object o) {
+    assertNotNull(clazz);
+    assertNotNull(o);
+    assertTrue(clazz.isInstance(o));
+    return clazz.cast(o);
+  }
+
 }



Mime
View raw message