lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1446834 - in /lucene/dev/branches/branch_4x: ./ dev-tools/ lucene/ lucene/analysis/ lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/ lucene/benchmark/ lucene/codecs/ lucene/core/ lucene/core/src/test/org/apache/l...
Date Sat, 16 Feb 2013 01:29:21 GMT
Author: markrmiller
Date: Sat Feb 16 01:29:19 2013
New Revision: 1446834

URL: http://svn.apache.org/r1446834
Log:
SOLR-4459: The Replication 'index move' rather than copy optimization doesn't kick in when
using NRTCachingDirectory or the rate limiting feature.

Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/dev-tools/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/BUILD.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/CHANGES.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/LICENSE.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/MIGRATE.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/README.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/SYSTEM_REQUIREMENTS.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/analysis/   (props changed)
    lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/backwards/   (props changed)
    lucene/dev/branches/branch_4x/lucene/benchmark/   (props changed)
    lucene/dev/branches/branch_4x/lucene/build.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/codecs/   (props changed)
    lucene/dev/branches/branch_4x/lucene/common-build.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSort.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/demo/   (props changed)
    lucene/dev/branches/branch_4x/lucene/facet/   (props changed)
    lucene/dev/branches/branch_4x/lucene/grouping/   (props changed)
    lucene/dev/branches/branch_4x/lucene/highlighter/   (props changed)
    lucene/dev/branches/branch_4x/lucene/ivy-settings.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/join/   (props changed)
    lucene/dev/branches/branch_4x/lucene/licenses/   (props changed)
    lucene/dev/branches/branch_4x/lucene/memory/   (props changed)
    lucene/dev/branches/branch_4x/lucene/misc/   (props changed)
    lucene/dev/branches/branch_4x/lucene/module-build.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/queries/   (props changed)
    lucene/dev/branches/branch_4x/lucene/queryparser/   (props changed)
    lucene/dev/branches/branch_4x/lucene/sandbox/   (props changed)
    lucene/dev/branches/branch_4x/lucene/site/   (props changed)
    lucene/dev/branches/branch_4x/lucene/spatial/   (props changed)
    lucene/dev/branches/branch_4x/lucene/suggest/   (props changed)
    lucene/dev/branches/branch_4x/lucene/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/lucene/tools/   (props changed)
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/solr/LICENSE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/README.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/SYSTEM_REQUIREMENTS.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/build.xml   (props changed)
    lucene/dev/branches/branch_4x/solr/cloud-dev/   (props changed)
    lucene/dev/branches/branch_4x/solr/common-build.xml   (props changed)
    lucene/dev/branches/branch_4x/solr/contrib/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
    lucene/dev/branches/branch_4x/solr/example/   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/scripts/   (props changed)
    lucene/dev/branches/branch_4x/solr/site/   (props changed)
    lucene/dev/branches/branch_4x/solr/solrj/   (props changed)
    lucene/dev/branches/branch_4x/solr/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/solr/testlogging.properties   (props changed)
    lucene/dev/branches/branch_4x/solr/webapp/   (props changed)

Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1446834&r1=1446833&r2=1446834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Sat Feb 16 01:29:19 2013
@@ -83,7 +83,7 @@ Bug Fixes
 * SOLR-3926: Solr should support better way of finding active sorts (Eirik Lygre via
   Erick Erickson)
 
-* SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman)
+* SOLR-4342: Fix DataImportHandler stats to be a proper Map (hossman)
 
 * SOLR-3967: langid.enforceSchema option checks source field instead of target field (janhoy)
 
@@ -101,6 +101,10 @@ Bug Fixes
 
 * SOLR-4463: Fix SolrCoreState reference counting. (Mark Miller)
 
+* SOLR-4459: The Replication 'index move' rather than copy optimization doesn't
+  kick in when using NRTCachingDirectory or the rate limiting feature.
+  (Mark Miller)
+
 Optimizations
 ----------------------
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java?rev=1446834&r1=1446833&r2=1446834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
Sat Feb 16 01:29:19 2013
@@ -103,6 +103,10 @@ public abstract class DirectoryFactory i
   /**
    * Override for more efficient moves.
    * 
+   * Intended for use with replication - use
+   * carefully - some Directory wrappers will
+   * cache files for example.
+   * 
    * @throws IOException If there is a low-level I/O error.
    */
   public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext)
throws IOException {

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java?rev=1446834&r1=1446833&r2=1446834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
Sat Feb 16 01:29:19 2013
@@ -23,6 +23,8 @@ import org.apache.commons.io.FileUtils;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.NRTCachingDirectory;
+import org.apache.lucene.store.RateLimitedDirectoryWrapper;
 
 /**
  * Directory provider which mimics original Solr 
@@ -69,15 +71,29 @@ public class StandardDirectoryFactory ex
   /**
    * Override for more efficient moves.
    * 
+   * Intended for use with replication - use
+   * carefully - some Directory wrappers will
+   * cache files for example.
+   * 
+   * This implementation works with two wrappers:
+   * NRTCachingDirectory and RateLimitedDirectoryWrapper.
+   * 
+   * You should first {@link Directory#sync(java.util.Collection)} any file that will be

+   * moved or avoid cached files through settings.
+   * 
    * @throws IOException
    *           If there is a low-level I/O error.
    */
   @Override
   public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext)
       throws IOException {
-    if (fromDir instanceof FSDirectory && toDir instanceof FSDirectory) {
-      File dir1 = ((FSDirectory) fromDir).getDirectory();
-      File dir2 = ((FSDirectory) toDir).getDirectory();
+    
+    Directory baseFromDir = getBaseDir(fromDir);
+    Directory baseToDir = getBaseDir(fromDir);
+    
+    if (baseFromDir instanceof FSDirectory && baseToDir instanceof FSDirectory) {
+      File dir1 = ((FSDirectory) baseFromDir).getDirectory();
+      File dir2 = ((FSDirectory) baseToDir).getDirectory();
       File indexFileInTmpDir = new File(dir1, fileName);
       File indexFileInIndex = new File(dir2, fileName);
       boolean success = indexFileInTmpDir.renameTo(indexFileInIndex);
@@ -89,4 +105,18 @@ public class StandardDirectoryFactory ex
     super.move(fromDir, toDir, fileName, ioContext);
   }
 
+  // special hack to work with NRTCachingDirectory and RateLimitedDirectoryWrapper
+  private Directory getBaseDir(Directory dir) {
+    Directory baseDir;
+    if (dir instanceof NRTCachingDirectory) {
+      baseDir = ((NRTCachingDirectory)dir).getDelegate();
+    } else if (dir instanceof RateLimitedDirectoryWrapper) {
+      baseDir = ((RateLimitedDirectoryWrapper)dir).getDelegate();
+    } else {
+      baseDir = dir;
+    }
+    
+    return baseDir;
+  }
+
 }



Mime
View raw message