lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject [1/2] lucene-solr:master: SOLR-9859: replication.properties cannot be updated after being written and neither eplication.properties or ndex.properties are durable in the face of a crash.
Date Wed, 28 Dec 2016 23:54:33 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/master dc6dcdda8 -> 262049fc8


SOLR-9859: replication.properties cannot be updated after being written and neither eplication.properties
or ndex.properties are durable in the face of a crash.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/96ed221f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/96ed221f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/96ed221f

Branch: refs/heads/master
Commit: 96ed221fb6924dd167591004a5eaf70d53f92e4f
Parents: dc6dcdd
Author: markrmiller <markrmiller@apache.org>
Authored: Wed Dec 28 17:40:03 2016 -0500
Committer: markrmiller <markrmiller@apache.org>
Committed: Wed Dec 28 18:52:19 2016 -0500

----------------------------------------------------------------------
 solr/CHANGES.txt                                |  3 +++
 .../org/apache/solr/core/DirectoryFactory.java  | 15 ++++++++++++
 .../apache/solr/core/HdfsDirectoryFactory.java  |  9 +++++++
 .../solr/core/StandardDirectoryFactory.java     | 22 +++++++++++++++++
 .../org/apache/solr/handler/IndexFetcher.java   | 25 +++++++++++---------
 .../solr/handler/TestReplicationHandler.java    |  7 +++++-
 6 files changed, 69 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96ed221f/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 7a708a6..c3cac28 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -287,6 +287,9 @@ Bug Fixes
 
 * SOLR-9699,SOLR-4668: fix exception from core status in parallel with core reload (Mikhail
Khludnev)
 
+* SOLR-9859: replication.properties cannot be updated after being written and neither replication.properties
or 
+  index.properties are durable in the face of a crash. (Pushkar Raste, Chris de Kok, Cao
Manh Dat, Mark Miller)
+
 Other Changes
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96ed221f/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
index 228260a..396a30d 100644
--- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
@@ -19,6 +19,7 @@ package org.apache.solr.core;
 import java.io.Closeable;
 import java.io.File;
 import java.io.FileFilter;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.util.Collection;
@@ -184,6 +185,20 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
     fromDir.deleteFile(fileName);
   }
   
+  // sub classes perform an atomic rename if possible, otherwise fall back to delete + rename
+  // this is important to support for index roll over durability after crashes
+  public void renameWithOverwrite(Directory dir, String fileName, String toName) throws IOException
{
+    try {
+      dir.deleteFile(toName);
+    } catch (FileNotFoundException e) {
+
+    } catch (Exception e) {
+      log.error("Exception deleting file", e);
+    }
+
+    dir.rename(fileName, toName);
+  }
+  
   /**
    * Returns the Directory for a given path, using the specified rawLockType.
    * Will return the same Directory instance for the same path.

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96ed221f/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
index b003287..d481e03 100644
--- a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
@@ -29,8 +29,10 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -568,4 +570,11 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements
Sol
       }
     }
   }
+  
+  // perform an atomic rename if possible
+  public void renameWithOverwrite(Directory dir, String fileName, String toName) throws IOException
{
+    String hdfsDirPath = getPath(dir);
+    FileContext fileContext = FileContext.getFileContext(getConf());
+    fileContext.rename(new Path(hdfsDirPath + "/" + fileName), new Path(hdfsDirPath + "/"
+ toName), Options.Rename.OVERWRITE);
+  }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96ed221f/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
index 532655b..d418137 100644
--- a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
@@ -18,6 +18,11 @@ package org.apache.solr.core;
 import java.io.File;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.Locale;
 
 import org.apache.commons.io.FileUtils;
@@ -151,5 +156,22 @@ public class StandardDirectoryFactory extends CachingDirectoryFactory
{
     
     return baseDir;
   }
+  
+  // perform an atomic rename if possible
+  public void renameWithOverwrite(Directory dir, String fileName, String toName) throws IOException
{
+    Directory baseDir = getBaseDir(dir);
+    if (baseDir instanceof FSDirectory) {
+      Path path = ((FSDirectory) baseDir).getDirectory().toAbsolutePath();
+      try {
+      Files.move(FileSystems.getDefault().getPath(path.toString(), fileName),
+          FileSystems.getDefault().getPath(path.toString(), toName), StandardCopyOption.ATOMIC_MOVE,
StandardCopyOption.REPLACE_EXISTING);
+      } catch (AtomicMoveNotSupportedException e) {
+        Files.move(FileSystems.getDefault().getPath(path.toString(), fileName),
+            FileSystems.getDefault().getPath(path.toString(), toName), StandardCopyOption.REPLACE_EXISTING);
+      }
+    } else {
+      super.renameWithOverwrite(dir, fileName, toName);
+    }
+  }
 
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96ed221f/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index bdbd4e7..8bdd2b8 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -685,15 +685,19 @@ public class IndexFetcher {
         sb = readToStringBuilder(replicationTime, props.getProperty(REPLICATION_FAILED_AT_LIST));
         props.setProperty(REPLICATION_FAILED_AT_LIST, sb.toString());
       }
-
-      final IndexOutput out = dir.createOutput(REPLICATION_PROPERTIES, DirectoryFactory.IOCONTEXT_NO_CACHE);
+      
+      
+      String tmpFileName = REPLICATION_PROPERTIES + "." + System.nanoTime();
+      final IndexOutput out = dir.createOutput(tmpFileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
       Writer outFile = new OutputStreamWriter(new PropertiesOutputStream(out), StandardCharsets.UTF_8);
       try {
         props.store(outFile, "Replication details");
-        dir.sync(Collections.singleton(REPLICATION_PROPERTIES));
+        dir.sync(Collections.singleton(tmpFileName));
       } finally {
         IOUtils.closeQuietly(outFile);
       }
+      
+      solrCore.getDirectoryFactory().renameWithOverwrite(dir, tmpFileName, REPLICATION_PROPERTIES);
     } catch (Exception e) {
       LOG.warn("Exception while updating statistics", e);
     } finally {
@@ -1206,24 +1210,23 @@ public class IndexFetcher {
           IOUtils.closeQuietly(is);
         }
       }
-      try {
-        dir.deleteFile(IndexFetcher.INDEX_PROPERTIES);
-      } catch (IOException e) {
-        // no problem
-      }
-      final IndexOutput out = dir.createOutput(IndexFetcher.INDEX_PROPERTIES, DirectoryFactory.IOCONTEXT_NO_CACHE);
+
+      String tmpFileName = IndexFetcher.INDEX_PROPERTIES + "." + System.nanoTime();
+      final IndexOutput out = dir.createOutput(tmpFileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
       p.put("index", tmpIdxDirName);
       Writer os = null;
       try {
         os = new OutputStreamWriter(new PropertiesOutputStream(out), StandardCharsets.UTF_8);
-        p.store(os, IndexFetcher.INDEX_PROPERTIES);
-        dir.sync(Collections.singleton(INDEX_PROPERTIES));
+        p.store(os, tmpFileName);
+        dir.sync(Collections.singleton(tmpFileName));
       } catch (Exception e) {
         throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
             "Unable to write " + IndexFetcher.INDEX_PROPERTIES, e);
       } finally {
         IOUtils.closeQuietly(os);
       }
+      
+      solrCore.getDirectoryFactory().renameWithOverwrite(dir, tmpFileName, IndexFetcher.INDEX_PROPERTIES);
       return true;
 
     } catch (IOException e1) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96ed221f/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
index 08c462b..685ef99 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
@@ -35,6 +35,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Date;
+import java.util.List;
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -304,7 +305,11 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
     // check details on the slave a couple of times before & after fetching
     for (int i = 0; i < 3; i++) {
       NamedList<Object> details = getDetails(slaveClient);
-      
+      List replicatedAtCount = (List) ((NamedList) details.get("slave")).get("indexReplicatedAtList");
+      if (i > 0) {
+        assertEquals(i, replicatedAtCount.size());
+      }
+
       assertEquals("slave isMaster?", 
                    "false", details.get("isMaster"));
       assertEquals("slave isSlave?", 


Mime
View raw message