lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1212677 - in /lucene/dev/branches/solrcloud/solr/core/src: java/org/apache/solr/cloud/ java/org/apache/solr/handler/ java/org/apache/solr/search/ test-files/solr/conf/
Date Fri, 09 Dec 2011 23:08:25 GMT
Author: markrmiller
Date: Fri Dec  9 23:08:24 2011
New Revision: 1212677

URL: http://svn.apache.org/viewvc?rev=1212677&view=rev
Log:
we don't need the custom work on the replication handler if we use a more standard master
setup with replicateOnCommit and if we don't use MockDirFactory which breaks things around
indexDir/newIndexDir

Modified:
    lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/RecoveryStrat.java
    lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
    lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
    lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
    lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-distrib-update.xml

Modified: lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/RecoveryStrat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/RecoveryStrat.java?rev=1212677&r1=1212676&r2=1212677&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/RecoveryStrat.java
(original)
+++ lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/RecoveryStrat.java
Fri Dec  9 23:08:24 2011
@@ -135,7 +135,6 @@ public class RecoveryStrat {
       
       ModifiableSolrParams solrParams = new ModifiableSolrParams();
       solrParams.set(ReplicationHandler.MASTER_URL, leaderUrl + "replication");
-      solrParams.set(ReplicationHandler.CMD_FORCE, true);
       
       replicationHandler.doFetch(solrParams);
       

Modified: lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java?rev=1212677&r1=1212676&r2=1212677&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
(original)
+++ lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
Fri Dec  9 23:08:24 2011
@@ -119,7 +119,6 @@ public class ReplicationHandler extends 
   public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception
{
     rsp.setHttpCaching(false);
     final SolrParams solrParams = req.getParams();
-    boolean force = solrParams.getBool(CMD_FORCE, false);
     String command = solrParams.get(COMMAND);
     if (command == null) {
       rsp.add(STATUS, OK_STATUS);
@@ -130,14 +129,6 @@ public class ReplicationHandler extends 
     // It gives the current 'replicateable' index version
     if (command.equals(CMD_INDEX_VERSION)) {
       IndexCommit commitPoint = indexCommitPoint;  // make a copy so it won't change
-      
-      // this is only set after commit or optimize or something - if it's not set,
-      // just use the most recent
-      if (commitPoint == null || force) {
-        commitPoint = core.getDeletionPolicy().getLatestCommit();
-        indexCommitPoint = commitPoint;
-      }
-      
       if (commitPoint != null && replicationEnabled.get()) {
         //
         // There is a race condition here.  The commit point may be changed / deleted by
the time
@@ -285,22 +276,17 @@ public class ReplicationHandler extends 
     if (!snapPullLock.tryLock())
       return;
     try {
-      
+      tempSnapPuller = snapPuller;
       if (masterUrl != null) {
         NamedList<Object> nl = solrParams.toNamedList();
         nl.remove(SnapPuller.POLL_INTERVAL);
         tempSnapPuller = new SnapPuller(nl, this, core);
-      } else {
-        tempSnapPuller = snapPuller;
       }
-      
-      tempSnapPuller.fetchLatestIndex(core, solrParams == null ? false : solrParams.getBool(CMD_FORCE,
false));
+      tempSnapPuller.fetchLatestIndex(core);
     } catch (Exception e) {
       LOG.error("SnapPull failed ", e);
     } finally {
-      if (snapPuller != null) {
-        tempSnapPuller = snapPuller;
-      }
+      tempSnapPuller = snapPuller;
       snapPullLock.unlock();
     }
   }
@@ -362,9 +348,7 @@ public class ReplicationHandler extends 
       Collection<String> files = new HashSet<String>(commit.getFileNames());
       for (String fileName : files) {
         if(fileName.endsWith(".lock")) continue;
-        // use new dir in case we are replicating from a full index replication
-        // and have not yet reloaded the core
-        File file = new File(core.getNewIndexDir(), fileName);
+        File file = new File(core.getIndexDir(), fileName);
         Map<String, Object> fileMeta = getFileInfo(file);
         result.add(fileMeta);
       }
@@ -447,7 +431,7 @@ public class ReplicationHandler extends 
   }
 
   boolean isPollingDisabled() {
-    return snapPuller == null ? false : snapPuller.isPollingDisabled();
+    return snapPuller.isPollingDisabled();
   }
 
   int getTimesReplicatedSinceStartup() {
@@ -459,7 +443,7 @@ public class ReplicationHandler extends 
   }
 
   long getIndexSize() {
-    return FileUtils.sizeOfDirectory(new File(core.getNewIndexDir()));
+    return FileUtils.sizeOfDirectory(new File(core.getIndexDir()));
   }
 
   /**
@@ -774,9 +758,9 @@ public class ReplicationHandler extends 
   }
 
 
-  void refreshCommitpoint(boolean force) {
+  void refreshCommitpoint() {
     IndexCommit commitPoint = core.getDeletionPolicy().getLatestCommit();
-    if(force || replicateOnCommit || (replicateOnOptimize && commitPoint.getSegmentCount()
== 1)) {
+    if(replicateOnCommit || (replicateOnOptimize && commitPoint.getSegmentCount()
== 1)) {
       indexCommitPoint = commitPoint;
     }
   }
@@ -1033,9 +1017,7 @@ public class ReplicationHandler extends 
           file = new File(core.getResourceLoader().getConfigDir(), cfileName);
         } else {
           //else read from the indexdirectory
-          // use new dir in case we are replicating from a full index replication
-          // and have not yet reloaded the core
-          file = new File(core.getNewIndexDir(), fileName);
+          file = new File(core.getIndexDir(), fileName);
         }
         if (file.exists() && file.canRead()) {
           inputStream = new FileInputStream(file);
@@ -1097,8 +1079,6 @@ public class ReplicationHandler extends 
 
   public static final String COMMAND = "command";
 
-  public static final String CMD_FORCE = "force";
-  
   public static final String CMD_DETAILS = "details";
 
   public static final String CMD_BACKUP = "backup";

Modified: lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/handler/SnapPuller.java?rev=1212677&r1=1212676&r2=1212677&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
(original)
+++ lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
Fri Dec  9 23:08:24 2011
@@ -33,6 +33,7 @@ import static org.apache.solr.handler.Re
 import org.apache.solr.request.LocalSolrQueryRequest;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.update.CommitUpdateCommand;
 import org.apache.solr.util.RefCounted;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -172,16 +173,12 @@ public class SnapPuller {
 
   /**
    * Gets the latest commit version and generation from the master
-   * @param force 
    */
   @SuppressWarnings("unchecked")
-  NamedList getLatestVersion(boolean force) throws IOException {
+  NamedList getLatestVersion() throws IOException {
     PostMethod post = new PostMethod(masterUrl);
     post.addParameter(COMMAND, CMD_INDEX_VERSION);
     post.addParameter("wt", "javabin");
-    if (force) {
-      post.addParameter(ReplicationHandler.CMD_FORCE, "true");
-    }
     return getNamedListResponse(post);
   }
 
@@ -247,20 +244,20 @@ public class SnapPuller {
   @SuppressWarnings("unchecked")
   boolean successfulInstall = false;
 
-  public boolean fetchLatestIndex(SolrCore core, boolean force) throws IOException {
+  boolean fetchLatestIndex(SolrCore core) throws IOException {
     replicationStartTime = System.currentTimeMillis();
     try {
       //get the current 'replicateable' index version in the master
       NamedList response = null;
       try {
-        response = getLatestVersion(force);
+        response = getLatestVersion();
       } catch (Exception e) {
         LOG.error("Master at: " + masterUrl + " is not available. Index fetch failed. Exception:
" + e.getMessage());
         return false;
       }
       long latestVersion = (Long) response.get(CMD_INDEX_VERSION);
       long latestGeneration = (Long) response.get(GENERATION);
-      if (latestVersion == 0L && !force) {
+      if (latestVersion == 0L) {
         //there is nothing to be replicated
         return false;
       }
@@ -273,7 +270,7 @@ public class SnapPuller {
         if (searcherRefCounted != null)
           searcherRefCounted.decref();
       }
-      if (!force && commit.getVersion() == latestVersion && commit.getGeneration()
== latestGeneration) {
+      if (commit.getVersion() == latestVersion && commit.getGeneration() == latestGeneration)
{
         //master and slave are alsready in sync just return
         LOG.info("Slave in sync with master.");
         return false;
@@ -301,7 +298,7 @@ public class SnapPuller {
       boolean deleteTmpIdxDir = true;
       File indexDir = null ;
       try {
-        indexDir = new File(core.getNewIndexDir());
+        indexDir = new File(core.getIndexDir());
         downloadIndexFiles(isFullCopyNeeded, tmpIndexDir, latestVersion);
         LOG.info("Total time taken for download : " + ((System.currentTimeMillis() - replicationStartTime)
/ 1000) + " secs");
         Collection<Map<String, Object>> modifiedConfFiles = getModifiedConfFiles(confFilesToDownload);
@@ -328,7 +325,7 @@ public class SnapPuller {
           }
           if (successfulInstall) {
             logReplicationTimeAndConfFiles(modifiedConfFiles, successfulInstall);
-            doCommit(isFullCopyNeeded);
+            doCommit();
           }
         }
         replicationStartTime = 0;
@@ -473,12 +470,20 @@ public class SnapPuller {
     return sb;
   }
 
-  private void doCommit(boolean isFullCopyNeeded) throws IOException {
-    // reboot the writer on the new index and get a new searcher
-    solrCore.getUpdateHandler().newIndexWriter();
-    solrCore.getSearcher(true, false, null);
-
-    replicationHandler.refreshCommitpoint(isFullCopyNeeded);
+  private void doCommit() throws IOException {
+    SolrQueryRequest req = new LocalSolrQueryRequest(solrCore,
+        new ModifiableSolrParams());
+    try {
+      
+      // reboot the writer on the new index and get a new searcher
+      solrCore.getUpdateHandler().newIndexWriter();
+      // update our commit point to the right dir
+      solrCore.getUpdateHandler().commit(new CommitUpdateCommand(req, false));
+      
+      replicationHandler.refreshCommitpoint();
+    } finally {
+      req.close();
+    }
   }
 
 
@@ -540,7 +545,7 @@ public class SnapPuller {
    */
   private void downloadIndexFiles(boolean downloadCompleteIndex, File tmpIdxDir, long latestVersion)
throws Exception {
     for (Map<String, Object> file : filesToDownload) {
-      File localIndexFile = new File(solrCore.getNewIndexDir(), (String) file.get(NAME));
+      File localIndexFile = new File(solrCore.getIndexDir(), (String) file.get(NAME));
       if (!localIndexFile.exists() || downloadCompleteIndex) {
         fileFetcher = new FileFetcher(tmpIdxDir, file, (String) file.get(NAME), false, latestVersion);
         currentFile = file;
@@ -560,7 +565,7 @@ public class SnapPuller {
    */
   private boolean isIndexStale() {
     for (Map<String, Object> file : filesToDownload) {
-      File localIndexFile = new File(solrCore.getNewIndexDir(), (String) file
+      File localIndexFile = new File(solrCore.getIndexDir(), (String) file
               .get(NAME));
       if (localIndexFile.exists()
               && localIndexFile.length() != (Long) file.get(SIZE)) {

Modified: lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1212677&r1=1212676&r2=1212677&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
(original)
+++ lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
Fri Dec  9 23:08:24 2011
@@ -131,6 +131,8 @@ public class SolrIndexSearcher extends I
     if (dir instanceof FSDirectory) {
       FSDirectory fsDirectory = (FSDirectory) dir;
       indexDir = fsDirectory.getDirectory().getAbsolutePath();
+    } else {
+      log.warn("WARNING: Directory impl does not support setting indexDir: " + dir.getClass().getName());
     }
 
     this.closeReader = closeReader;

Modified: lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-distrib-update.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-distrib-update.xml?rev=1212677&r1=1212676&r2=1212677&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-distrib-update.xml
(original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-distrib-update.xml
Fri Dec  9 23:08:24 2011
@@ -21,7 +21,7 @@
 		the default, is filesystem based. solr.RAMDirectoryFactory is memory based 
 		and not persistent. -->
 	<directoryFactory name="DirectoryFactory"
-		class="${solr.directoryFactory:solr.StandardDirectoryFactory}" />
+		class="solr.StandardDirectoryFactory" />
 
 	<updateHandler class="solr.DirectUpdateHandler2">
 
@@ -39,7 +39,9 @@
 	</requestDispatcher>
 	
 	<requestHandler name="/replication" class="solr.ReplicationHandler">
-
+      <lst name="master">
+        <str name="replicateAfter">commit</str>
+      </lst>
     </requestHandler>
     
     <query>



Mime
View raw message