lucene-solr-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From no...@apache.org
Subject svn commit: r812922 - in /lucene/solr/trunk/src/java/org/apache/solr/handler: ReplicationHandler.java SnapPuller.java
Date Wed, 09 Sep 2009 12:25:31 GMT
Author: noble
Date: Wed Sep  9 12:25:31 2009
New Revision: 812922

URL: http://svn.apache.org/viewvc?rev=812922&view=rev
Log:
SOLR-1330 the details command shows current replication status when no replication is going
on . enhanced details command to show history

Modified:
    lucene/solr/trunk/src/java/org/apache/solr/handler/ReplicationHandler.java
    lucene/solr/trunk/src/java/org/apache/solr/handler/SnapPuller.java

Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/ReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/ReplicationHandler.java?rev=812922&r1=812921&r2=812922&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/ReplicationHandler.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/ReplicationHandler.java Wed Sep  9
12:25:31 2009
@@ -191,7 +191,7 @@
         NamedList nl = new NamedList();
         nl.add("indexVersion", c.getVersion());
         nl.add(GENERATION, c.getGeneration());
-        nl.add(CMD_GET_FILE_LIST, c.getFileNames().toString());
+        nl.add(CMD_GET_FILE_LIST, c.getFileNames());
         l.add(nl);
       } catch (IOException e) {
         LOG.warn("Exception while reading files for commit " + c, e);
@@ -502,20 +502,28 @@
         }
         list.add("isPollingDisabled", String.valueOf(isPollingDisabled()));
         list.add("isReplicating", String.valueOf(isReplicating()));
+        long elapsed = getTimeElapsed(snapPuller);
+        long val = SnapPuller.getTotalBytesDownloaded(snapPuller);
+        if (elapsed > 0) {
+          list.add("timeElapsed", elapsed);
+          list.add("bytesDownloaded", val);
+          list.add("downloadSpeed", val / elapsed);
+        }
+        Properties props = loadReplicationProperties();
+        addVal(list, SnapPuller.PREVIOUS_CYCLE_TIME_TAKEN, props, Long.class);
+        addVal(list, SnapPuller.INDEX_REPLICATED_AT, props, Date.class);
+        addVal(list, SnapPuller.CONF_FILES_REPLICATED_AT, props, Date.class);
+        addVal(list, SnapPuller.REPLICATION_FAILED_AT, props, Date.class);
+        addVal(list, SnapPuller.TIMES_FAILED, props, Integer.class);
+        addVal(list, SnapPuller.TIMES_INDEX_REPLICATED, props, Integer.class);
+        addVal(list, SnapPuller.LAST_CYCLE_BYTES_DOWNLOADED, props, Long.class);
+        addVal(list, SnapPuller.TIMES_CONFIG_REPLICATED, props, Integer.class);
+        addVal(list, SnapPuller.CONF_FILES_REPLICATED, props, String.class);
       }
       if (isMaster) {
-        if (includeConfFiles != null)
-          list.add("confFilesToReplicate", includeConfFiles);
-        String replicateAfterString="";
-        if (replicateOnCommit)
-          replicateAfterString += "commit, ";
-        if (replicateOnOptimize)
-          replicateAfterString += "optimize, ";
-        if(replicateOnStart)
-          replicateAfterString += "startup, ";
-        if(replicateAfterString.lastIndexOf(',') > -1)
-          replicateAfterString = replicateAfterString.substring(0, replicateAfterString.lastIndexOf(','));
-        list.add(REPLICATE_AFTER, replicateAfterString);
+        if (includeConfFiles != null) list.add("confFilesToReplicate", includeConfFiles);
+        list.add(REPLICATE_AFTER, getReplicateAfterStrings());
+        list.add("replicationEnabled", String.valueOf(replicationEnabled.get()));
       }
     }
     return list;
@@ -523,14 +531,13 @@
 
   /**
    * Used for showing statistics and progress information.
+   *
    * @param showSlaveDetails
    */
   private NamedList<Object> getReplicationDetails(boolean showSlaveDetails) {
-    String timeLastReplicated = "", confFilesReplicated = "", confFilesReplicatedTime = "",
timesIndexReplicated = "", timesConfigReplicated = "";
     NamedList<Object> details = new SimpleOrderedMap<Object>();
     NamedList<Object> master = new SimpleOrderedMap<Object>();
     NamedList<Object> slave = new SimpleOrderedMap<Object>();
-    FileInputStream inFile = null;
 
     details.add("indexSize", readableSize(getIndexSize()));
     details.add("indexPath", core.getIndexDir());
@@ -544,18 +551,9 @@
     IndexCommit commit = indexCommitPoint;  // make a copy so it won't change
 
     if (isMaster) {
-      if (includeConfFiles != null)
-        master.add(CONF_FILES, includeConfFiles);
-      String replicateAfterString="";
-      if (replicateOnCommit)
-        replicateAfterString += "commit, ";
-      if (replicateOnOptimize)
-        replicateAfterString += "optimize, ";
-      if(replicateOnStart)
-        replicateAfterString += "startup, ";
-      if(replicateAfterString.lastIndexOf(',') > -1)
-        replicateAfterString = replicateAfterString.substring(0, replicateAfterString.lastIndexOf(','));
-      master.add(REPLICATE_AFTER, replicateAfterString);
+      if (includeConfFiles != null) master.add(CONF_FILES, includeConfFiles);
+      master.add(REPLICATE_AFTER, getReplicateAfterStrings());
+      master.add("replicationEnabled", String.valueOf(replicationEnabled.get()));
     }
 
     if (isMaster && commit != null) {
@@ -565,161 +563,189 @@
 
     SnapPuller snapPuller = tempSnapPuller;
     if (showSlaveDetails && snapPuller != null) {
-      try {
-        Properties props = new Properties();
-        File f = new File(core.getDataDir(), SnapPuller.REPLICATION_PROPERTIES);
-        if (f.exists()) {
-          inFile = new FileInputStream(f);
-          props.load(inFile);
-          timeLastReplicated = props.getProperty("indexReplicatedAt");
-          if (props.containsKey("timesIndexReplicated"))
-            timesIndexReplicated = props.getProperty("timesIndexReplicated");
-          if (props.containsKey("confFilesReplicated"))
-            confFilesReplicated = props.getProperty("confFilesReplicated");
-          if (props.containsKey("confFilesReplicatedAt"))
-            confFilesReplicatedTime = props.getProperty("confFilesReplicatedAt");
-          if (props.containsKey("timesConfigReplicated"))
-            timesConfigReplicated = props.getProperty("timesConfigReplicated");
-        }
-      } catch (Exception e) {
-        LOG.warn("Exception while reading " + SnapPuller.REPLICATION_PROPERTIES);
-      } finally {
-        IOUtils.closeQuietly(inFile);
-      }
+      Properties props = loadReplicationProperties();
       try {
         NamedList<String> command = new NamedList<String>();
-        command.add(COMMAND,CMD_DETAILS);
-        command.add("slave","false");
+        command.add(COMMAND, CMD_DETAILS);
+        command.add("slave", "false");
         NamedList nl = snapPuller.getCommandResponse(command);
         slave.add("masterDetails", nl.get(CMD_DETAILS));
       } catch (Exception e) {
-        LOG.warn("Exception while invoking a 'details' method on master ", e);
-        slave.add(ERR_STATUS,"invalid_master");
+        LOG.warn("Exception while invoking 'details' method for replication on master ",
e);
+        slave.add(ERR_STATUS, "invalid_master");
       }
       slave.add(MASTER_URL, snapPuller.getMasterUrl());
       if (snapPuller.getPollInterval() != null) {
         slave.add(SnapPuller.POLL_INTERVAL, snapPuller.getPollInterval());
       }
       if (snapPuller.getNextScheduledExecTime() != null && !isPollingDisabled())
{
-        Date d = new Date(snapPuller.getNextScheduledExecTime());
-        slave.add("nextExecutionAt", d.toString());
+        slave.add(NEXT_EXECUTION_AT, new Date(snapPuller.getNextScheduledExecTime()).toString());
       } else if (isPollingDisabled()) {
-        slave.add("nextExecutionAt", "Polling disabled");
-      } else
-        slave.add("nextExecutionAt", "");
-
-      if (timeLastReplicated != null && timeLastReplicated.length() > 0) {
-        Date d = new Date(Long.valueOf(timeLastReplicated));
-        slave.add("indexReplicatedAt", d.toString());
-      } else {
-        slave.add("indexReplicatedAt", "");
-      }
-      slave.add("timesIndexReplicated", timesIndexReplicated);
-      slave.add("confFilesReplicated", confFilesReplicated);
-      slave.add("timesConfigReplicated", timesConfigReplicated);
-      if (confFilesReplicatedTime != null && confFilesReplicatedTime.length() >
0) {
-        Date d = new Date(Long.valueOf(confFilesReplicatedTime));
-        slave.add("confFilesReplicatedAt", d.toString());
-      } else {
-        slave.add("confFilesReplicatedAt", confFilesReplicatedTime);
+        slave.add(NEXT_EXECUTION_AT, "Polling disabled");
       }
-
-      try {
-        long bytesToDownload = 0;
-        List<String> filesToDownload = new ArrayList<String>();
-        if (snapPuller.getFilesToDownload() != null) {
+      addVal(slave, SnapPuller.INDEX_REPLICATED_AT, props, Date.class);
+      addVal(slave, SnapPuller.INDEX_REPLICATED_AT_LIST, props, List.class);
+      addVal(slave, SnapPuller.REPLICATION_FAILED_AT_LIST, props, List.class);
+      addVal(slave, SnapPuller.TIMES_INDEX_REPLICATED, props, Integer.class);
+      addVal(slave, SnapPuller.CONF_FILES_REPLICATED, props, Integer.class);
+      addVal(slave, SnapPuller.TIMES_CONFIG_REPLICATED, props, Integer.class);
+      addVal(slave, SnapPuller.CONF_FILES_REPLICATED_AT, props, Integer.class);
+      addVal(slave, SnapPuller.LAST_CYCLE_BYTES_DOWNLOADED, props, Long.class);
+      addVal(slave, SnapPuller.TIMES_FAILED, props, Integer.class);
+      addVal(slave, SnapPuller.REPLICATION_FAILED_AT, props, Date.class);
+      addVal(slave, SnapPuller.PREVIOUS_CYCLE_TIME_TAKEN, props, Long.class);
+
+
+      boolean isReplicating = isReplicating();
+      slave.add("isReplicating", String.valueOf(isReplicating));
+      if (isReplicating) {
+        try {
+          long bytesToDownload = 0;
+          List<String> filesToDownload = new ArrayList<String>();
           for (Map<String, Object> file : snapPuller.getFilesToDownload()) {
             filesToDownload.add((String) file.get(NAME));
             bytesToDownload += (Long) file.get(SIZE);
           }
-        }
 
-        //get list of conf files to download
-        for (Map<String, Object> file : snapPuller.getConfFilesToDownload()) {
-          filesToDownload.add((String) file.get(NAME));
-          bytesToDownload += (Long) file.get(SIZE);
-        }
-
-        slave.add("filesToDownload", filesToDownload.toString());
-        slave.add("numFilesToDownload", String.valueOf(filesToDownload.size()));
-        slave.add("bytesToDownload", readableSize(bytesToDownload));
-
-        long bytesDownloaded = 0;
-        List<String> filesDownloaded = new ArrayList<String>();
-        for (Map<String, Object> file : snapPuller.getFilesDownloaded()) {
-          filesDownloaded.add((String) file.get(NAME));
-          bytesDownloaded += (Long) file.get(SIZE);
-        }
-
-        //get list of conf files downloaded
-        for (Map<String, Object> file : snapPuller.getConfFilesDownloaded()) {
-          filesDownloaded.add((String) file.get(NAME));
-          bytesDownloaded += (Long) file.get(SIZE);
-        }
-
-        slave.add("filesDownloaded", filesDownloaded.toString());
-        slave.add("numFilesDownloaded", String.valueOf(filesDownloaded.size()));
-
-        Map<String, Object> currentFile = snapPuller.getCurrentFile();
-        String currFile = null;
-        long currFileSize = 0, currFileSizeDownloaded = 0;
-        float percentDownloaded = 0;
-        if (currentFile != null) {
-          currFile = (String) currentFile.get(NAME);
-          currFileSize = (Long) currentFile.get(SIZE);
-          if (currentFile.containsKey("bytesDownloaded")) {
-            currFileSizeDownloaded = (Long) currentFile.get("bytesDownloaded");
-            bytesDownloaded += currFileSizeDownloaded;
-            if (currFileSize > 0)
-              percentDownloaded = (currFileSizeDownloaded * 100) / currFileSize;
+          //get list of conf files to download
+          for (Map<String, Object> file : snapPuller.getConfFilesToDownload()) {
+            filesToDownload.add((String) file.get(NAME));
+            bytesToDownload += (Long) file.get(SIZE);
           }
-        }
 
-        long timeElapsed = 0, estimatedTimeRemaining = 0;
-        Date replicationStartTime = null;
-        if (snapPuller.getReplicationStartTime() > 0) {
-          replicationStartTime = new Date(snapPuller.getReplicationStartTime());
-          timeElapsed = (System.currentTimeMillis() - snapPuller.getReplicationStartTime())
/ 1000;
-        }
-        if (replicationStartTime != null) {
-          slave.add("replicationStartTime", replicationStartTime.toString());
-        }
-        slave.add("timeElapsed", String.valueOf(timeElapsed) + "s");
-
-        if (bytesDownloaded > 0)
-          estimatedTimeRemaining = ((bytesToDownload - bytesDownloaded) * timeElapsed) /
bytesDownloaded;
-        float totalPercent = 0;
-        long downloadSpeed = 0;
-        if (bytesToDownload > 0)
-          totalPercent = (bytesDownloaded * 100) / bytesToDownload;
-        if (timeElapsed > 0)
-          downloadSpeed = (bytesDownloaded / timeElapsed);
-        if (currFile != null)
-          slave.add("currentFile", currFile);
-        slave.add("currentFileSize", readableSize(currFileSize));
-        slave.add("currentFileSizeDownloaded", readableSize(currFileSizeDownloaded));
-        slave.add("currentFileSizePercent", String.valueOf(percentDownloaded));
-        slave.add("bytesDownloaded", readableSize(bytesDownloaded));
-        slave.add("totalPercent", String.valueOf(totalPercent));
-        slave.add("timeRemaining", String.valueOf(estimatedTimeRemaining) + "s");
-        slave.add("downloadSpeed", readableSize(downloadSpeed));
-        slave.add("isPollingDisabled", String.valueOf(isPollingDisabled()));
-        slave.add("isReplicating", String.valueOf(isReplicating()));
-      } catch (Exception e) {
-        LOG.error("Exception while writing details: ", e);
-      }
+          slave.add("filesToDownload", filesToDownload);
+          slave.add("numFilesToDownload", String.valueOf(filesToDownload.size()));
+          slave.add("bytesToDownload", readableSize(bytesToDownload));
+
+          long bytesDownloaded = 0;
+          List<String> filesDownloaded = new ArrayList<String>();
+          for (Map<String, Object> file : snapPuller.getFilesDownloaded()) {
+            filesDownloaded.add((String) file.get(NAME));
+            bytesDownloaded += (Long) file.get(SIZE);
+          }
+
+          //get list of conf files downloaded
+          for (Map<String, Object> file : snapPuller.getConfFilesDownloaded()) {
+            filesDownloaded.add((String) file.get(NAME));
+            bytesDownloaded += (Long) file.get(SIZE);
+          }
+
+          Map<String, Object> currentFile = snapPuller.getCurrentFile();
+          String currFile = null;
+          long currFileSize = 0, currFileSizeDownloaded = 0;
+          float percentDownloaded = 0;
+          if (currentFile != null) {
+            currFile = (String) currentFile.get(NAME);
+            currFileSize = (Long) currentFile.get(SIZE);
+            if (currentFile.containsKey("bytesDownloaded")) {
+              currFileSizeDownloaded = (Long) currentFile.get("bytesDownloaded");
+              bytesDownloaded += currFileSizeDownloaded;
+              if (currFileSize > 0)
+                percentDownloaded = (currFileSizeDownloaded * 100) / currFileSize;
+            }
+          }
+          slave.add("filesDownloaded", filesDownloaded);
+          slave.add("numFilesDownloaded", String.valueOf(filesDownloaded.size()));
+
+          long estimatedTimeRemaining = 0;
+
+          if (snapPuller.getReplicationStartTime() > 0) {
+            slave.add("replicationStartTime", new Date(snapPuller.getReplicationStartTime()).toString());
+          }
+          long elapsed = getTimeElapsed(snapPuller);
+          slave.add("timeElapsed", String.valueOf(elapsed) + "s");
+
+          if (bytesDownloaded > 0)
+            estimatedTimeRemaining = ((bytesToDownload - bytesDownloaded) * elapsed) / bytesDownloaded;
+          float totalPercent = 0;
+          long downloadSpeed = 0;
+          if (bytesToDownload > 0)
+            totalPercent = (bytesDownloaded * 100) / bytesToDownload;
+          if (elapsed > 0)
+            downloadSpeed = (bytesDownloaded / elapsed);
+          if (currFile != null)
+            slave.add("currentFile", currFile);
+          slave.add("currentFileSize", readableSize(currFileSize));
+          slave.add("currentFileSizeDownloaded", readableSize(currFileSizeDownloaded));
+          slave.add("currentFileSizePercent", String.valueOf(percentDownloaded));
+          slave.add("bytesDownloaded", readableSize(bytesDownloaded));
+          slave.add("totalPercent", String.valueOf(totalPercent));
+          slave.add("timeRemaining", String.valueOf(estimatedTimeRemaining) + "s");
+          slave.add("downloadSpeed", readableSize(downloadSpeed));
+          slave.add("isPollingDisabled", String.valueOf(isPollingDisabled()));
+        } catch (Exception e) {
+          LOG.error("Exception while writing replication details: ", e);
+        }
+      }
+      if (isMaster)
+        details.add("master", master);
+      if (isSlave && showSlaveDetails)
+        details.add("slave", slave);
+      NamedList snapshotStats = snapShootDetails;
+      if (snapshotStats != null)
+        details.add(CMD_BACKUP, snapshotStats);
     }
-    if(isMaster)
-      details.add("master", master);
-    if(isSlave && showSlaveDetails)
-      details.add("slave", slave);
-    NamedList snapshotStats = snapShootDetails;
-    if (snapshotStats != null)
-      details.add(CMD_BACKUP, snapshotStats);
     return details;
   }
 
-  void refreshCommitpoint(){
+  private void addVal(NamedList nl, String key, Properties props, Class clzz) {
+    String s = props.getProperty(key);
+    if (s == null || s.trim().length() == 0) return;
+    if (clzz == Date.class) {
+      try {
+        Long l = Long.parseLong(s);
+        nl.add(key, new Date(l).toString());
+      } catch (NumberFormatException e) {/*no op*/ }
+    } else if (clzz == List.class) {
+      String ss[] = s.split(",");
+      List<String> l = new ArrayList<String>();
+      for (int i = 0; i < ss.length; i++) {
+        l.add(new Date(Long.valueOf(ss[i])).toString());
+      }
+      nl.add(key, l);
+    } else {
+      nl.add(key, s);
+    }
+
+  }
+
+  private List<String> getReplicateAfterStrings() {
+    List<String> replicateAfter = new ArrayList<String>();
+    if (replicateOnCommit)
+      replicateAfter.add("commit");
+    if (replicateOnOptimize)
+      replicateAfter.add("optimize");
+    if (replicateOnStart)
+      replicateAfter.add("startup");
+    return replicateAfter;
+  }
+
+  private long getTimeElapsed(SnapPuller snapPuller) {
+    long timeElapsed = 0;
+    if (snapPuller.getReplicationStartTime() > 0)
+      timeElapsed = (System.currentTimeMillis() - snapPuller.getReplicationStartTime()) /
1000;
+    return timeElapsed;
+  }
+
+  Properties loadReplicationProperties() {
+    FileInputStream inFile = null;
+    Properties props = new Properties();
+    try {
+      File f = new File(core.getDataDir(), SnapPuller.REPLICATION_PROPERTIES);
+      if (f.exists()) {
+        inFile = new FileInputStream(f);
+        props.load(inFile);
+      }
+    } catch (Exception e) {
+      LOG.warn("Exception while reading " + SnapPuller.REPLICATION_PROPERTIES);
+    } finally {
+      IOUtils.closeQuietly(inFile);
+    }
+    return props;
+  }
+
+
+  void refreshCommitpoint() {
     IndexCommit commitPoint = core.getDeletionPolicy().getLatestCommit();
     if(replicateOnCommit && !commitPoint.isOptimized()){
       indexCommitPoint = commitPoint;
@@ -778,8 +804,8 @@
             LOG.warn("The update handler being used is not an instance or sub-class of DirectUpdateHandler2.
" +
                     "Replicate on Startup cannot work.");
           }
-          if(s.get().getReader().getIndexCommit() != null)
-            if(s.get().getReader().getIndexCommit().getGeneration() != 1L)
+          if (s.get().getReader().getIndexCommit() != null)
+            if (s.get().getReader().getIndexCommit().getGeneration() != 1L)
               indexCommitPoint = s.get().getReader().getIndexCommit();
         } catch (IOException e) {
           LOG.warn("Unable to get IndexCommit on startup", e);
@@ -1045,5 +1071,5 @@
 
   public static final String OK_STATUS = "OK";
 
-
+  public static final String NEXT_EXECUTION_AT = "nextExecutionAt";
 }

Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/SnapPuller.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/SnapPuller.java?rev=812922&r1=812921&r2=812922&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/SnapPuller.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/SnapPuller.java Wed Sep  9 12:25:31
2009
@@ -104,8 +104,6 @@
   private static HttpClient client;
   // HttpClient for this instance if connectionTimeout or readTimeout has been specified
   private final HttpClient myHttpClient;
-  private static final String HTTP_BASIC_AUTH_USER = "httpBasicAuthUser";
-  private static final String HTTP_BASIC_AUTH_PASSWORD = "httpBasicAuthPassword";
 
   private static synchronized HttpClient createHttpClient(String connTimeout, String readTimeout)
{
     if (connTimeout == null && readTimeout == null && client != null)  return
client;
@@ -184,7 +182,7 @@
     PostMethod post = new PostMethod(masterUrl);
     for (Map.Entry<String, String> c : commands) {
       post.addParameter(c.getKey(),c.getValue());
-    } 
+    }
     post.addParameter("wt", "javabin");
     return getNamedListResponse(post);
   }
@@ -227,12 +225,12 @@
    * downloaded. It also downloads the conf files (if they are modified).
    *
    * @param core the SolrCore
-   *
    * @return true on success, false if slave is already in sync
-   *
    * @throws IOException if an exception occurs
    */
   @SuppressWarnings("unchecked")
+  boolean successfulInstall = false;
+
   boolean fetchLatestIndex(SolrCore core) throws IOException {
     replicationStartTime = System.currentTimeMillis();
     try {
@@ -241,7 +239,7 @@
       try {
         response = getLatestVersion();
       } catch (Exception e) {
-        LOG.error("Master at: "+masterUrl + " is not available. Index fetch failed. Exception:
" + e.getMessage());
+        LOG.error("Master at: " + masterUrl + " is not available. Index fetch failed. Exception:
" + e.getMessage());
         return false;
       }
       long latestVersion = (Long) response.get(CMD_INDEX_VERSION);
@@ -281,7 +279,7 @@
       File tmpIndexDir = createTempindexDir(core);
       if (isIndexStale())
         isFullCopyNeeded = true;
-      boolean successfulInstall = false;
+      successfulInstall = false;
       boolean deleteTmpIdxDir = true;
       try {
         File indexDir = new File(core.getIndexDir());
@@ -297,7 +295,7 @@
           }
           if (successfulInstall) {
             LOG.info("Configuration files are modified, core will be reloaded");
-            logReplicationTimeAndConfFiles(modifiedConfFiles);//write to a file time of replication
and conf files.
+            logReplicationTimeAndConfFiles(modifiedConfFiles, successfulInstall);//write
to a file time of replication and conf files.
             reloadCore();
           }
         } else {
@@ -309,7 +307,7 @@
             successfulInstall = copyIndexFiles(tmpIndexDir, indexDir);
           }
           if (successfulInstall) {
-            logReplicationTimeAndConfFiles(modifiedConfFiles);
+            logReplicationTimeAndConfFiles(modifiedConfFiles, successfulInstall);
             doCommit();
           }
         }
@@ -326,6 +324,9 @@
       }
       return successfulInstall;
     } finally {
+      if (!successfulInstall) {
+        logReplicationTimeAndConfFiles(null, successfulInstall);
+      }
       filesToDownload = filesDownloaded = confFilesDownloaded = confFilesToDownload = null;
       replicationStartTime = 0;
       fileFetcher = null;
@@ -357,47 +358,100 @@
    * Helper method to record the last replication's details so that we can show them on the
statistics page across
    * restarts.
    */
-  private void logReplicationTimeAndConfFiles(Collection<Map<String, Object>>
modifiedConfFiles) {
+  private void logReplicationTimeAndConfFiles(Collection<Map<String, Object>>
modifiedConfFiles, boolean successfulInstall) {
     FileOutputStream outFile = null;
-    FileInputStream inFile = null;
     List<String> confFiles = new ArrayList<String>();
     if (modifiedConfFiles != null && !modifiedConfFiles.isEmpty())
       for (Map<String, Object> map1 : modifiedConfFiles)
         confFiles.add((String) map1.get(NAME));
 
-    Properties props = new Properties();
+    Properties props = replicationHandler.loadReplicationProperties();
     long replicationTime = System.currentTimeMillis();
+    long replicationTimeTaken = (replicationTime - getReplicationStartTime()) / 1000;
     try {
-      File f = new File(solrCore.getDataDir(), REPLICATION_PROPERTIES);
       int indexCount = 1, confFilesCount = 1;
-      if (f.exists()) {
-        inFile = new FileInputStream(f);
-        props.load(inFile);
-      }
-      if (props.containsKey("timesIndexReplicated")) {
-        indexCount = Integer.valueOf(props.getProperty("timesIndexReplicated")) + 1;
+      if (props.containsKey(TIMES_INDEX_REPLICATED)) {
+        indexCount = Integer.valueOf(props.getProperty(TIMES_INDEX_REPLICATED)) + 1;
       }
-      props.setProperty("timesIndexReplicated", String.valueOf(indexCount));
-      props.setProperty("indexReplicatedAt", String.valueOf(replicationTime));
+      StringBuffer sb = readToStringBuffer(replicationTime, props.getProperty(INDEX_REPLICATED_AT_LIST));
+      props.setProperty(INDEX_REPLICATED_AT_LIST, sb.toString());
+      props.setProperty(INDEX_REPLICATED_AT, String.valueOf(replicationTime));
+      props.setProperty(PREVIOUS_CYCLE_TIME_TAKEN, String.valueOf(replicationTimeTaken));
+      props.setProperty(TIMES_INDEX_REPLICATED, String.valueOf(indexCount));
       if (modifiedConfFiles != null && !modifiedConfFiles.isEmpty()) {
-        props.setProperty("confFilesReplicated", confFiles.toString());
-        props.setProperty("confFilesReplicatedAt", String.valueOf(replicationTime));
-        if (props.containsKey("timesConfigReplicated")) {
-          confFilesCount = Integer.valueOf(props.getProperty("timesConfigReplicated")) +
1;
+        props.setProperty(CONF_FILES_REPLICATED, confFiles.toString());
+        props.setProperty(CONF_FILES_REPLICATED_AT, String.valueOf(replicationTime));
+        if (props.containsKey(TIMES_CONFIG_REPLICATED)) {
+          confFilesCount = Integer.valueOf(props.getProperty(TIMES_CONFIG_REPLICATED)) +
1;
+        }
+        props.setProperty(TIMES_CONFIG_REPLICATED, String.valueOf(confFilesCount));
+      }
+
+      props.setProperty(LAST_CYCLE_BYTES_DOWNLOADED, String.valueOf(getTotalBytesDownloaded(this)));
+      if (!successfulInstall) {
+        int numFailures = 1;
+        if (props.containsKey(TIMES_FAILED)) {
+          numFailures = Integer.valueOf(props.getProperty(TIMES_FAILED)) + 1;
         }
-        props.setProperty("timesConfigReplicated", String.valueOf(confFilesCount));
+        props.setProperty(TIMES_FAILED, String.valueOf(numFailures));
+        props.setProperty(REPLICATION_FAILED_AT, String.valueOf(replicationTime));
+        sb = readToStringBuffer(replicationTime, props.getProperty(REPLICATION_FAILED_AT_LIST));
+        props.setProperty(REPLICATION_FAILED_AT_LIST, sb.toString());
       }
+      File f = new File(solrCore.getDataDir(), REPLICATION_PROPERTIES);
       outFile = new FileOutputStream(f);
       props.store(outFile, "Replication details");
+      outFile.close();
     } catch (Exception e) {
       LOG.warn("Exception while updating statistics", e);
     }
     finally {
-      IOUtils.closeQuietly(inFile);
       IOUtils.closeQuietly(outFile);
     }
   }
 
+  static long getTotalBytesDownloaded(SnapPuller snappuller) {
+    long bytesDownloaded = 0;
+    //get size from list of files to download
+    for (Map<String, Object> file : snappuller.getFilesDownloaded()) {
+      bytesDownloaded += (Long) file.get(SIZE);
+    }
+
+    //get size from list of conf files to download
+    for (Map<String, Object> file : snappuller.getConfFilesDownloaded()) {
+      bytesDownloaded += (Long) file.get(SIZE);
+    }
+
+    //get size from current file being downloaded
+    Map<String, Object> currentFile = snappuller.getCurrentFile();
+    if (currentFile != null) {
+      if (currentFile.containsKey("bytesDownloaded")) {
+        bytesDownloaded += (Long) currentFile.get("bytesDownloaded");
+      }
+    }
+    return bytesDownloaded;
+  }
+
+  private StringBuffer readToStringBuffer(long replicationTime, String str) {
+    StringBuffer sb = new StringBuffer();
+    List<String> l = new ArrayList<String>();
+    if (str != null && str.length() != 0) {
+      String[] ss = str.split(",");
+      for (int i = 0; i < ss.length; i++) {
+        l.add(ss[i]);
+      }
+    }
+    sb.append(replicationTime);
+    if (!l.isEmpty()) {
+      for (int i = 0; i < l.size() || i < 9; i++) {
+        if (i == l.size() || i == 9) break;
+        String s = l.get(i);
+        sb.append(",").append(s);
+      }
+    }
+    return sb;
+  }
+
   private void doCommit() throws IOException {
     CommitUpdateCommand cmd = new CommitUpdateCommand(false);
     cmd.waitFlush = true;
@@ -1092,6 +1146,32 @@
   private static final Pattern INTERVAL_PATTERN = Pattern.compile("(\\d*?):(\\d*?):(\\d*)");
 
   private static final String HTTP_CONN_TIMEOUT = "httpConnTimeout";
-  
+
   private static final String HTTP_READ_TIMEOUT = "httpReadTimeout";
+
+  private static final String HTTP_BASIC_AUTH_USER = "httpBasicAuthUser";
+
+  private static final String HTTP_BASIC_AUTH_PASSWORD = "httpBasicAuthPassword";
+
+  static final String INDEX_REPLICATED_AT = "indexReplicatedAt";
+
+  static final String TIMES_INDEX_REPLICATED = "timesIndexReplicated";
+
+  static final String CONF_FILES_REPLICATED = "confFilesReplicated";
+
+  static final String CONF_FILES_REPLICATED_AT = "confFilesReplicatedAt";
+
+  static final String TIMES_CONFIG_REPLICATED = "timesConfigReplicated";
+
+  static final String LAST_CYCLE_BYTES_DOWNLOADED = "lastCycleBytesDownloaded";
+
+  static final String TIMES_FAILED = "timesFailed";
+
+  static final String REPLICATION_FAILED_AT = "replicationFailedAt";
+
+  static final String PREVIOUS_CYCLE_TIME_TAKEN = "previousCycleTimeInSeconds";
+
+  static final String INDEX_REPLICATED_AT_LIST = "indexReplicatedAtList";
+
+  static final String REPLICATION_FAILED_AT_LIST = "replicationFailedAtList";
 }



Mime
View raw message