lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1676520 - in /lucene/dev/trunk/solr: ./ core/src/java/org/apache/solr/update/ core/src/test-files/solr/collection1/conf/ core/src/test/org/apache/solr/search/
Date Tue, 28 Apr 2015 12:48:45 GMT
Author: markrmiller
Date: Tue Apr 28 12:48:44 2015
New Revision: 1676520

URL: http://svn.apache.org/r1676520
Log:
SOLR-7437: Make HDFS transaction log replication factor configurable.

Modified:
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java
    lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1676520&r1=1676519&r2=1676520&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Tue Apr 28 12:48:44 2015
@@ -151,6 +151,8 @@ New Features
   Example: prices:{ type:range, field:price, mincount:1, start:0, end:100, gap:10 }
   (yonik)
 
+* SOLR-7437: Make HDFS transaction log replication factor configurable. (Mark Miller)
+
 Bug Fixes
 ----------------------
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java?rev=1676520&r1=1676519&r2=1676520&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java Tue
Apr 28 12:48:44 2015
@@ -68,11 +68,11 @@ public class HdfsTransactionLog extends
 
   private volatile boolean isClosed = false;
 
-  HdfsTransactionLog(FileSystem fs, Path tlogFile, Collection<String> globalStrings)
{
-    this(fs, tlogFile, globalStrings, false);
+  HdfsTransactionLog(FileSystem fs, Path tlogFile, Collection<String> globalStrings,
Integer tlogDfsReplication) {
+    this(fs, tlogFile, globalStrings, false, tlogDfsReplication);
   }
 
-  HdfsTransactionLog(FileSystem fs, Path tlogFile, Collection<String> globalStrings,
boolean openExisting) {
+  HdfsTransactionLog(FileSystem fs, Path tlogFile, Collection<String> globalStrings,
boolean openExisting, Integer tlogDfsReplication) {
     super();
     boolean success = false;
     this.fs = fs;
@@ -95,7 +95,7 @@ public class HdfsTransactionLog extends
       } else {
         fs.delete(tlogFile, false);
         
-        tlogOutStream = fs.create(tlogFile, (short)1);
+        tlogOutStream = fs.create(tlogFile, (short)tlogDfsReplication.intValue());
         tlogOutStream.hsync();
       }
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java?rev=1676520&r1=1676519&r2=1676520&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java Tue Apr
28 12:48:44 2015
@@ -48,6 +48,7 @@ public class HdfsUpdateLog extends Updat
   private FileSystem fs;
   private volatile Path tlogDir;
   private final String confDir;
+  private Integer tlogDfsReplication;
   
   // used internally by tests to track total count of failed tran log loads in init
   public static AtomicLong INIT_FAILED_LOGS_COUNT = new AtomicLong();
@@ -87,9 +88,12 @@ public class HdfsUpdateLog extends Updat
 
     numRecordsToKeep = objToInt(info.initArgs.get("numRecordsToKeep"), 100);
     maxNumLogsToKeep = objToInt(info.initArgs.get("maxNumLogsToKeep"), 10);
+    
+    tlogDfsReplication = (Integer) info.initArgs.get( "tlogDfsReplication");
+    if (tlogDfsReplication == null) tlogDfsReplication = 1;
 
-    log.info("Initializing HdfsUpdateLog: dataDir={} defaultSyncLevel={} numRecordsToKeep={}
maxNumLogsToKeep={}",
-        dataDir, defaultSyncLevel, numRecordsToKeep, maxNumLogsToKeep);
+    log.info("Initializing HdfsUpdateLog: dataDir={} defaultSyncLevel={} numRecordsToKeep={}
maxNumLogsToKeep={} tlogDfsReplication={}",
+        dataDir, defaultSyncLevel, numRecordsToKeep, maxNumLogsToKeep, tlogDfsReplication);
   }
 
   private Configuration getConf() {
@@ -186,7 +190,7 @@ public class HdfsUpdateLog extends Updat
     for (String oldLogName : tlogFiles) {
       Path f = new Path(tlogDir, oldLogName);
       try {
-        oldLog = new HdfsTransactionLog(fs, f, null, true);
+        oldLog = new HdfsTransactionLog(fs, f, null, true, tlogDfsReplication);
         addOldLog(oldLog, false); // don't remove old logs on startup since more
                                   // than one may be uncapped.
       } catch (Exception e) {
@@ -296,7 +300,7 @@ public class HdfsUpdateLog extends Updat
       String newLogName = String.format(Locale.ROOT, LOG_FILENAME_PATTERN,
           TLOG_NAME, id);
       HdfsTransactionLog ntlog = new HdfsTransactionLog(fs, new Path(tlogDir, newLogName),
-          globalStrings);
+          globalStrings, tlogDfsReplication);
       tlog = ntlog;
       
       if (tlog != ntlog) {
@@ -333,7 +337,7 @@ public class HdfsUpdateLog extends Updat
     }
   }
   
-  private String[] getLogList(Path tlogDir) throws FileNotFoundException, IOException {
+  public String[] getLogList(Path tlogDir) throws FileNotFoundException, IOException {
     final String prefix = TLOG_NAME+'.';
     FileStatus[] files = fs.listStatus(tlogDir, new PathFilter() {
       

Modified: lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml?rev=1676520&r1=1676519&r2=1676520&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml (original)
+++ lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml Tue
Apr 28 12:48:44 2015
@@ -51,6 +51,7 @@
       <str name="dir">${solr.ulog.dir:}</str>
       <str name="maxNumLogsToKeep">${solr.ulog.maxNumLogsToKeep:10}</str>
       <str name="numRecordsToKeep">${solr.ulog.numRecordsToKeep:100}</str>
+      <int name="tlogDfsReplication">${solr.ulog.tlogDfsReplication:2}</int>
     </updateLog>
   </updateHandler>
 

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java?rev=1676520&r1=1676519&r2=1676520&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java Tue Apr
28 12:48:44 2015
@@ -37,7 +37,9 @@ import java.util.concurrent.TimeUnit;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.solr.SolrTestCaseJ4;
@@ -119,7 +121,30 @@ public class TestRecoveryHdfs extends So
     }
   }
 
-
+  @Test
+  public void testReplicationFactor() throws Exception {
+    clearIndex(); 
+    
+    HdfsUpdateLog ulog = (HdfsUpdateLog) h.getCore().getUpdateHandler().getUpdateLog();
+    
+    assertU(commit());
+    addAndGetVersion(sdoc("id", "REP1"), null);
+    assertU(commit());
+
+    String[] logList = ulog.getLogList(new Path(ulog.getLogDir()));
+    boolean foundRep2 = false;
+    for (String tl : logList) {
+       FileStatus status = fs.getFileStatus(new Path(ulog.getLogDir(), tl));
+       if (status.getReplication() == 2) {
+         foundRep2 = true;
+         break;
+       }
+    }
+    
+    assertTrue("Expected to find tlogs with a replication factor of 2", foundRep2);
+  }
+  
+  
   @Test
   public void testLogReplay() throws Exception {
     try {



Mime
View raw message