lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sar...@apache.org
Subject svn commit: r1134829 - in /lucene/dev/trunk: ./ lucene/ lucene/contrib/ lucene/contrib/misc/src/java/org/apache/lucene/index/ lucene/contrib/misc/src/test/org/apache/lucene/index/ lucene/src/java/org/apache/lucene/index/ solr/
Date Sat, 11 Jun 2011 23:39:04 GMT
Author: sarowe
Date: Sat Jun 11 23:39:03 2011
New Revision: 1134829

URL: http://svn.apache.org/viewvc?rev=1134829&view=rev
Log:
LUCENE-3188: contrib/misc IndexSplitter creates indexes with incorrect SegmentInfos.counter;
added CheckIndex check & fix for this problem

Modified:
    lucene/dev/trunk/   (props changed)
    lucene/dev/trunk/lucene/   (props changed)
    lucene/dev/trunk/lucene/contrib/CHANGES.txt
    lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
    lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java
    lucene/dev/trunk/solr/   (props changed)

Modified: lucene/dev/trunk/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/CHANGES.txt?rev=1134829&r1=1134828&r2=1134829&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/contrib/CHANGES.txt Sat Jun 11 23:39:03 2011
@@ -81,9 +81,13 @@ API Changes
 
 Bug Fixes
 
-  * LUCENE-3185: Fix bug in NRTCachingDirectory.deleteFile that would
-    always throw exception and sometimes fail to actually delete the
-    file.  (Mike McCandless)
+ * LUCENE-3185: Fix bug in NRTCachingDirectory.deleteFile that would
+   always throw exception and sometimes fail to actually delete the
+   file.  (Mike McCandless)
+
+ * LUCENE-3188: contrib/misc IndexSplitter creates indexes with incorrect
+   SegmentInfos.counter; added CheckIndex check & fix for this problem.
+   (Ivan Dimitrov Vasilev via Steve Rowe)
 
 Build
 

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java?rev=1134829&r1=1134828&r2=1134829&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
(original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
Sat Jun 11 23:39:03 2011
@@ -147,6 +147,7 @@ public class IndexSplitter {
     destDir.mkdirs();
     FSDirectory destFSDir = FSDirectory.open(destDir);
     SegmentInfos destInfos = new SegmentInfos(codecs);
+    destInfos.counter = infos.counter;
     for (String n : segs) {
       SegmentInfo info = getInfo(n);
       destInfos.add(info);

Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java?rev=1134829&r1=1134828&r2=1134829&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
(original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
Sat Jun 11 23:39:03 2011
@@ -20,6 +20,7 @@ import java.io.File;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
@@ -91,4 +92,64 @@ public class TestIndexSplitter extends L
     r.close();
     fsDir.close();
   }
+
+  public void testDeleteThenOptimize() throws Exception {
+    // Create directories where the indexes will reside
+    File indexPath = new File(TEMP_DIR, "testfilesplitter");
+    _TestUtil.rmDir(indexPath);
+    indexPath.mkdirs();
+    File indexSplitPath = new File(TEMP_DIR, "testfilesplitterdest");
+    _TestUtil.rmDir(indexSplitPath);
+    indexSplitPath.mkdirs();
+    
+    // Create the original index
+    LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
+    mergePolicy.setNoCFSRatio(1);
+    IndexWriterConfig iwConfig
+        = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
+              .setOpenMode(OpenMode.CREATE)
+              .setMergePolicy(mergePolicy);
+    Directory fsDir = newFSDirectory(indexPath);
+    IndexWriter indexWriter = new IndexWriter(fsDir, iwConfig);
+    Document doc = new Document();
+    doc.add(new Field("content", "doc 1", Field.Store.YES, Field.Index.ANALYZED_NO_NORMS));
+    indexWriter.addDocument(doc);
+    doc = new Document();
+    doc.add(new Field("content", "doc 2", Field.Store.YES, Field.Index.ANALYZED_NO_NORMS));
+    indexWriter.addDocument(doc);
+    indexWriter.close();
+    fsDir.close();
+    
+    // Create the split index
+    IndexSplitter indexSplitter = new IndexSplitter(indexPath);
+    String splitSegName = indexSplitter.infos.info(0).name;
+    indexSplitter.split(indexSplitPath, new String[] {splitSegName});
+
+    // Delete the first document in the split index
+    Directory fsDirDest = newFSDirectory(indexSplitPath);
+    IndexReader indexReader = IndexReader.open(fsDirDest, false);
+    indexReader.deleteDocument(0);
+    assertEquals(1, indexReader.numDocs());
+    indexReader.close();
+    fsDirDest.close();
+
+    // Optimize the split index
+    mergePolicy = new LogByteSizeMergePolicy();
+    mergePolicy.setNoCFSRatio(1);
+    iwConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
+                   .setOpenMode(OpenMode.APPEND)
+                   .setMergePolicy(mergePolicy);
+    fsDirDest = newFSDirectory(indexSplitPath);
+    indexWriter = new IndexWriter(fsDirDest, iwConfig);
+    indexWriter.optimize();
+    indexWriter.close();
+    fsDirDest.close();
+
+    // Read the number of docs in the index
+    fsDirDest = newFSDirectory(indexSplitPath);
+    indexReader = IndexReader.open(fsDirDest);
+	  assertEquals(1, indexReader.numDocs());
+    indexReader.close();
+    fsDirDest.close();
+  }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1134829&r1=1134828&r2=1134829&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java Sat Jun 11 23:39:03
2011
@@ -119,6 +119,12 @@ public class CheckIndex {
      * argument). */
     public boolean partial;
 
+    /** The greatest segment name. */
+    public int maxSegmentName;
+
+    /** Whether the SegmentInfos.counter is greater than any of the segments' names. */
+    public boolean validCounter; 
+
     /** Holds the userData of the last commit in the index */
     public Map<String, String> userData;
 
@@ -416,6 +422,10 @@ public class CheckIndex {
 
     for(int i=0;i<numSegments;i++) {
       final SegmentInfo info = sis.info(i);
+      int segmentName = Integer.parseInt(info.name.substring(1), Character.MAX_RADIX);
+      if (segmentName > result.maxSegmentName) {
+        result.maxSegmentName = segmentName;
+      }
       if (onlySegments != null && !onlySegments.contains(info.name))
         continue;
       Status.SegmentInfoStatus segInfoStat = new Status.SegmentInfoStatus();
@@ -555,10 +565,19 @@ public class CheckIndex {
 
     if (0 == result.numBadSegments) {
       result.clean = true;
-      msg("No problems were detected with this index.\n");
     } else
       msg("WARNING: " + result.numBadSegments + " broken segments (containing " + result.totLoseDocCount
+ " documents) detected");
 
+    if ( ! (result.validCounter = (result.maxSegmentName < sis.counter))) {
+      result.clean = false;
+      result.newSegments.counter = result.maxSegmentName + 1; 
+      msg("ERROR: Next segment name counter " + sis.counter + " is not greater than max segment
name " + result.maxSegmentName);
+    }
+    
+    if (result.clean) {
+      msg("No problems were detected with this index.\n");
+    }
+
     return result;
   }
 



Mime
View raw message