Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@lucene.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1664633 - in /lucene/dev/trunk/lucene: CHANGES.txt
 core/src/java/org/apache/lucene/index/CheckIndex.java
 core/src/test/org/apache/lucene/index/TestCheckIndex.java
Date: Fri, 06 Mar 2015 14:37:44 -0000
To: commits@lucene.apache.org
From: rmuir@apache.org
Message-Id: <20150306143745.597DAAC0237@hades.apache.org>

Author: rmuir
Date: Fri Mar  6 14:37:44 2015
New Revision: 1664633

URL: http://svn.apache.org/r1664633
Log:
LUCENE-6341: Add a -fast option to CheckIndex

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1664633&r1=1664632&r2=1664633&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Fri Mar  6 14:37:44 2015
@@ -19,6 +19,8 @@ New Features
   for counting ranges that align with the underlying terms as defined by the
   NumberRangePrefixTree (e.g. familiar date units like days).  (David Smiley)
 
+* LUCENE-6341: Add a -fast option to CheckIndex. (Robert Muir)
+
 API Changes
 
 * LUCENE-3312: The API of oal.document was restructured to

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1664633&r1=1664632&r2=1664633&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Fri Mar  6 14:37:44 2015
@@ -411,6 +411,20 @@ public class CheckIndex implements Close
   }
 
   private boolean verbose;
+  
+  /** See {@link #getChecksumsOnly}. */
+  public boolean getChecksumsOnly() {
+    return checksumsOnly;
+  }
+  
+  /** 
+   * If true, only validate physical integrity for all files. 
+   * Note that the returned nested status objects (e.g. storedFieldStatus) will be null.  */
+  public void setChecksumsOnly(boolean v) {
+    checksumsOnly = v;
+  }
+  
+  private boolean checksumsOnly;
 
   /** Set infoStream where messages should go.  If null, no
    *  messages are printed.  If verbose is true then more
@@ -668,42 +682,45 @@ public class CheckIndex implements Close
           }
         }
         
-        // Test Livedocs
-        segInfoStat.liveDocStatus = testLiveDocs(reader, infoStream, failFast);
-
-        // Test Fieldinfos
-        segInfoStat.fieldInfoStatus = testFieldInfos(reader, infoStream, failFast);
-        
-        // Test Field Norms
-        segInfoStat.fieldNormStatus = testFieldNorms(reader, infoStream, failFast);
-
-        // Test the Term Index
-        segInfoStat.termIndexStatus = testPostings(reader, infoStream, verbose, failFast);
-
-        // Test Stored Fields
-        segInfoStat.storedFieldStatus = testStoredFields(reader, infoStream, failFast);
-
-        // Test Term Vectors
-        segInfoStat.termVectorStatus = testTermVectors(reader, infoStream, verbose, crossCheckTermVectors, failFast);
-
-        segInfoStat.docValuesStatus = testDocValues(reader, infoStream, failFast);
-
-        // Rethrow the first exception we encountered
-        //  This will cause stats for failed segments to be incremented properly
-        if (segInfoStat.liveDocStatus.error != null) {
-          throw new RuntimeException("Live docs test failed");
-        } else if (segInfoStat.fieldInfoStatus.error != null) {
-          throw new RuntimeException("Field Info test failed");
-        } else if (segInfoStat.fieldNormStatus.error != null) {
-          throw new RuntimeException("Field Norm test failed");
-        } else if (segInfoStat.termIndexStatus.error != null) {
-          throw new RuntimeException("Term Index test failed");
-        } else if (segInfoStat.storedFieldStatus.error != null) {
-          throw new RuntimeException("Stored Field test failed");
-        } else if (segInfoStat.termVectorStatus.error != null) {
-          throw new RuntimeException("Term Vector test failed");
-        }  else if (segInfoStat.docValuesStatus.error != null) {
-          throw new RuntimeException("DocValues test failed");
+        if (checksumsOnly == false) {
+          // Test Livedocs
+          segInfoStat.liveDocStatus = testLiveDocs(reader, infoStream, failFast);
+
+          // Test Fieldinfos
+          segInfoStat.fieldInfoStatus = testFieldInfos(reader, infoStream, failFast);
+        
+          // Test Field Norms
+          segInfoStat.fieldNormStatus = testFieldNorms(reader, infoStream, failFast);
+
+          // Test the Term Index
+          segInfoStat.termIndexStatus = testPostings(reader, infoStream, verbose, failFast);
+
+          // Test Stored Fields
+          segInfoStat.storedFieldStatus = testStoredFields(reader, infoStream, failFast);
+
+          // Test Term Vectors
+          segInfoStat.termVectorStatus = testTermVectors(reader, infoStream, verbose, crossCheckTermVectors, failFast);
+
+          // Test Docvalues
+          segInfoStat.docValuesStatus = testDocValues(reader, infoStream, failFast);
+
+          // Rethrow the first exception we encountered
+          //  This will cause stats for failed segments to be incremented properly
+          if (segInfoStat.liveDocStatus.error != null) {
+            throw new RuntimeException("Live docs test failed");
+          } else if (segInfoStat.fieldInfoStatus.error != null) {
+            throw new RuntimeException("Field Info test failed");
+          } else if (segInfoStat.fieldNormStatus.error != null) {
+            throw new RuntimeException("Field Norm test failed");
+          } else if (segInfoStat.termIndexStatus.error != null) {
+            throw new RuntimeException("Term Index test failed");
+          } else if (segInfoStat.storedFieldStatus.error != null) {
+            throw new RuntimeException("Stored Field test failed");
+          } else if (segInfoStat.termVectorStatus.error != null) {
+            throw new RuntimeException("Term Vector test failed");
+          }  else if (segInfoStat.docValuesStatus.error != null) {
+            throw new RuntimeException("DocValues test failed");
+          }
         }
 
         msg(infoStream, "");
@@ -2110,13 +2127,16 @@ public class CheckIndex implements Close
     boolean doExorcise = false;
     boolean doCrossCheckTermVectors = false;
     boolean verbose = false;
+    boolean doChecksumsOnly = false;
     List<String> onlySegments = new ArrayList<>();
     String indexPath = null;
     String dirImpl = null;
     int i = 0;
     while(i < args.length) {
       String arg = args[i];
-      if ("-exorcise".equals(arg)) {
+      if ("-fast".equals(arg)) {
+        doChecksumsOnly = true;
+      } else if ("-exorcise".equals(arg)) {
         doExorcise = true;
       } else if ("-crossCheckTermVectors".equals(arg)) {
         doCrossCheckTermVectors = true;
@@ -2151,6 +2171,7 @@ public class CheckIndex implements Close
       System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-exorcise] [-crossCheckTermVectors] [-segment X] [-segment Y] [-dir-impl X]\n" +
                          "\n" +
                          "  -exorcise: actually write a new segments_N file, removing any problematic segments\n" +
+                         "  -fast: just verify file checksums, omitting logical integrity checks\n" + 
                          "  -crossCheckTermVectors: verifies that term vectors match postings; THIS IS VERY SLOW!\n" +
                          "  -codec X: when exorcising, codec to write the new segments_N file with\n" +
                          "  -verbose: print additional details\n" +
@@ -2185,6 +2206,11 @@ public class CheckIndex implements Close
       System.out.println("ERROR: cannot specify both -exorcise and -segment");
       return 1;
     }
+    
+    if (doChecksumsOnly && doCrossCheckTermVectors) {
+      System.out.println("ERROR: cannot specify both -fast and -crossCheckTermVectors");
+      return 1;
+    }
 
     System.out.println("\nOpening index @ " + indexPath + "\n");
     Directory directory = null;
@@ -2204,6 +2230,7 @@ public class CheckIndex implements Close
     try (Directory dir = directory;
          CheckIndex checker = new CheckIndex(dir)) {
       checker.setCrossCheckTermVectors(doCrossCheckTermVectors);
+      checker.setChecksumsOnly(doChecksumsOnly);
       checker.setInfoStream(System.out, verbose);
       
       Status result = checker.checkIndex(onlySegments);

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java?rev=1664633&r1=1664632&r2=1664633&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java Fri Mar  6 14:37:44 2015
@@ -24,9 +24,11 @@ import java.util.List;
 import java.util.ArrayList;
 
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LineFileDocs;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.CannedTokenStream;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.Token;
@@ -118,6 +120,54 @@ public class TestCheckIndex extends Luce
     dir.close(); // checkindex
   }
   
+  public void testChecksumsOnly() throws IOException {
+    LineFileDocs lf = new LineFileDocs(random());
+    Directory dir = newDirectory();
+    Analyzer analyzer = new MockAnalyzer(random());
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(analyzer));
+    for (int i = 0; i < 100; i++) {
+      iw.addDocument(lf.nextDoc());
+    }
+    iw.addDocument(new Document());
+    iw.commit();
+    iw.close();
+    lf.close();
+    
+    ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
+    CheckIndex checker = new CheckIndex(dir);
+    checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8));
+    if (VERBOSE) checker.setInfoStream(System.out);
+    CheckIndex.Status indexStatus = checker.checkIndex();
+    assertTrue(indexStatus.clean);
+    checker.close();
+    dir.close();
+    analyzer.close();
+  }
+  
+  public void testChecksumsOnlyVerbose() throws IOException {
+    LineFileDocs lf = new LineFileDocs(random());
+    Directory dir = newDirectory();
+    Analyzer analyzer = new MockAnalyzer(random());
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(analyzer));
+    for (int i = 0; i < 100; i++) {
+      iw.addDocument(lf.nextDoc());
+    }
+    iw.addDocument(new Document());
+    iw.commit();
+    iw.close();
+    lf.close();
+    
+    ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
+    CheckIndex checker = new CheckIndex(dir);
+    checker.setInfoStream(new PrintStream(bos, true, IOUtils.UTF_8));
+    if (VERBOSE) checker.setInfoStream(System.out);
+    CheckIndex.Status indexStatus = checker.checkIndex();
+    assertTrue(indexStatus.clean);
+    checker.close();
+    dir.close();
+    analyzer.close();
+  }
+  
   public void testObtainsLock() throws IOException {
     Directory dir = newDirectory();
     IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));