lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1586712 - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/compressing/ lucene/core/src/java/org/apache/lucene/store/
Date Fri, 11 Apr 2014 16:27:26 GMT
Author: rmuir
Date: Fri Apr 11 16:27:26 2014
New Revision: 1586712

URL: http://svn.apache.org/r1586712
Log:
LUCENE-5602: always verify term vectors on bulk merge

Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/CHANGES.txt
    lucene/dev/branches/branch_4x/lucene/core/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java

Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1586712&r1=1586711&r2=1586712&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Fri Apr 11 16:27:26 2014
@@ -7,7 +7,8 @@ http://s.apache.org/luceneversions
 
 New Features
 
-* placeolder
+* LUCENE-5602: Checksums are automatically verified on the default term
+  vectors format when performing a bulk merge. (Adrien Grand, Robert Muir)
 
 ======================= Lucene 4.8.0 =======================
 

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java?rev=1586712&r1=1586711&r2=1586712&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
Fri Apr 11 16:27:26 2014
@@ -158,6 +158,10 @@ public final class CompressingTermVector
   int getPackedIntsVersion() {
     return packedIntsVersion;
   }
+  
+  int getVersion() {
+    return version;
+  }
 
   CompressingStoredFieldsIndexReader getIndex() {
     return indexReader;

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java?rev=1586712&r1=1586711&r2=1586712&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java
Fri Apr 11 16:27:26 2014
@@ -37,6 +37,8 @@ import org.apache.lucene.index.IndexFile
 import org.apache.lucene.index.MergeState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.store.BufferedChecksumIndexInput;
+import org.apache.lucene.store.ChecksumIndexInput;
 import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
@@ -749,6 +751,7 @@ public final class CompressingTermVector
       final Bits liveDocs = reader.getLiveDocs();
 
       if (matchingVectorsReader == null
+          || matchingVectorsReader.getVersion() != VERSION_CURRENT
           || matchingVectorsReader.getCompressionMode() != compressionMode
           || matchingVectorsReader.getChunkSize() != chunkSize
           || matchingVectorsReader.getPackedIntsVersion() != PackedInts.VERSION_CURRENT)
{
@@ -761,12 +764,19 @@ public final class CompressingTermVector
         }
       } else {
         final CompressingStoredFieldsIndexReader index = matchingVectorsReader.getIndex();
-        final IndexInput vectorsStream = matchingVectorsReader.getVectorsStream();
+        final IndexInput vectorsStreamOrig = matchingVectorsReader.getVectorsStream();
+        vectorsStreamOrig.seek(0);
+        final ChecksumIndexInput vectorsStream = new BufferedChecksumIndexInput(vectorsStreamOrig.clone());
+        
         for (int i = nextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; ) {
-          if (pendingDocs.isEmpty()
-              && (i == 0 || index.getStartPointer(i - 1) < index.getStartPointer(i)))
{ // start of a chunk
-            final long startPointer = index.getStartPointer(i);
+          // We make sure to move the checksum input in any case, otherwise the final
+          // integrity check might need to read the whole file a second time
+          final long startPointer = index.getStartPointer(i);
+          if (startPointer > vectorsStream.getFilePointer()) {
             vectorsStream.seek(startPointer);
+          }
+          if (pendingDocs.isEmpty()
+              && (i == 0 || index.getStartPointer(i - 1) < startPointer)) { //
start of a chunk
             final int docBase = vectorsStream.readVInt();
             final int chunkDocs = vectorsStream.readVInt();
             assert docBase + chunkDocs <= matchingSegmentReader.maxDoc();
@@ -798,6 +808,9 @@ public final class CompressingTermVector
             i = nextLiveDoc(i + 1, liveDocs, maxDoc);
           }
         }
+        
+        vectorsStream.seek(vectorsStream.length() - CodecUtil.footerLength());
+        CodecUtil.checkFooter(vectorsStream);
       }
     }
     finish(mergeState.fieldInfos, docCount);

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java?rev=1586712&r1=1586711&r2=1586712&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java
Fri Apr 11 16:27:26 2014
@@ -46,7 +46,7 @@ public abstract class ChecksumIndexInput
   public void seek(long pos) throws IOException {
     final long skip = pos - getFilePointer();
     if (skip < 0) {
-      throw new IllegalStateException(ChecksumIndexInput.class + " cannot seed backward");
+      throw new IllegalStateException(getClass() + " cannot seek backwards");
     }
     skipBytes(skip);
   }



Mime
View raw message