lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1628688 - in /lucene/dev/branches/lucene5969/lucene: codecs/src/java/org/apache/lucene/codecs/blockterms/ test-framework/src/java/org/apache/lucene/codecs/lucene41ords/ test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/ test-...
Date Wed, 01 Oct 2014 12:27:12 GMT
Author: rmuir
Date: Wed Oct  1 12:27:11 2014
New Revision: 1628688

URL: http://svn.apache.org/r1628688
Log:
LUCENE-5969: clean up unnecessary back compat and add segment header

Modified:
    lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
    lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java
    lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
    lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
    lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java
    lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java
    lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java
    lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java

Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
(original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
Wed Oct  1 12:27:11 2014
@@ -33,15 +33,12 @@ import org.apache.lucene.index.DocsAndPo
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Accountables;
@@ -108,24 +105,19 @@ public class BlockTermsReader extends Fi
     }
   }
   
-  // private String segment;
-  
-  public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos,
SegmentInfo info, PostingsReaderBase postingsReader, IOContext context,
-                          String segmentSuffix)
-    throws IOException {
+  public BlockTermsReader(TermsIndexReaderBase indexReader, PostingsReaderBase postingsReader,
SegmentReadState state) throws IOException {
     
     this.postingsReader = postingsReader;
-
-    // this.segment = segment;
-    in = dir.openInput(IndexFileNames.segmentFileName(info.name, segmentSuffix, BlockTermsWriter.TERMS_EXTENSION),
-                       context);
+    
+    String filename = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix,
BlockTermsWriter.TERMS_EXTENSION);
+    in = state.directory.openInput(filename, state.context);
 
     boolean success = false;
     try {
       CodecUtil.checkSegmentHeader(in, BlockTermsWriter.CODEC_NAME, 
                                        BlockTermsWriter.VERSION_START,
                                        BlockTermsWriter.VERSION_CURRENT,
-                                       info.getId(), segmentSuffix);
+                                       state.segmentInfo.getId(), state.segmentSuffix);
 
       // Have PostingsReader init itself
       postingsReader.init(in);
@@ -148,13 +140,13 @@ public class BlockTermsReader extends Fi
         final long numTerms = in.readVLong();
         assert numTerms >= 0;
         final long termsStartPointer = in.readVLong();
-        final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+        final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
         final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY
? -1 : in.readVLong();
         final long sumDocFreq = in.readVLong();
         final int docCount = in.readVInt();
         final int longsSize = in.readVInt();
-        if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must
be <= #docs
-          throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " +
info.getDocCount(), in);
+        if (docCount < 0 || docCount > state.segmentInfo.getDocCount()) { // #docs
with field must be <= #docs
+          throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " +
state.segmentInfo.getDocCount(), in);
         }
         if (sumDocFreq < docCount) {  // #postings must be >= #docs with field
           throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount:
" + docCount, in);

Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java
(original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java
Wed Oct  1 12:27:11 2014
@@ -59,7 +59,7 @@ import org.apache.lucene.util.RamUsageEs
 
 public class BlockTermsWriter extends FieldsConsumer implements Closeable {
 
-  final static String CODEC_NAME = "BLOCK_TERMS_DICT";
+  final static String CODEC_NAME = "BlockTermsWriter";
 
   // Initial format
   public static final int VERSION_START = 4;
@@ -281,9 +281,6 @@ public class BlockTermsWriter extends Fi
       // EOF marker:
       out.writeVInt(0);
 
-      this.sumTotalTermFreq = sumTotalTermFreq;
-      this.sumDocFreq = sumDocFreq;
-      this.docCount = docCount;
       fieldIndexWriter.finish(out.getFilePointer());
       if (numTerms > 0) {
         fields.add(new FieldMetaData(fieldInfo,

Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
(original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
Wed Oct  1 12:27:11 2014
@@ -24,9 +24,8 @@ import java.util.HashMap;
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.store.Directory;
+import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.Accountable;
@@ -46,27 +45,23 @@ public class VariableGapTermsIndexReader
 
   final HashMap<String,FieldIndexData> fields = new HashMap<>();
   
-  // start of the field info data
-  private long dirOffset;
-  
-  private final int version;
-
-  final String segment;
-  public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment,
String segmentSuffix, IOContext context)
-    throws IOException {
-    final IndexInput in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix,
VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), new IOContext(context, true));
-    this.segment = segment;
+  public VariableGapTermsIndexReader(SegmentReadState state) throws IOException {
+    String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, 
+                                                     state.segmentSuffix, 
+                                                     VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION);
+    final IndexInput in = state.directory.openInput(fileName, new IOContext(state.context,
true));
     boolean success = false;
 
     try {
       
-      version = readHeader(in);
+      CodecUtil.checkSegmentHeader(in, VariableGapTermsIndexWriter.CODEC_NAME,
+                                       VariableGapTermsIndexWriter.VERSION_START,
+                                       VariableGapTermsIndexWriter.VERSION_CURRENT,
+                                       state.segmentInfo.getId(), state.segmentSuffix);
       
-      if (version >= VariableGapTermsIndexWriter.VERSION_CHECKSUM) {
-        CodecUtil.checksumEntireFile(in);
-      }
+      CodecUtil.checksumEntireFile(in);
 
-      seekDir(in, dirOffset);
+      seekDir(in);
 
       // Read directory
       final int numFields = in.readVInt();
@@ -77,7 +72,7 @@ public class VariableGapTermsIndexReader
       for(int i=0;i<numFields;i++) {
         final int field = in.readVInt();
         final long indexStart = in.readVLong();
-        final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+        final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
         FieldIndexData previous = fields.put(fieldInfo.name, new FieldIndexData(in, fieldInfo,
indexStart));
         if (previous != null) {
           throw new CorruptIndexException("duplicate field: " + fieldInfo.name, in);
@@ -92,15 +87,6 @@ public class VariableGapTermsIndexReader
       }
     }
   }
-  
-  private int readHeader(IndexInput input) throws IOException {
-    int version = CodecUtil.checkHeader(input, VariableGapTermsIndexWriter.CODEC_NAME,
-      VariableGapTermsIndexWriter.VERSION_START, VariableGapTermsIndexWriter.VERSION_CURRENT);
-    if (version < VariableGapTermsIndexWriter.VERSION_APPEND_ONLY) {
-      dirOffset = input.readLong();
-    }
-    return version;
-  }
 
   private static class IndexEnum extends FieldIndexEnum {
     private final BytesRefFSTEnum<Long> fstEnum;
@@ -206,14 +192,9 @@ public class VariableGapTermsIndexReader
   @Override
   public void close() throws IOException {}
 
-  private void seekDir(IndexInput input, long dirOffset) throws IOException {
-    if (version >= VariableGapTermsIndexWriter.VERSION_CHECKSUM) {
-      input.seek(input.length() - CodecUtil.footerLength() - 8);
-      dirOffset = input.readLong();
-    } else if (version >= VariableGapTermsIndexWriter.VERSION_APPEND_ONLY) {
-      input.seek(input.length() - 8);
-      dirOffset = input.readLong();
-    }
+  private void seekDir(IndexInput input) throws IOException {
+    input.seek(input.length() - CodecUtil.footerLength() - 8);
+    long dirOffset = input.readLong();
     input.seek(dirOffset);
   }
 

Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
(original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
Wed Oct  1 12:27:11 2014
@@ -52,11 +52,9 @@ public class VariableGapTermsIndexWriter
   /** Extension of terms index file */
   static final String TERMS_INDEX_EXTENSION = "tiv";
 
-  final static String CODEC_NAME = "VARIABLE_GAP_TERMS_INDEX";
-  final static int VERSION_START = 0;
-  final static int VERSION_APPEND_ONLY = 1;
-  final static int VERSION_CHECKSUM = 2;
-  final static int VERSION_CURRENT = VERSION_CHECKSUM;
+  final static String CODEC_NAME = "VariableGapTermsIndex";
+  final static int VERSION_START = 3;
+  final static int VERSION_CURRENT = VERSION_START;
 
   private final List<FSTFieldWriter> fields = new ArrayList<>();
   
@@ -184,7 +182,7 @@ public class VariableGapTermsIndexWriter
     try {
       fieldInfos = state.fieldInfos;
       this.policy = policy;
-      writeHeader(out);
+      CodecUtil.writeSegmentHeader(out, CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(),
state.segmentSuffix);
       success = true;
     } finally {
       if (!success) {
@@ -192,10 +190,6 @@ public class VariableGapTermsIndexWriter
       }
     }
   }
-  
-  private void writeHeader(IndexOutput out) throws IOException {
-    CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
-  }
 
   @Override
   public FieldWriter addField(FieldInfo field, long termsFilePointer) throws IOException
{

Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java
(original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java
Wed Oct  1 12:27:11 2014
@@ -35,7 +35,6 @@ import org.apache.lucene.codecs.lucene41
 import org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.BytesRef;
 
 // TODO: we could make separate base class that can wrapp
 // any PostingsBaseFormat and make it ord-able...
@@ -110,13 +109,7 @@ public final class Lucene41WithOrds exte
 
     success = false;
     try {
-      FieldsProducer ret = new BlockTermsReader(indexReader,
-                                                state.directory,
-                                                state.fieldInfos,
-                                                state.segmentInfo,
-                                                postings,
-                                                state.context,
-                                                state.segmentSuffix);
+      FieldsProducer ret = new BlockTermsReader(indexReader, postings, state);
       success = true;
       return ret;
     } finally {

Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java
(original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java
Wed Oct  1 12:27:11 2014
@@ -103,10 +103,7 @@ public final class Lucene41VarGapDocFreq
 
     boolean success = false;
     try {
-      indexReader = new VariableGapTermsIndexReader(state.directory,
-                                                 state.fieldInfos,
-                                                 state.segmentInfo.name,
-                                                 state.segmentSuffix, state.context);
+      indexReader = new VariableGapTermsIndexReader(state);
       success = true;
     } finally {
       if (!success) {
@@ -116,13 +113,7 @@ public final class Lucene41VarGapDocFreq
 
     success = false;
     try {
-      FieldsProducer ret = new BlockTermsReader(indexReader,
-                                                state.directory,
-                                                state.fieldInfos,
-                                                state.segmentInfo,
-                                                postings,
-                                                state.context,
-                                                state.segmentSuffix);
+      FieldsProducer ret = new BlockTermsReader(indexReader, postings, state);
       success = true;
       return ret;
     } finally {

Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java
(original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java
Wed Oct  1 12:27:11 2014
@@ -100,10 +100,7 @@ public final class Lucene41VarGapFixedIn
 
     boolean success = false;
     try {
-      indexReader = new VariableGapTermsIndexReader(state.directory,
-                                                 state.fieldInfos,
-                                                 state.segmentInfo.name,
-                                                 state.segmentSuffix, state.context);
+      indexReader = new VariableGapTermsIndexReader(state);
       success = true;
     } finally {
       if (!success) {
@@ -113,13 +110,7 @@ public final class Lucene41VarGapFixedIn
 
     success = false;
     try {
-      FieldsProducer ret = new BlockTermsReader(indexReader,
-                                                state.directory,
-                                                state.fieldInfos,
-                                                state.segmentInfo,
-                                                postings,
-                                                state.context,
-                                                state.segmentSuffix);
+      FieldsProducer ret = new BlockTermsReader(indexReader, postings, state);
       success = true;
       return ret;
     } finally {

Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
(original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
Wed Oct  1 12:27:11 2014
@@ -351,10 +351,7 @@ public final class MockRandomPostingsFor
           if (LuceneTestCase.VERBOSE) {
             System.out.println("MockRandomCodec: variable-gap terms index");
           }
-          indexReader = new VariableGapTermsIndexReader(state.directory,
-                                                        state.fieldInfos,
-                                                        state.segmentInfo.name,
-                                                        state.segmentSuffix, state.context);
+          indexReader = new VariableGapTermsIndexReader(state);
 
         }
 
@@ -367,13 +364,7 @@ public final class MockRandomPostingsFor
 
       success = false;
       try {
-        fields = new BlockTermsReader(indexReader,
-                                      state.directory,
-                                      state.fieldInfos,
-                                      state.segmentInfo,
-                                      postingsReader,
-                                      state.context,
-                                      state.segmentSuffix);
+        fields = new BlockTermsReader(indexReader, postingsReader, state);
         success = true;
       } finally {
         if (!success) {



Mime
View raw message