lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1344039 - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/lucene3x/ lucene/core/src/java/org/apache/lucene/codecs/lucene40/ lucene/core/src/java/org/apache/lucene/codecs/simpletext/ ...
Date Tue, 29 May 2012 22:45:32 GMT
Author: rmuir
Date: Tue May 29 22:45:32 2012
New Revision: 1344039

URL: http://svn.apache.org/viewvc?rev=1344039&view=rev
Log:
LUCENE-4088: unindexed (stored-only) fields prevent future indexing of offsets

Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
    lucene/dev/branches/branch_4x/lucene/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosReader.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosWriter.java

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java?rev=1344039&r1=1344038&r2=1344039&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java
Tue May 29 22:45:32 2012
@@ -82,7 +82,9 @@ class Lucene3xFieldInfosReader extends F
         boolean omitNorms = (bits & OMIT_NORMS) != 0;
         boolean storePayloads = (bits & STORE_PAYLOADS) != 0;
         final IndexOptions indexOptions;
-        if ((bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0) {
+        if (!isIndexed) {
+          indexOptions = null;
+        } else if ((bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0) {
           indexOptions = IndexOptions.DOCS_ONLY;
         } else if ((bits & OMIT_POSITIONS) != 0) {
           if (format <= FORMAT_OMIT_POSITIONS) {

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java?rev=1344039&r1=1344038&r2=1344039&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java
Tue May 29 22:45:32 2012
@@ -63,7 +63,9 @@ public class Lucene40FieldInfosReader ex
         boolean omitNorms = (bits & Lucene40FieldInfosWriter.OMIT_NORMS) != 0;
         boolean storePayloads = (bits & Lucene40FieldInfosWriter.STORE_PAYLOADS) != 0;
         final IndexOptions indexOptions;
-        if ((bits & Lucene40FieldInfosWriter.OMIT_TERM_FREQ_AND_POSITIONS) != 0) {
+        if (!isIndexed) {
+          indexOptions = null;
+        } else if ((bits & Lucene40FieldInfosWriter.OMIT_TERM_FREQ_AND_POSITIONS) !=
0) {
           indexOptions = IndexOptions.DOCS_ONLY;
         } else if ((bits & Lucene40FieldInfosWriter.OMIT_POSITIONS) != 0) {
           indexOptions = IndexOptions.DOCS_AND_FREQS;
@@ -76,7 +78,7 @@ public class Lucene40FieldInfosReader ex
         // LUCENE-3027: past indices were able to write
         // storePayloads=true when omitTFAP is also true,
         // which is invalid.  We correct that, here:
-        if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
+        if (isIndexed && indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
< 0) {
           storePayloads = false;
         }
         // DV Types are packed in one byte

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java?rev=1344039&r1=1344038&r2=1344039&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java
Tue May 29 22:45:32 2012
@@ -61,18 +61,20 @@ public class Lucene40FieldInfosWriter ex
       output.writeVInt(infos.size());
       for (FieldInfo fi : infos) {
         IndexOptions indexOptions = fi.getIndexOptions();
-        assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0
|| !fi.hasPayloads();
         byte bits = 0x0;
-        if (fi.isIndexed()) bits |= IS_INDEXED;
         if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
         if (fi.omitsNorms()) bits |= OMIT_NORMS;
         if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
-        if (indexOptions == IndexOptions.DOCS_ONLY) {
-          bits |= OMIT_TERM_FREQ_AND_POSITIONS;
-        } else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
{
-          bits |= STORE_OFFSETS_IN_POSTINGS;
-        } else if (indexOptions == IndexOptions.DOCS_AND_FREQS) {
-          bits |= OMIT_POSITIONS;
+        if (fi.isIndexed()) {
+          bits |= IS_INDEXED;
+          assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >=
0 || !fi.hasPayloads();
+          if (indexOptions == IndexOptions.DOCS_ONLY) {
+            bits |= OMIT_TERM_FREQ_AND_POSITIONS;
+          } else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
{
+            bits |= STORE_OFFSETS_IN_POSTINGS;
+          } else if (indexOptions == IndexOptions.DOCS_AND_FREQS) {
+            bits |= OMIT_POSITIONS;
+          }
         }
         output.writeString(fi.name);
         output.writeVInt(fi.number);

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java?rev=1344039&r1=1344038&r2=1344039&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
Tue May 29 22:45:32 2012
@@ -72,6 +72,15 @@ public class SimpleTextFieldInfosReader 
         assert StringHelper.startsWith(scratch, ISINDEXED);
         boolean isIndexed = Boolean.parseBoolean(readString(ISINDEXED.length, scratch));
         
+        final IndexOptions indexOptions;
+        if (isIndexed) {
+          SimpleTextUtil.readLine(input, scratch);
+          assert StringHelper.startsWith(scratch, INDEXOPTIONS);
+          indexOptions = IndexOptions.valueOf(readString(INDEXOPTIONS.length, scratch));
         
+        } else {
+          indexOptions = null;
+        }
+        
         SimpleTextUtil.readLine(input, scratch);
         assert StringHelper.startsWith(scratch, STORETV);
         boolean storeTermVector = Boolean.parseBoolean(readString(STORETV.length, scratch));
@@ -95,10 +104,6 @@ public class SimpleTextFieldInfosReader 
         final DocValues.Type docValuesType = docValuesType(dvType);
         
         SimpleTextUtil.readLine(input, scratch);
-        assert StringHelper.startsWith(scratch, INDEXOPTIONS);
-        IndexOptions indexOptions = IndexOptions.valueOf(readString(INDEXOPTIONS.length,
scratch));
-      
-        SimpleTextUtil.readLine(input, scratch);
         assert StringHelper.startsWith(scratch, NUM_ATTS);
         int numAtts = Integer.parseInt(readString(NUM_ATTS.length, scratch));
         Map<String,String> atts = new HashMap<String,String>();

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java?rev=1344039&r1=1344038&r2=1344039&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
Tue May 29 22:45:32 2012
@@ -68,8 +68,6 @@ public class SimpleTextFieldInfosWriter 
       SimpleTextUtil.writeNewline(out);
       
       for (FieldInfo fi : infos) {
-        assert fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
>= 0 || !fi.hasPayloads();
-
         SimpleTextUtil.write(out, NAME);
         SimpleTextUtil.write(out, fi.name, scratch);
         SimpleTextUtil.writeNewline(out);
@@ -82,6 +80,13 @@ public class SimpleTextFieldInfosWriter 
         SimpleTextUtil.write(out, Boolean.toString(fi.isIndexed()), scratch);
         SimpleTextUtil.writeNewline(out);
         
+        if (fi.isIndexed()) {
+          assert fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
>= 0 || !fi.hasPayloads();
+          SimpleTextUtil.write(out, INDEXOPTIONS);
+          SimpleTextUtil.write(out, fi.getIndexOptions().toString(), scratch);
+          SimpleTextUtil.writeNewline(out);
+        }
+        
         SimpleTextUtil.write(out, STORETV);
         SimpleTextUtil.write(out, Boolean.toString(fi.hasVectors()), scratch);
         SimpleTextUtil.writeNewline(out);
@@ -101,11 +106,7 @@ public class SimpleTextFieldInfosWriter 
         SimpleTextUtil.write(out, DOCVALUES);
         SimpleTextUtil.write(out, getDocValuesType(fi.getDocValuesType()), scratch);
         SimpleTextUtil.writeNewline(out);
-        
-        SimpleTextUtil.write(out, INDEXOPTIONS);
-        SimpleTextUtil.write(out, fi.getIndexOptions().toString(), scratch);
-        SimpleTextUtil.writeNewline(out);
-        
+               
         Map<String,String> atts = fi.attributes();
         int numAtts = atts == null ? 0 : atts.size();
         SimpleTextUtil.write(out, NUM_ATTS);

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1344039&r1=1344038&r2=1344039&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
Tue May 29 22:45:32 2012
@@ -85,12 +85,11 @@ public final class FieldInfo {
       this.storeTermVector = false;
       this.storePayloads = false;
       this.omitNorms = false;
-      this.indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+      this.indexOptions = null;
       this.normType = null;
     }
     this.attributes = attributes;
     assert checkConsistency();
-    assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !storePayloads;
   }
 
   private boolean checkConsistency() {
@@ -99,17 +98,16 @@ public final class FieldInfo {
       assert !storePayloads;
       assert !omitNorms;
       assert normType == null;
-      assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+      assert indexOptions == null;
     } else {
       assert indexOptions != null;
       if (omitNorms) {
         assert normType == null;
       }
+      // Cannot store payloads unless positions are indexed:
+      assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 ||
!this.storePayloads;
     }
 
-    // Cannot store payloads unless positions are indexed:
-    assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !this.storePayloads;
-
     return true;
   }
 
@@ -131,15 +129,18 @@ public final class FieldInfo {
         this.normType = null;
       }
       if (this.indexOptions != indexOptions) {
-        // downgrade
-        this.indexOptions = this.indexOptions.compareTo(indexOptions) < 0 ? this.indexOptions
: indexOptions;
+        if (this.indexOptions == null) {
+          this.indexOptions = indexOptions;
+        } else {
+          // downgrade
+          this.indexOptions = this.indexOptions.compareTo(indexOptions) < 0 ? this.indexOptions
: indexOptions;
+        }
         if (this.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0)
{
           // cannot store payloads if we don't store positions:
           this.storePayloads = false;
         }
       }
     }
-    assert this.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0
|| !this.storePayloads;
     assert checkConsistency();
   }
 
@@ -148,7 +149,7 @@ public final class FieldInfo {
     assert checkConsistency();
   }
   
-  /** @return IndexOptions for the field */
+  /** @return IndexOptions for the field, or null if the field is not indexed */
   public IndexOptions getIndexOptions() {
     return indexOptions;
   }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1344039&r1=1344038&r2=1344039&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
Tue May 29 22:45:32 2012
@@ -42,10 +42,9 @@ final class FreqProxTermsWriterPerField 
   final FieldInfo fieldInfo;
   final DocumentsWriterPerThread.DocState docState;
   final FieldInvertState fieldState;
-  IndexOptions indexOptions;
-  private boolean writeFreq;
-  private boolean writeProx;
-  private boolean writeOffsets;
+  private boolean hasFreq;
+  private boolean hasProx;
+  private boolean hasOffsets;
   PayloadAttribute payloadAttribute;
   OffsetAttribute offsetAttribute;
 
@@ -60,7 +59,7 @@ final class FreqProxTermsWriterPerField 
 
   @Override
   int getStreamCount() {
-    if (!writeProx) {
+    if (!hasProx) {
       return 1;
     } else {
       return 2;
@@ -92,10 +91,14 @@ final class FreqProxTermsWriterPerField 
   }
 
   private void setIndexOptions(IndexOptions indexOptions) {
-    this.indexOptions = indexOptions;
-    writeFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
-    writeProx = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
-    writeOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
>= 0;
+    if (indexOptions == null) {
+      // field could later be updated with indexed=true, so set everything on
+      hasFreq = hasProx = hasOffsets = true;
+    } else {
+      hasFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+      hasProx = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+      hasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
>= 0;
+    }
   }
 
   @Override
@@ -115,7 +118,7 @@ final class FreqProxTermsWriterPerField 
     } else {
       payloadAttribute = null;
     }
-    if (writeOffsets) {
+    if (hasOffsets) {
       offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class);
     } else {
       offsetAttribute = null;
@@ -124,7 +127,7 @@ final class FreqProxTermsWriterPerField 
 
   void writeProx(final int termID, int proxCode) {
     //System.out.println("writeProx termID=" + termID + " proxCode=" + proxCode);
-    assert writeProx;
+    assert hasProx;
     final Payload payload;
     if (payloadAttribute == null) {
       payload = null;
@@ -146,7 +149,7 @@ final class FreqProxTermsWriterPerField 
   }
 
   void writeOffsets(final int termID, int prevOffset) {
-    assert writeOffsets;
+    assert hasOffsets;
     final int startOffset = offsetAttribute.startOffset();
     final int endOffset = offsetAttribute.endOffset();
     //System.out.println("writeOffsets termID=" + termID + " prevOffset=" + prevOffset +
" startOff=" + startOffset + " endOff=" + endOffset);
@@ -165,18 +168,18 @@ final class FreqProxTermsWriterPerField 
 
     FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
     postings.lastDocIDs[termID] = docState.docID;
-    if (!writeFreq) {
+    if (!hasFreq) {
       postings.lastDocCodes[termID] = docState.docID;
     } else {
       postings.lastDocCodes[termID] = docState.docID << 1;
       postings.docFreqs[termID] = 1;
-      if (writeProx) {
+      if (hasProx) {
         writeProx(termID, fieldState.position);
-        if (writeOffsets) {
+        if (hasOffsets) {
           writeOffsets(termID, fieldState.offset);
         }
       } else {
-        assert !writeOffsets;
+        assert !hasOffsets;
       }
     }
     fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency);
@@ -190,9 +193,9 @@ final class FreqProxTermsWriterPerField 
 
     FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
 
-    assert !writeFreq || postings.docFreqs[termID] > 0;
+    assert !hasFreq || postings.docFreqs[termID] > 0;
 
-    if (!writeFreq) {
+    if (!hasFreq) {
       assert postings.docFreqs == null;
       if (docState.docID != postings.lastDocIDs[termID]) {
         assert docState.docID > postings.lastDocIDs[termID];
@@ -218,21 +221,21 @@ final class FreqProxTermsWriterPerField 
       fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency);
       postings.lastDocCodes[termID] = (docState.docID - postings.lastDocIDs[termID]) <<
1;
       postings.lastDocIDs[termID] = docState.docID;
-      if (writeProx) {
+      if (hasProx) {
         writeProx(termID, fieldState.position);
-        if (writeOffsets) {
+        if (hasOffsets) {
           writeOffsets(termID, fieldState.offset);
         }
       } else {
-        assert !writeOffsets;
+        assert !hasOffsets;
       }
       fieldState.uniqueTermCount++;
     } else {
       fieldState.maxTermFrequency = Math.max(fieldState.maxTermFrequency, ++postings.docFreqs[termID]);
-      if (writeProx) {
+      if (hasProx) {
         writeProx(termID, fieldState.position-postings.lastPositions[termID]);
       }
-      if (writeOffsets) {
+      if (hasOffsets) {
         writeOffsets(termID, postings.lastOffsets[termID]);
       }
     }
@@ -240,7 +243,7 @@ final class FreqProxTermsWriterPerField 
 
   @Override
   ParallelPostingsArray createPostingsArray(int size) {
-    return new FreqProxPostingsArray(size, writeFreq, writeProx, writeOffsets);
+    return new FreqProxPostingsArray(size, hasFreq, hasProx, hasOffsets);
   }
 
   static final class FreqProxPostingsArray extends ParallelPostingsArray {
@@ -323,6 +326,10 @@ final class FreqProxTermsWriterPerField 
   void flush(String fieldName, FieldsConsumer consumer,  final SegmentWriteState state)
     throws CorruptIndexException, IOException {
 
+    if (!fieldInfo.isIndexed()) {
+      return; // nothing to flush, don't bother the codec with the unindexed field
+    }
+    
     final TermsConsumer termsConsumer = consumer.addField(fieldInfo);
     final Comparator<BytesRef> termComp = termsConsumer.getComparator();
 
@@ -335,14 +342,15 @@ final class FreqProxTermsWriterPerField 
     // new segment to the directory according to
     // currentFieldIndexOptions:
     final IndexOptions currentFieldIndexOptions = fieldInfo.getIndexOptions();
+    assert currentFieldIndexOptions != null;
 
     final boolean writeTermFreq = currentFieldIndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS)
>= 0;
     final boolean writePositions = currentFieldIndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
>= 0;
     final boolean writeOffsets = currentFieldIndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
>= 0;
 
-    final boolean readTermFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >=
0;
-    final boolean readPositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
>= 0;
-    final boolean readOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
>= 0;
+    final boolean readTermFreq = this.hasFreq;
+    final boolean readPositions = this.hasProx;
+    final boolean readOffsets = this.hasOffsets;
 
     //System.out.println("flush readTF=" + readTermFreq + " readPos=" + readPositions + "
readOffs=" + readOffsets);
 
@@ -517,7 +525,11 @@ final class FreqProxTermsWriterPerField 
                 final int endOffset = startOffset + prox.readVInt();
                 offset = startOffset;
                 if (writePositions) {
-                  postingsConsumer.addPosition(position, thisPayload, startOffset, endOffset);
+                  if (writeOffsets) {
+                    postingsConsumer.addPosition(position, thisPayload, startOffset, endOffset);
+                  } else {
+                    postingsConsumer.addPosition(position, thisPayload, -1, -1);
+                  }
                 }
               } else if (writePositions) {
                 postingsConsumer.addPosition(position, thisPayload, -1, -1);

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java?rev=1344039&r1=1344038&r2=1344039&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
Tue May 29 22:45:32 2012
@@ -33,6 +33,7 @@ import org.apache.lucene.document.FieldT
 import org.apache.lucene.document.IntField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.store.Directory;
@@ -346,6 +347,41 @@ public class TestPostingsOffsets extends
     r.close();
     dir.close();
   }
+  
+  public void testWithUnindexedFields() throws Exception {
+    Directory dir = newDirectory();
+    RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);
+    for (int i = 0; i < 100; i++) {
+      Document doc = new Document();
+      // ensure at least one doc is indexed with offsets
+      if (i < 99 && random().nextInt(2) == 0) {
+        // stored only
+        FieldType ft = new FieldType();
+        ft.setIndexed(false);
+        ft.setStored(true);
+        doc.add(new Field("foo", "boo!", ft));
+      } else {
+        FieldType ft = new FieldType(TextField.TYPE_STORED);
+        ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+        if (random().nextBoolean()) {
+          // store some term vectors for the checkindex cross-check
+          ft.setStoreTermVectors(true);
+          ft.setStoreTermVectorPositions(true);
+          ft.setStoreTermVectorOffsets(true);
+        }
+        doc.add(new Field("foo", "bar", ft));
+      }
+      riw.addDocument(doc);
+    }
+    CompositeReader ir = riw.getReader();
+    SlowCompositeReaderWrapper slow = new SlowCompositeReaderWrapper(ir);
+    FieldInfos fis = slow.getFieldInfos();
+    assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, fis.fieldInfo("foo").getIndexOptions());
+    slow.close();
+    ir.close();
+    riw.close();
+    dir.close();
+  }
 
   private Token makeToken(String text, int posIncr, int startOffset, int endOffset) {
     final Token t = new Token();

Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosReader.java?rev=1344039&r1=1344038&r2=1344039&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosReader.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosReader.java
Tue May 29 22:45:32 2012
@@ -66,7 +66,9 @@ class PreFlexRWFieldInfosReader extends 
         boolean omitNorms = (bits & PreFlexRWFieldInfosWriter.OMIT_NORMS) != 0;
         boolean storePayloads = (bits & PreFlexRWFieldInfosWriter.STORE_PAYLOADS) !=
0;
         final IndexOptions indexOptions;
-        if ((bits & PreFlexRWFieldInfosWriter.OMIT_TERM_FREQ_AND_POSITIONS) != 0) {
+        if (!isIndexed) {
+          indexOptions = null;
+        } else if ((bits & PreFlexRWFieldInfosWriter.OMIT_TERM_FREQ_AND_POSITIONS) !=
0) {
           indexOptions = IndexOptions.DOCS_ONLY;
         } else if ((bits & PreFlexRWFieldInfosWriter.OMIT_POSITIONS) != 0) {
           if (format <= PreFlexRWFieldInfosWriter.FORMAT_OMIT_POSITIONS) {

Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosWriter.java?rev=1344039&r1=1344038&r2=1344039&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosWriter.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosWriter.java
Tue May 29 22:45:32 2012
@@ -62,16 +62,18 @@ class PreFlexRWFieldInfosWriter extends 
       output.writeVInt(FORMAT_PREFLEX_RW);
       output.writeVInt(infos.size());
       for (FieldInfo fi : infos) {
-        assert fi.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.hasPayloads();
         byte bits = 0x0;
-        if (fi.isIndexed()) bits |= IS_INDEXED;
         if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
         if (fi.omitsNorms()) bits |= OMIT_NORMS;
         if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
-        if (fi.getIndexOptions() == IndexOptions.DOCS_ONLY) {
-          bits |= OMIT_TERM_FREQ_AND_POSITIONS;
-        } else if (fi.getIndexOptions() == IndexOptions.DOCS_AND_FREQS) {
-          bits |= OMIT_POSITIONS;
+        if (fi.isIndexed())  {
+          bits |= IS_INDEXED;
+          assert fi.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.hasPayloads();
+          if (fi.getIndexOptions() == IndexOptions.DOCS_ONLY) {
+            bits |= OMIT_TERM_FREQ_AND_POSITIONS;
+          } else if (fi.getIndexOptions() == IndexOptions.DOCS_AND_FREQS) {
+            bits |= OMIT_POSITIONS;
+          }
         }
         output.writeString(fi.name);
         /*



Mime
View raw message