lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1367777 [8/14] - in /lucene/dev/branches/pforcodec_3892: ./ dev-tools/ dev-tools/eclipse/ dev-tools/maven/ dev-tools/scripts/ lucene/ lucene/analysis/ lucene/analysis/common/ lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ l...
Date Tue, 31 Jul 2012 20:59:01 GMT
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java Tue Jul 31 20:58:32 2012
@@ -178,7 +178,7 @@ public class PulsingPostingsReader exten
   }
 
   @Override
-  public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+  public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
     PulsingTermState termState = (PulsingTermState) _termState;
     if (termState.postingsSize != -1) {
       PulsingDocsEnum postings;
@@ -202,18 +202,18 @@ public class PulsingPostingsReader exten
       return postings.reset(liveDocs, termState);
     } else {
       if (reuse instanceof PulsingDocsEnum) {
-        DocsEnum wrapped = wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, getOther(reuse), needsFreqs);
+        DocsEnum wrapped = wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, getOther(reuse), flags);
         setOther(wrapped, reuse); // wrapped.other = reuse
         return wrapped;
       } else {
-        return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse, needsFreqs);
+        return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse, flags);
       }
     }
   }
 
   @Override
   public DocsAndPositionsEnum docsAndPositions(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsAndPositionsEnum reuse,
-                                               boolean needsOffsets) throws IOException {
+                                               int flags) throws IOException {
 
     final PulsingTermState termState = (PulsingTermState) _termState;
 
@@ -240,11 +240,11 @@ public class PulsingPostingsReader exten
     } else {
       if (reuse instanceof PulsingDocsAndPositionsEnum) {
         DocsAndPositionsEnum wrapped = wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, (DocsAndPositionsEnum) getOther(reuse),
-                                                                              needsOffsets);
+                                                                              flags);
         setOther(wrapped, reuse); // wrapped.other = reuse
         return wrapped;
       } else {
-        return wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, reuse, needsOffsets);
+        return wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, reuse, flags);
       }
     }
   }
@@ -282,6 +282,7 @@ public class PulsingPostingsReader exten
       postings.reset(postingsBytes, 0, termState.postingsSize);
       docID = -1;
       accum = 0;
+      freq = 1;
       payloadLength = 0;
       this.liveDocs = liveDocs;
       return this;
@@ -349,7 +350,6 @@ public class PulsingPostingsReader exten
 
     @Override
     public int freq() throws IOException {
-      assert indexOptions != IndexOptions.DOCS_ONLY;
       return freq;
     }
 

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java Tue Jul 31 20:58:32 2012
@@ -258,7 +258,7 @@ public class SepPostingsReader extends P
   }
 
   @Override
-  public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+  public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
     final SepTermState termState = (SepTermState) _termState;
     SepDocsEnum docsEnum;
     if (reuse == null || !(reuse instanceof SepDocsEnum)) {
@@ -278,13 +278,9 @@ public class SepPostingsReader extends P
 
   @Override
   public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs,
-                                               DocsAndPositionsEnum reuse, boolean needsOffsets)
+                                               DocsAndPositionsEnum reuse, int flags)
     throws IOException {
 
-    if (needsOffsets) {
-      return null;
-    }
-
     assert fieldInfo.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
     final SepTermState termState = (SepTermState) _termState;
     SepDocsAndPositionsEnum postingsEnum;
@@ -370,6 +366,7 @@ public class SepPostingsReader extends P
       count = 0;
       doc = -1;
       accum = 0;
+      freq = 1;
       skipped = false;
 
       return this;
@@ -403,7 +400,6 @@ public class SepPostingsReader extends P
 
     @Override
     public int freq() throws IOException {
-      assert !omitTF;
       return freq;
     }
 

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java Tue Jul 31 20:58:32 2012
@@ -194,33 +194,24 @@ class SimpleTextFieldsReader extends Fie
     }
  
     @Override
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
-      if (needsFreqs && indexOptions == IndexOptions.DOCS_ONLY) {
-        return null;
-      }
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
       SimpleTextDocsEnum docsEnum;
       if (reuse != null && reuse instanceof SimpleTextDocsEnum && ((SimpleTextDocsEnum) reuse).canReuse(SimpleTextFieldsReader.this.in)) {
         docsEnum = (SimpleTextDocsEnum) reuse;
       } else {
         docsEnum = new SimpleTextDocsEnum();
       }
-      return docsEnum.reset(docsStart, liveDocs, !needsFreqs);
+      return docsEnum.reset(docsStart, liveDocs, indexOptions == IndexOptions.DOCS_ONLY);
     }
 
     @Override
-    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
+    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
 
       if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
         // Positions were not indexed
         return null;
       }
 
-      if (needsOffsets &&
-          indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) < 0) {
-        // Offsets were not indexed
-        return null;
-      }
-
       SimpleTextDocsAndPositionsEnum docsAndPositionsEnum;
       if (reuse != null && reuse instanceof SimpleTextDocsAndPositionsEnum && ((SimpleTextDocsAndPositionsEnum) reuse).canReuse(SimpleTextFieldsReader.this.in)) {
         docsAndPositionsEnum = (SimpleTextDocsAndPositionsEnum) reuse;
@@ -260,6 +251,7 @@ class SimpleTextFieldsReader extends Fie
       in.seek(fp);
       this.omitTF = omitTF;
       docID = -1;
+      tf = 1;
       return this;
     }
 
@@ -270,7 +262,6 @@ class SimpleTextFieldsReader extends Fie
 
     @Override
     public int freq() throws IOException {
-      assert !omitTF;
       return tf;
     }
 

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java Tue Jul 31 20:58:32 2012
@@ -357,22 +357,19 @@ public class SimpleTextTermVectorsReader
     }
 
     @Override
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
       // TODO: reuse
       SimpleTVDocsEnum e = new SimpleTVDocsEnum();
-      e.reset(liveDocs, needsFreqs ? current.getValue().freq : -1);
+      e.reset(liveDocs, (flags & DocsEnum.FLAG_FREQS) == 0 ? 1 : current.getValue().freq);
       return e;
     }
 
     @Override
-    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
+    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
       SimpleTVPostings postings = current.getValue();
       if (postings.positions == null && postings.startOffsets == null) {
         return null;
       }
-      if (needsOffsets && (postings.startOffsets == null || postings.endOffsets == null)) {
-        return null;
-      }
       // TODO: reuse
       SimpleTVDocsAndPositionsEnum e = new SimpleTVDocsAndPositionsEnum();
       e.reset(liveDocs, postings.positions, postings.startOffsets, postings.endOffsets);
@@ -507,12 +504,20 @@ public class SimpleTextTermVectorsReader
 
     @Override
     public int startOffset() {
-      return startOffsets[nextPos-1];
+      if (startOffsets == null) {
+        return -1;
+      } else {
+        return startOffsets[nextPos-1];
+      }
     }
 
     @Override
     public int endOffset() {
-      return endOffsets[nextPos-1];
+      if (endOffsets == null) {
+        return -1;
+      } else {
+        return endOffsets[nextPos-1];
+      }
     }
   }
 }

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java Tue Jul 31 20:58:32 2012
@@ -116,9 +116,19 @@ public abstract class AtomicReader exten
   }
 
   /** Returns {@link DocsEnum} for the specified field &
-   *  term.  This may return null, if either the field or
+   *  term.  This will return null if either the field or
    *  term does not exist. */
-  public final DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, boolean needsFreqs) throws IOException {
+  public final DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term) throws IOException {
+    return termDocsEnum(liveDocs, field, term, DocsEnum.FLAG_FREQS);
+  }
+
+  /** Returns {@link DocsEnum} for the specified field &
+   *  term, with control over whether freqs are required.
+   *  Some codecs may be able to optimize their
+   *  implementation when freqs are not required. This will
+   *  return null if the field or term does not
+   *  exist.  See {@link TermsEnum#docs(Bits,DocsEnum,int)}. */
+  public final DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, int flags) throws IOException {
     assert field != null;
     assert term != null;
     final Fields fields = fields();
@@ -127,18 +137,30 @@ public abstract class AtomicReader exten
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator(null);
         if (termsEnum.seekExact(term, true)) {
-          return termsEnum.docs(liveDocs, null, needsFreqs);
+          return termsEnum.docs(liveDocs, null, flags);
         }
       }
     }
     return null;
   }
+  
+  /** Returns {@link DocsAndPositionsEnum} for the specified
+   *  field & term.  This will return null if the
+   *  field or term does not exist or positions weren't indexed. 
+   *  @see #termPositionsEnum(Bits, String, BytesRef, int) */
+  public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term) throws IOException {
+    return termPositionsEnum(liveDocs, field, term, DocsAndPositionsEnum.FLAG_OFFSETS | DocsAndPositionsEnum.FLAG_PAYLOADS);
+  }
+
 
   /** Returns {@link DocsAndPositionsEnum} for the specified
-   *  field & term.  This may return null, if either the
-   *  field or term does not exist, or needsOffsets is
-   *  true but offsets were not indexed for this field. */
-  public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, boolean needsOffsets) throws IOException {
+   *  field & term, with control over whether offsets and payloads are
+   *  required.  Some codecs may be able to optimize their
+   *  implementation when offsets and/or payloads are not required.
+   *  This will return null if the field or term
+   *  does not exist or positions weren't indexed.  See
+   *  {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)}. */
+  public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, int flags) throws IOException {
     assert field != null;
     assert term != null;
     final Fields fields = fields();
@@ -147,7 +169,7 @@ public abstract class AtomicReader exten
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator(null);
         if (termsEnum.seekExact(term, true)) {
-          return termsEnum.docsAndPositions(liveDocs, null, needsOffsets);
+          return termsEnum.docsAndPositions(liveDocs, null, flags);
         }
       }
     }

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java Tue Jul 31 20:58:32 2012
@@ -396,7 +396,8 @@ class BufferedDeletesStream {
       // System.out.println("  term=" + term);
 
       if (termsEnum.seekExact(term.bytes(), false)) {
-        DocsEnum docsEnum = termsEnum.docs(rld.getLiveDocs(), docs, false);
+        // we don't need term frequencies for this
+        DocsEnum docsEnum = termsEnum.docs(rld.getLiveDocs(), docs, 0);
         //System.out.println("BDS: got docsEnum=" + docsEnum);
 
         if (docsEnum != null) {

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Tue Jul 31 20:58:32 2012
@@ -34,6 +34,7 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.document.FieldType; // for javadocs
 import org.apache.lucene.index.DocValues.SortedSource;
 import org.apache.lucene.index.DocValues.Source;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
@@ -682,7 +683,6 @@ public class CheckIndex {
     DocsEnum docs = null;
     DocsEnum docsAndFreqs = null;
     DocsAndPositionsEnum postings = null;
-    DocsAndPositionsEnum offsets = null;
     
     String lastField = null;
     final FieldsEnum fieldsEnum = fields.iterator();
@@ -699,11 +699,11 @@ public class CheckIndex {
       
       // check that the field is in fieldinfos, and is indexed.
       // TODO: add a separate test to check this for different reader impls
-      FieldInfo fi = fieldInfos.fieldInfo(field);
-      if (fi == null) {
+      FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+      if (fieldInfo == null) {
         throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, no fieldInfos for: " + field);
       }
-      if (!fi.isIndexed()) {
+      if (!fieldInfo.isIndexed()) {
         throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, isIndexed == false for: " + field);
       }
       
@@ -755,10 +755,8 @@ public class CheckIndex {
         status.totFreq += docFreq;
         sumDocFreq += docFreq;
         
-        docs = termsEnum.docs(liveDocs, docs, false);
-        docsAndFreqs = termsEnum.docs(liveDocs, docsAndFreqs, true);
-        postings = termsEnum.docsAndPositions(liveDocs, postings, false);
-        offsets = termsEnum.docsAndPositions(liveDocs, offsets, true);
+        docs = termsEnum.docs(liveDocs, docs);
+        postings = termsEnum.docsAndPositions(liveDocs, postings);
         
         if (hasOrd) {
           long ord = -1;
@@ -779,34 +777,17 @@ public class CheckIndex {
         status.termCount++;
         
         final DocsEnum docs2;
-        final DocsEnum docsAndFreqs2;
         final boolean hasPositions;
-        final boolean hasFreqs;
-        final boolean hasOffsets;
-        if (offsets != null) {
-          docs2 = postings = offsets;
-          docsAndFreqs2 = postings = offsets;
-          hasOffsets = true;
-          hasPositions = true;
-          hasFreqs = true;
-        } else if (postings != null) {
+        // if we are checking vectors, we have freqs implicitly
+        final boolean hasFreqs = isVectors || fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+        // if we are checking vectors, offsets are a free-for-all anyway
+        final boolean hasOffsets = isVectors || fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+        if (postings != null) {
           docs2 = postings;
-          docsAndFreqs2 = postings;
-          hasOffsets = false;
           hasPositions = true;
-          hasFreqs = true;
-        } else if (docsAndFreqs != null) {
-          docs2 = docsAndFreqs;
-          docsAndFreqs2 = docsAndFreqs;
-          hasOffsets = false;
-          hasPositions = false;
-          hasFreqs = true;
         } else {
           docs2 = docs;
-          docsAndFreqs2 = null;
-          hasOffsets = false;
           hasPositions = false;
-          hasFreqs = false;
         }
         
         int lastDoc = -1;
@@ -820,7 +801,7 @@ public class CheckIndex {
           visitedDocs.set(doc);
           int freq = -1;
           if (hasFreqs) {
-            freq = docsAndFreqs2.freq();
+            freq = docs2.freq();
             if (freq <= 0) {
               throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
             }
@@ -886,12 +867,12 @@ public class CheckIndex {
         }
         
         final long totalTermFreq2 = termsEnum.totalTermFreq();
-        final boolean hasTotalTermFreq = postings != null && totalTermFreq2 != -1;
+        final boolean hasTotalTermFreq = hasFreqs && totalTermFreq2 != -1;
         
         // Re-count if there are deleted docs:
         if (liveDocs != null) {
           if (hasFreqs) {
-            final DocsEnum docsNoDel = termsEnum.docs(null, docsAndFreqs, true);
+            final DocsEnum docsNoDel = termsEnum.docs(null, docsAndFreqs);
             docCount = 0;
             totalTermFreq = 0;
             while(docsNoDel.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
@@ -900,7 +881,7 @@ public class CheckIndex {
               totalTermFreq += docsNoDel.freq();
             }
           } else {
-            final DocsEnum docsNoDel = termsEnum.docs(null, docs, false);
+            final DocsEnum docsNoDel = termsEnum.docs(null, docs, 0);
             docCount = 0;
             totalTermFreq = -1;
             while(docsNoDel.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
@@ -927,7 +908,7 @@ public class CheckIndex {
         if (hasPositions) {
           for(int idx=0;idx<7;idx++) {
             final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
-            postings = termsEnum.docsAndPositions(liveDocs, postings, hasOffsets);
+            postings = termsEnum.docsAndPositions(liveDocs, postings);
             final int docID = postings.advance(skipDocID);
             if (docID == DocIdSetIterator.NO_MORE_DOCS) {
               break;
@@ -992,7 +973,7 @@ public class CheckIndex {
         } else {
           for(int idx=0;idx<7;idx++) {
             final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
-            docs = termsEnum.docs(liveDocs, docs, false);
+            docs = termsEnum.docs(liveDocs, docs, 0);
             final int docID = docs.advance(skipDocID);
             if (docID == DocIdSetIterator.NO_MORE_DOCS) {
               break;
@@ -1062,7 +1043,7 @@ public class CheckIndex {
           }
           
           int expectedDocFreq = termsEnum.docFreq();
-          DocsEnum d = termsEnum.docs(null, null, false);
+          DocsEnum d = termsEnum.docs(null, null, 0);
           int docFreq = 0;
           while (d.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
             docFreq++;
@@ -1103,7 +1084,7 @@ public class CheckIndex {
                 throw new RuntimeException("seek to existing term " + seekTerms[i] + " failed");
               }
               
-              docs = termsEnum.docs(liveDocs, docs, false);
+              docs = termsEnum.docs(liveDocs, docs, 0);
               if (docs == null) {
                 throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]);
               }
@@ -1121,7 +1102,7 @@ public class CheckIndex {
               }
               
               totDocFreq += termsEnum.docFreq();
-              docs = termsEnum.docs(null, docs, false);
+              docs = termsEnum.docs(null, docs, 0);
               if (docs == null) {
                 throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]);
               }
@@ -1450,6 +1431,7 @@ public class CheckIndex {
             if (crossCheckTermVectors) {
               Terms terms = tfv.terms(field);
               termsEnum = terms.iterator(termsEnum);
+              final boolean postingsHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
 
               Terms postingsTerms = postingsFields.terms(field);
               if (postingsTerms == null) {
@@ -1460,44 +1442,20 @@ public class CheckIndex {
               BytesRef term = null;
               while ((term = termsEnum.next()) != null) {
                 
-                final boolean hasPositions;
-                final boolean hasOffsets;
-                final boolean hasFreqs;
-
-                // TODO: really we need a reflection/query
-                // API so we can just ask what was indexed
-                // instead of "probing"...
+                final boolean hasProx;
 
-                // Try offsets:
-                postings = termsEnum.docsAndPositions(null, postings, true);
+                // Try positions:
+                postings = termsEnum.docsAndPositions(null, postings);
                 if (postings == null) {
-                  hasOffsets = false;
-                  // Try only positions:
-                  postings = termsEnum.docsAndPositions(null, postings, false);
-                  if (postings == null) {
-                    hasPositions = false;
-                    // Try docIDs & freqs:
-                    docs = termsEnum.docs(null, docs, true);
-                    if (docs == null) {
-                      // OK, only docIDs:
-                      hasFreqs = false;
-                      docs = termsEnum.docs(null, docs, false);
-                    } else {
-                      hasFreqs = true;
-                    }
-                  } else {
-                    hasPositions = true;
-                    hasFreqs = true;
-                  }
+                  hasProx = false;
+                  // Try docIDs & freqs:
+                  docs = termsEnum.docs(null, docs);
                 } else {
-                  hasOffsets = true;
-                  // NOTE: may be a lie... but we accept -1
-                  hasPositions = true;
-                  hasFreqs = true;
+                  hasProx = true;
                 }
 
                 final DocsEnum docs2;
-                if (hasPositions || hasOffsets) {
+                if (hasProx) {
                   assert postings != null;
                   docs2 = postings;
                 } else {
@@ -1506,30 +1464,16 @@ public class CheckIndex {
                 }
 
                 final DocsEnum postingsDocs2;
-                final boolean postingsHasFreq;
                 if (!postingsTermsEnum.seekExact(term, true)) {
                   throw new RuntimeException("vector term=" + term + " field=" + field + " does not exist in postings; doc=" + j);
                 }
-                postingsPostings = postingsTermsEnum.docsAndPositions(null, postingsPostings, true);
+                postingsPostings = postingsTermsEnum.docsAndPositions(null, postingsPostings);
                 if (postingsPostings == null) {
-                  // Term vectors were indexed w/ offsets but postings were not
-                  postingsPostings = postingsTermsEnum.docsAndPositions(null, postingsPostings, false);
-                  if (postingsPostings == null) {
-                    postingsDocs = postingsTermsEnum.docs(null, postingsDocs, true);
-                    if (postingsDocs == null) {
-                      postingsHasFreq = false;
-                      postingsDocs = postingsTermsEnum.docs(null, postingsDocs, false);
-                      if (postingsDocs == null) {
-                        throw new RuntimeException("vector term=" + term + " field=" + field + " does not exist in postings; doc=" + j);
-                      }
-                    } else {
-                      postingsHasFreq = true;
-                    }
-                  } else {
-                    postingsHasFreq = true;
+                  // Term vectors were indexed w/ pos but postings were not
+                  postingsDocs = postingsTermsEnum.docs(null, postingsDocs);
+                  if (postingsDocs == null) {
+                    throw new RuntimeException("vector term=" + term + " field=" + field + " does not exist in postings; doc=" + j);
                   }
-                } else {
-                  postingsHasFreq = true;
                 }
 
                 if (postingsPostings != null) {
@@ -1549,13 +1493,13 @@ public class CheckIndex {
                   throw new RuntimeException("vector for doc " + j + " didn't return docID=0: got docID=" + doc);
                 }
 
-                if (hasFreqs) {
+                if (postingsHasFreq) {
                   final int tf = docs2.freq();
                   if (postingsHasFreq && postingsDocs2.freq() != tf) {
                     throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": freq=" + tf + " differs from postings freq=" + postingsDocs2.freq());
                   }
                 
-                  if (hasPositions || hasOffsets) {
+                  if (hasProx) {
                     for (int i = 0; i < tf; i++) {
                       int pos = postings.nextPosition();
                       if (postingsPostings != null) {
@@ -1565,32 +1509,30 @@ public class CheckIndex {
                         }
                       }
 
-                      if (hasOffsets) {
-                        // Call the methods to at least make
-                        // sure they don't throw exc:
-                        final int startOffset = postings.startOffset();
-                        final int endOffset = postings.endOffset();
-                        // TODO: these are too anal...?
-                        /*
-                          if (endOffset < startOffset) {
-                          throw new RuntimeException("vector startOffset=" + startOffset + " is > endOffset=" + endOffset);
-                          }
-                          if (startOffset < lastStartOffset) {
-                          throw new RuntimeException("vector startOffset=" + startOffset + " is < prior startOffset=" + lastStartOffset);
-                          }
-                          lastStartOffset = startOffset;
-                        */
-
-                        if (postingsPostings != null) {
-                          final int postingsStartOffset = postingsPostings.startOffset();
-
-                          final int postingsEndOffset = postingsPostings.endOffset();
-                          if (startOffset != -1 && postingsStartOffset != -1 && startOffset != postingsStartOffset) {
-                            throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": startOffset=" + startOffset + " differs from postings startOffset=" + postingsStartOffset);
-                          }
-                          if (endOffset != -1 && postingsEndOffset != -1 && endOffset != postingsEndOffset) {
-                            throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": endOffset=" + endOffset + " differs from postings endOffset=" + postingsEndOffset);
-                          }
+                      // Call the methods to at least make
+                      // sure they don't throw exc:
+                      final int startOffset = postings.startOffset();
+                      final int endOffset = postings.endOffset();
+                      // TODO: these are too anal...?
+                      /*
+                        if (endOffset < startOffset) {
+                        throw new RuntimeException("vector startOffset=" + startOffset + " is > endOffset=" + endOffset);
+                        }
+                        if (startOffset < lastStartOffset) {
+                        throw new RuntimeException("vector startOffset=" + startOffset + " is < prior startOffset=" + lastStartOffset);
+                        }
+                        lastStartOffset = startOffset;
+                      */
+
+                      if (postingsPostings != null) {
+                        final int postingsStartOffset = postingsPostings.startOffset();
+
+                        final int postingsEndOffset = postingsPostings.endOffset();
+                        if (startOffset != -1 && postingsStartOffset != -1 && startOffset != postingsStartOffset) {
+                          throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": startOffset=" + startOffset + " differs from postings startOffset=" + postingsStartOffset);
+                        }
+                        if (endOffset != -1 && postingsEndOffset != -1 && endOffset != postingsEndOffset) {
+                          throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": endOffset=" + endOffset + " differs from postings endOffset=" + postingsEndOffset);
                         }
                       }
                     }

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Tue Jul 31 20:58:32 2012
@@ -243,27 +243,34 @@ public class ConcurrentMergeScheduler ex
     sync();
   }
 
-  /** Wait for any running merge threads to finish */
+  /** Wait for any running merge threads to finish. This call is not interruptible as used by {@link #close()}. */
   public void sync() {
-    while (true) {
-      MergeThread toSync = null;
-      synchronized (this) {
-        for (MergeThread t : mergeThreads) {
-          if (t.isAlive()) {
-            toSync = t;
-            break;
+    boolean interrupted = false;
+    try {
+      while (true) {
+        MergeThread toSync = null;
+        synchronized (this) {
+          for (MergeThread t : mergeThreads) {
+            if (t.isAlive()) {
+              toSync = t;
+              break;
+            }
           }
         }
-      }
-      if (toSync != null) {
-        try {
-          toSync.join();
-        } catch (InterruptedException ie) {
-          throw new ThreadInterruptedException(ie);
+        if (toSync != null) {
+          try {
+            toSync.join();
+          } catch (InterruptedException ie) {
+            // ignore this Exception, we will retry until all threads are dead
+            interrupted = true;
+          }
+        } else {
+          break;
         }
-      } else {
-        break;
       }
+    } finally {
+      // finally, restore interrupt status:
+      if (interrupted) Thread.currentThread().interrupt();
     }
   }
 

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java Tue Jul 31 20:58:32 2012
@@ -334,7 +334,7 @@ public class DocTermOrds {
       final int df = te.docFreq();
       if (df <= maxTermDocFreq) {
 
-        docsEnum = te.docs(liveDocs, docsEnum, false);
+        docsEnum = te.docs(liveDocs, docsEnum, 0);
 
         // dF, but takes deletions into account
         int actualDF = 0;
@@ -668,13 +668,13 @@ public class DocTermOrds {
     }
 
     @Override    
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
-      return termsEnum.docs(liveDocs, reuse, needsFreqs);
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
+      return termsEnum.docs(liveDocs, reuse, flags);
     }
 
     @Override    
-    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
-      return termsEnum.docsAndPositions(liveDocs, reuse, needsOffsets);
+    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
+      return termsEnum.docsAndPositions(liveDocs, reuse, flags);
     }
 
     @Override

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java Tue Jul 31 20:58:32 2012
@@ -23,7 +23,14 @@ import org.apache.lucene.util.BytesRef;
 
 /** Also iterates through positions. */
 public abstract class DocsAndPositionsEnum extends DocsEnum {
-
+  /** Flag to pass to {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)}
+   *  if you require offsets in the returned enum. */
+  public static final int FLAG_OFFSETS = 0x1;
+
+  /** Flag to pass to  {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)}
+   *  if you require payloads in the returned enum. */
+  public static final int FLAG_PAYLOADS = 0x2;
+  
   /** Returns the next position.  You should only call this
    *  up to {@link DocsEnum#freq()} times else
    *  the behavior is not defined.  If positions were not

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java Tue Jul 31 20:58:32 2012
@@ -27,6 +27,10 @@ import org.apache.lucene.util.AttributeS
  *  any of the per-doc methods. */
 public abstract class DocsEnum extends DocIdSetIterator {
 
+  /** Flag to pass to {@link TermsEnum#docs(Bits,DocsEnum,int)}
+   *  if you require term frequencies in the returned enum. */
+  public static final int FLAG_FREQS = 0x1;
+
   private AttributeSource atts = null;
 
   /** Returns term frequency in the current document.  Do

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java Tue Jul 31 20:58:32 2012
@@ -181,13 +181,13 @@ public class FilterAtomicReader extends 
     }
 
     @Override
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
-      return in.docs(liveDocs, reuse, needsFreqs);
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
+      return in.docs(liveDocs, reuse, flags);
     }
 
     @Override
-    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
-      return in.docsAndPositions(liveDocs, reuse, needsOffsets);
+    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
+      return in.docsAndPositions(liveDocs, reuse, flags);
     }
 
     @Override

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java Tue Jul 31 20:58:32 2012
@@ -166,13 +166,13 @@ public abstract class FilteredTermsEnum 
   }
 
   @Override
-  public DocsEnum docs(Bits bits, DocsEnum reuse, boolean needsFreqs) throws IOException {
-    return tenum.docs(bits, reuse, needsFreqs);
+  public DocsEnum docs(Bits bits, DocsEnum reuse, int flags) throws IOException {
+    return tenum.docs(bits, reuse, flags);
   }
     
   @Override
-  public DocsAndPositionsEnum docsAndPositions(Bits bits, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
-    return tenum.docsAndPositions(bits, reuse, needsOffsets);
+  public DocsAndPositionsEnum docsAndPositions(Bits bits, DocsAndPositionsEnum reuse, int flags) throws IOException {
+    return tenum.docsAndPositions(bits, reuse, flags);
   }
   
   /** This enum does not support seeking!

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Tue Jul 31 20:58:32 2012
@@ -25,6 +25,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Matcher;
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.NoSuchDirectoryException;
@@ -146,57 +147,61 @@ final class IndexFileDeleter {
       // it means the directory is empty, so ignore it.
       files = new String[0];
     }
-
-    for (String fileName : files) {
-
-      if (!fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
-
-        // Add this file to refCounts with initial count 0:
-        getRefCount(fileName);
-
-        if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
-
-          // This is a commit (segments or segments_N), and
-          // it's valid (<= the max gen).  Load it, then
-          // incref all files it refers to:
-          if (infoStream.isEnabled("IFD")) {
-            infoStream.message("IFD", "init: load commit \"" + fileName + "\"");
-          }
-          SegmentInfos sis = new SegmentInfos();
-          try {
-            sis.read(directory, fileName);
-          } catch (FileNotFoundException e) {
-            // LUCENE-948: on NFS (and maybe others), if
-            // you have writers switching back and forth
-            // between machines, it's very likely that the
-            // dir listing will be stale and will claim a
-            // file segments_X exists when in fact it
-            // doesn't.  So, we catch this and handle it
-            // as if the file does not exist
+    
+    if (currentSegmentsFile != null) {
+      Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
+      for (String fileName : files) {
+        m.reset(fileName);
+        if (!fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)
+            && (m.matches() || fileName.startsWith(IndexFileNames.SEGMENTS))) {
+          
+          // Add this file to refCounts with initial count 0:
+          getRefCount(fileName);
+          
+          if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
+            
+            // This is a commit (segments or segments_N), and
+            // it's valid (<= the max gen).  Load it, then
+            // incref all files it refers to:
             if (infoStream.isEnabled("IFD")) {
-              infoStream.message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
+              infoStream.message("IFD", "init: load commit \"" + fileName + "\"");
             }
-            sis = null;
-          } catch (IOException e) {
-            if (SegmentInfos.generationFromSegmentsFileName(fileName) <= currentGen && directory.fileLength(fileName) > 0) {
-              throw e;
-            } else {
-              // Most likely we are opening an index that
-              // has an aborted "future" commit, so suppress
-              // exc in this case
+            SegmentInfos sis = new SegmentInfos();
+            try {
+              sis.read(directory, fileName);
+            } catch (FileNotFoundException e) {
+              // LUCENE-948: on NFS (and maybe others), if
+              // you have writers switching back and forth
+              // between machines, it's very likely that the
+              // dir listing will be stale and will claim a
+              // file segments_X exists when in fact it
+              // doesn't.  So, we catch this and handle it
+              // as if the file does not exist
+              if (infoStream.isEnabled("IFD")) {
+                infoStream.message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
+              }
               sis = null;
+            } catch (IOException e) {
+              if (SegmentInfos.generationFromSegmentsFileName(fileName) <= currentGen && directory.fileLength(fileName) > 0) {
+                throw e;
+              } else {
+                // Most likely we are opening an index that
+                // has an aborted "future" commit, so suppress
+                // exc in this case
+                sis = null;
+              }
             }
-          }
-          if (sis != null) {
-            final CommitPoint commitPoint = new CommitPoint(commitsToDelete, directory, sis);
-            if (sis.getGeneration() == segmentInfos.getGeneration()) {
-              currentCommitPoint = commitPoint;
-            }
-            commits.add(commitPoint);
-            incRef(sis, true);
-
-            if (lastSegmentInfos == null || sis.getGeneration() > lastSegmentInfos.getGeneration()) {
-              lastSegmentInfos = sis;
+            if (sis != null) {
+              final CommitPoint commitPoint = new CommitPoint(commitsToDelete, directory, sis);
+              if (sis.getGeneration() == segmentInfos.getGeneration()) {
+                currentCommitPoint = commitPoint;
+              }
+              commits.add(commitPoint);
+              incRef(sis, true);
+              
+              if (lastSegmentInfos == null || sis.getGeneration() > lastSegmentInfos.getGeneration()) {
+                lastSegmentInfos = sis;
+              }
             }
           }
         }

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java Tue Jul 31 20:58:32 2012
@@ -17,6 +17,8 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import java.util.regex.Pattern;
+
 import org.apache.lucene.codecs.Codec;
 
 // TODO: put all files under codec and remove all the static extensions here
@@ -189,4 +191,8 @@ public final class IndexFileNames {
     }
     return filename;
   }  
+
+  // All files created by codecs much match this pattern (we
+  // check this in SegmentInfo.java):
+  static final Pattern CODEC_FILE_PATTERN = Pattern.compile("_[a-z0-9]+(_.*)?\\..*");
 }

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Tue Jul 31 20:58:32 2012
@@ -842,7 +842,7 @@ public class IndexWriter implements Clos
         if (hitOOM) {
           rollbackInternal();
         } else {
-          closeInternal(waitForMerges, !hitOOM);
+          closeInternal(waitForMerges, true);
         }
       }
     }
@@ -870,7 +870,7 @@ public class IndexWriter implements Clos
   }
 
   private void closeInternal(boolean waitForMerges, boolean doFlush) throws IOException {
-
+    boolean interrupted = false;
     try {
 
       if (pendingCommit != null) {
@@ -883,26 +883,57 @@ public class IndexWriter implements Clos
 
       docWriter.close();
 
-      // Only allow a new merge to be triggered if we are
-      // going to wait for merges:
-      if (doFlush) {
-        flush(waitForMerges, true);
-      } else {
-        docWriter.abort(); // already closed
-      }
-
-      if (waitForMerges)
-        // Give merge scheduler last chance to run, in case
-        // any pending merges are waiting:
-        mergeScheduler.merge(this);
-
-      mergePolicy.close();
-
-      synchronized(this) {
-        finishMerges(waitForMerges);
-        stopMerges = true;
+      try {
+        // Only allow a new merge to be triggered if we are
+        // going to wait for merges:
+        if (doFlush) {
+          flush(waitForMerges, true);
+        } else {
+          docWriter.abort(); // already closed
+        }
+        
+      } finally {
+        try {
+          // clean up merge scheduler in all cases, although flushing may have failed:
+          interrupted = Thread.interrupted();
+        
+          if (waitForMerges) {
+            try {
+              // Give merge scheduler last chance to run, in case
+              // any pending merges are waiting:
+              mergeScheduler.merge(this);
+            } catch (ThreadInterruptedException tie) {
+              // ignore any interruption, does not matter
+              interrupted = true;
+              if (infoStream.isEnabled("IW")) {
+                infoStream.message("IW", "interrupted while waiting for final merges");
+              }
+            }
+          }
+          
+          synchronized(this) {
+            for (;;) {
+              try {
+                finishMerges(waitForMerges && !interrupted);
+                break;
+              } catch (ThreadInterruptedException tie) {
+                // by setting the interrupted status, the
+                // next call to finishMerges will pass false,
+                // so it will not wait
+                interrupted = true;
+                if (infoStream.isEnabled("IW")) {
+                  infoStream.message("IW", "interrupted while waiting for merges to finish");
+                }
+              }
+            }
+            stopMerges = true;
+          }
+          
+        } finally {
+          // shutdown policy, scheduler and all threads (this call is not interruptible):
+          IOUtils.closeWhileHandlingException(mergePolicy, mergeScheduler);
+        }
       }
-      mergeScheduler.close();
 
       if (infoStream.isEnabled("IW")) {
         infoStream.message("IW", "now call final commit()");
@@ -943,6 +974,8 @@ public class IndexWriter implements Clos
           }
         }
       }
+      // finally, restore interrupt status:
+      if (interrupted) Thread.currentThread().interrupt();
     }
   }
 
@@ -3052,7 +3085,7 @@ public class IndexWriter implements Clos
     checkpoint();
 
     if (infoStream.isEnabled("IW")) {
-      infoStream.message("IW", "after commit: " + segString());
+      infoStream.message("IW", "after commitMerge: " + segString());
     }
 
     if (merge.maxNumSegments != -1 && !dropSegment) {

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java Tue Jul 31 20:58:32 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import java.io.Closeable;
 import java.io.IOException;
 
 /** <p>Expert: {@link IndexWriter} uses an instance
@@ -26,7 +27,7 @@ import java.io.IOException;
  *
  * @lucene.experimental
 */
-public abstract class MergeScheduler {
+public abstract class MergeScheduler implements Closeable {
 
   /** Run the merges provided by {@link IndexWriter#getNextMerge()}. */
   public abstract void merge(IndexWriter writer) throws IOException;

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiFields.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiFields.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiFields.java Tue Jul 31 20:58:32 2012
@@ -122,32 +122,54 @@ public final class MultiFields extends F
   }
   
   /** Returns {@link DocsEnum} for the specified field &
-   *  term.  This may return null if the term does not
-   *  exist. */
-  public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, boolean needsFreqs) throws IOException {
+   *  term.  This will return null if the field or term does
+   *  not exist. */
+  public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException {
+    return getTermDocsEnum(r, liveDocs, field, term);
+  }
+  
+  /** Returns {@link DocsEnum} for the specified field &
+   *  term, with control over whether freqs are required.
+   *  Some codecs may be able to optimize their
+   *  implementation when freqs are not required.  This will
+   *  return null if the field or term does not exist.  See {@link
+   *  TermsEnum#docs(Bits,DocsEnum,int)}.*/
+  public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, int flags) throws IOException {
     assert field != null;
     assert term != null;
     final Terms terms = getTerms(r, field);
     if (terms != null) {
       final TermsEnum termsEnum = terms.iterator(null);
       if (termsEnum.seekExact(term, true)) {
-        return termsEnum.docs(liveDocs, null, needsFreqs);
+        return termsEnum.docs(liveDocs, null, flags);
       }
     }
     return null;
   }
 
   /** Returns {@link DocsAndPositionsEnum} for the specified
-   *  field & term.  This may return null if the term does
-   *  not exist or positions were not indexed. */
-  public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, boolean needsOffsets) throws IOException {
+   *  field & term.  This will return null if the field or
+   *  term does not exist or positions were not indexed. 
+   *  @see #getTermPositionsEnum(IndexReader, Bits, String, BytesRef, int) */
+  public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException {
+    return getTermPositionsEnum(r, liveDocs, field, term, DocsAndPositionsEnum.FLAG_OFFSETS | DocsAndPositionsEnum.FLAG_PAYLOADS);
+  }
+
+  /** Returns {@link DocsAndPositionsEnum} for the specified
+   *  field & term, with control over whether offsets and payloads are
+   *  required.  Some codecs may be able to optimize
+   *  their implementation when offsets and/or payloads are not
+   *  required. This will return null if the field or term does not
+   *  exist or positions were not indexed. See {@link
+   *  TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)}. */
+  public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, int flags) throws IOException {
     assert field != null;
     assert term != null;
     final Terms terms = getTerms(r, field);
     if (terms != null) {
       final TermsEnum termsEnum = terms.iterator(null);
       if (termsEnum.seekExact(term, true)) {
-        return termsEnum.docsAndPositions(liveDocs, null, needsOffsets);
+        return termsEnum.docsAndPositions(liveDocs, null, flags);
       }
     }
     return null;

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java Tue Jul 31 20:58:32 2012
@@ -344,7 +344,7 @@ public final class MultiTermsEnum extend
   }
 
   @Override
-  public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+  public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
     MultiDocsEnum docsEnum;
     // Can only reuse if incoming enum is also a MultiDocsEnum
     if (reuse != null && reuse instanceof MultiDocsEnum) {
@@ -394,16 +394,15 @@ public final class MultiTermsEnum extend
       }
 
       assert entry.index < docsEnum.subDocsEnum.length: entry.index + " vs " + docsEnum.subDocsEnum.length + "; " + subs.length;
-      final DocsEnum subDocsEnum = entry.terms.docs(b, docsEnum.subDocsEnum[entry.index], needsFreqs);
+      final DocsEnum subDocsEnum = entry.terms.docs(b, docsEnum.subDocsEnum[entry.index], flags);
       if (subDocsEnum != null) {
         docsEnum.subDocsEnum[entry.index] = subDocsEnum;
         subDocs[upto].docsEnum = subDocsEnum;
         subDocs[upto].slice = entry.subSlice;
         upto++;
       } else {
-        // One of our subs cannot provide freqs:
-        assert needsFreqs;
-        return null;
+        // One of our subs cannot provide a docsenum:
+        assert false;
       }
     }
 
@@ -415,7 +414,7 @@ public final class MultiTermsEnum extend
   }
 
   @Override
-  public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
+  public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
     MultiDocsAndPositionsEnum docsAndPositionsEnum;
     // Can only reuse if incoming enum is also a MultiDocsAndPositionsEnum
     if (reuse != null && reuse instanceof MultiDocsAndPositionsEnum) {
@@ -466,7 +465,7 @@ public final class MultiTermsEnum extend
       }
 
       assert entry.index < docsAndPositionsEnum.subDocsAndPositionsEnum.length: entry.index + " vs " + docsAndPositionsEnum.subDocsAndPositionsEnum.length + "; " + subs.length;
-      final DocsAndPositionsEnum subPostings = entry.terms.docsAndPositions(b, docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index], needsOffsets);
+      final DocsAndPositionsEnum subPostings = entry.terms.docsAndPositions(b, docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index], flags);
 
       if (subPostings != null) {
         docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index] = subPostings;
@@ -474,7 +473,7 @@ public final class MultiTermsEnum extend
         subDocsAndPositions[upto].slice = entry.subSlice;
         upto++;
       } else {
-        if (entry.terms.docs(b, null, false) != null) {
+        if (entry.terms.docs(b, null, 0) != null) {
           // At least one of our subs does not store
           // offsets or positions -- we can't correctly
           // produce a MultiDocsAndPositions enum

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java Tue Jul 31 20:58:32 2012
@@ -24,6 +24,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Matcher;
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.store.Directory;
@@ -242,16 +243,31 @@ public final class SegmentInfo {
   private Set<String> setFiles;
 
   public void setFiles(Set<String> files) {
+    checkFileNames(files);
     setFiles = files;
     sizeInBytes = -1;
   }
 
   public void addFiles(Collection<String> files) {
+    checkFileNames(files);
     setFiles.addAll(files);
+    sizeInBytes = -1;
   }
 
   public void addFile(String file) {
+    checkFileNames(Collections.singleton(file));
     setFiles.add(file);
+    sizeInBytes = -1;
+  }
+  
+  private void checkFileNames(Collection<String> files) {
+    Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
+    for (String file : files) {
+      m.reset(file);
+      if (!m.matches()) {
+        throw new IllegalArgumentException("invalid codec filename '" + file + "', must match: " + IndexFileNames.CODEC_FILE_PATTERN.pattern());
+      }
+    }
   }
     
   /**

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java Tue Jul 31 20:58:32 2012
@@ -140,26 +140,55 @@ public abstract class TermsEnum implemen
 
   /** Get {@link DocsEnum} for the current term.  Do not
    *  call this when the enum is unpositioned.  This method
-   *  may return null (if needsFreqs is true but freqs were
-   *  not indexed for this field).
+   *  will not return null.
+   *  
+   * @param liveDocs unset bits are documents that should not
+   * be returned
+   * @param reuse pass a prior DocsEnum for possible reuse */
+  public final DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+    return docs(liveDocs, reuse, DocsEnum.FLAG_FREQS);
+  }
+
+  /** Get {@link DocsEnum} for the current term, with
+   *  control over whether freqs are required.  Do not
+   *  call this when the enum is unpositioned.  This method
+   *  will not return null.
    *  
    * @param liveDocs unset bits are documents that should not
    * be returned
    * @param reuse pass a prior DocsEnum for possible reuse
-   * @param needsFreqs true if the caller intends to call
-   * {@link DocsEnum#freq}.  If you pass false you must not
-   * call {@link DocsEnum#freq} in the returned DocsEnum. */
-  public abstract DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException;
+   * @param flags specifies which optional per-document values
+   *        you require; see {@link DocsEnum#FLAG_FREQS} 
+   * @see #docs(Bits, DocsEnum, int) */
+  public abstract DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException;
 
   /** Get {@link DocsAndPositionsEnum} for the current term.
-   *  Do not call this when the enum is unpositioned.
-   *  This method will only return null if needsOffsets is
-   *  true but offsets were not indexed.
+   *  Do not call this when the enum is unpositioned.  This
+   *  method will return null if positions were not
+   *  indexed.
+   *  
+   *  @param liveDocs unset bits are documents that should not
+   *  be returned
+   *  @param reuse pass a prior DocsAndPositionsEnum for possible reuse
+   *  @see #docsAndPositions(Bits, DocsAndPositionsEnum, int) */
+  public final DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException {
+    return docsAndPositions(liveDocs, reuse, DocsAndPositionsEnum.FLAG_OFFSETS | DocsAndPositionsEnum.FLAG_PAYLOADS);
+  }
+
+  /** Get {@link DocsAndPositionsEnum} for the current term,
+   *  with control over whether offsets and payloads are
+   *  required.  Some codecs may be able to optimize their
+   *  implementation when offsets and/or payloads are not required.
+   *  Do not call this when the enum is unpositioned.  This
+   *  will return null if positions were not indexed.
+
    *  @param liveDocs unset bits are documents that should not
    *  be returned
    *  @param reuse pass a prior DocsAndPositionsEnum for possible reuse
-   *  @param needsOffsets true if offsets are required */
-  public abstract DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException;
+   *  @param flags specifies which optional per-position values you
+   *         require; see {@link DocsAndPositionsEnum#FLAG_OFFSETS} and 
+   *         {@link DocsAndPositionsEnum#FLAG_PAYLOADS}. */
+  public abstract DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException;
 
   /**
    * Expert: Returns the TermsEnums internal state to position the TermsEnum
@@ -220,12 +249,12 @@ public abstract class TermsEnum implemen
     }
 
     @Override
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) {
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) {
       throw new IllegalStateException("this method should never be called");
     }
       
     @Override
-    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) {
+    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) {
       throw new IllegalStateException("this method should never be called");
     }
       

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java Tue Jul 31 20:58:32 2012
@@ -361,11 +361,9 @@ public class BooleanQuery extends Query 
         final Scorer scorer = weight.scorer(context, true, false, acceptDocs);
         if (scorer == null) {
           return null;
-        }
-        if (scorer instanceof TermScorer) {
-          docsAndFreqs[i] = new DocsAndFreqs((TermScorer) scorer);
         } else {
-          docsAndFreqs[i] = new DocsAndFreqs((MatchOnlyTermScorer) scorer);
+          assert scorer instanceof TermScorer;
+          docsAndFreqs[i] = new DocsAndFreqs((TermScorer) scorer);
         }
       }
       return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord(

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java Tue Jul 31 20:58:32 2012
@@ -121,10 +121,6 @@ class ConjunctionTermScorer extends Scor
       this(termScorer, termScorer.getDocsEnum(), termScorer.getDocFreq());
     }
     
-    DocsAndFreqs(MatchOnlyTermScorer termScorer) {
-      this(termScorer, termScorer.getDocsEnum(), termScorer.getDocFreq());
-    }
-    
     DocsAndFreqs(Scorer scorer, DocsEnum docs, int docFreq) {
       this.docs = docs;
       this.docFreq = docFreq;

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java Tue Jul 31 20:58:32 2012
@@ -364,7 +364,7 @@ class FieldCacheImpl implements FieldCac
               break;
             }
             final byte termval = parser.parseByte(term);
-            docs = termsEnum.docs(null, docs, false);
+            docs = termsEnum.docs(null, docs, 0);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -437,7 +437,7 @@ class FieldCacheImpl implements FieldCac
               break;
             }
             final short termval = parser.parseShort(term);
-            docs = termsEnum.docs(null, docs, false);
+            docs = termsEnum.docs(null, docs, 0);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -541,7 +541,7 @@ class FieldCacheImpl implements FieldCac
               retArray = new int[maxDoc];
             }
 
-            docs = termsEnum.docs(null, docs, false);
+            docs = termsEnum.docs(null, docs, 0);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -608,7 +608,7 @@ class FieldCacheImpl implements FieldCac
             res = new FixedBitSet(maxDoc);
           }
 
-          docs = termsEnum.docs(null, docs, false);
+          docs = termsEnum.docs(null, docs, 0);
           // TODO: use bulk API
           while (true) {
             final int docID = docs.nextDoc();
@@ -691,7 +691,7 @@ class FieldCacheImpl implements FieldCac
               retArray = new float[maxDoc];
             }
             
-            docs = termsEnum.docs(null, docs, false);
+            docs = termsEnum.docs(null, docs, 0);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -779,7 +779,7 @@ class FieldCacheImpl implements FieldCac
               retArray = new long[maxDoc];
             }
 
-            docs = termsEnum.docs(null, docs, false);
+            docs = termsEnum.docs(null, docs, 0);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -868,7 +868,7 @@ class FieldCacheImpl implements FieldCac
               retArray = new double[maxDoc];
             }
 
-            docs = termsEnum.docs(null, docs, false);
+            docs = termsEnum.docs(null, docs, 0);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -1042,12 +1042,12 @@ class FieldCacheImpl implements FieldCac
       }
 
       @Override
-      public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+      public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
         throw new UnsupportedOperationException();
       }
 
       @Override
-      public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
+      public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
         throw new UnsupportedOperationException();
       }
 
@@ -1167,7 +1167,7 @@ class FieldCacheImpl implements FieldCac
             termOrdToBytesOffset = termOrdToBytesOffset.resize(ArrayUtil.oversize(1+termOrd, 1));
           }
           termOrdToBytesOffset.set(termOrd, bytes.copyUsingLengthPrefix(term));
-          docs = termsEnum.docs(null, docs, false);
+          docs = termsEnum.docs(null, docs, 0);
           while (true) {
             final int docID = docs.nextDoc();
             if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -1288,7 +1288,7 @@ class FieldCacheImpl implements FieldCac
             break;
           }
           final long pointer = bytes.copyUsingLengthPrefix(term);
-          docs = termsEnum.docs(null, docs, false);
+          docs = termsEnum.docs(null, docs, 0);
           while (true) {
             final int docID = docs.nextDoc();
             if (docID == DocIdSetIterator.NO_MORE_DOCS) {

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java Tue Jul 31 20:58:32 2012
@@ -272,14 +272,14 @@ public class FuzzyTermsEnum extends Term
   }
   
   @Override
-  public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
-    return actualEnum.docs(liveDocs, reuse, needsFreqs);
+  public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
+    return actualEnum.docs(liveDocs, reuse, flags);
   }
   
   @Override
   public DocsAndPositionsEnum docsAndPositions(Bits liveDocs,
-                                               DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
-    return actualEnum.docsAndPositions(liveDocs, reuse, needsOffsets);
+                                               DocsAndPositionsEnum reuse, int flags) throws IOException {
+    return actualEnum.docsAndPositions(liveDocs, reuse, flags);
   }
   
   @Override

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java Tue Jul 31 20:58:32 2012
@@ -226,11 +226,11 @@ public class MultiPhraseQuery extends Qu
             return null;
           }
           termsEnum.seekExact(term.bytes(), termState);
-          postingsEnum = termsEnum.docsAndPositions(liveDocs, null, false);
+          postingsEnum = termsEnum.docsAndPositions(liveDocs, null, 0);
 
           if (postingsEnum == null) {
             // term does exist, but has no positions
-            assert termsEnum.docs(liveDocs, null, false) != null: "termstate found but no term exists in reader";
+            assert termsEnum.docs(liveDocs, null, 0) != null: "termstate found but no term exists in reader";
             throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.text() + ")");
           }
 
@@ -484,7 +484,7 @@ class UnionDocsAndPositionsEnum extends 
         continue;
       }
       termsEnum.seekExact(term.bytes(), termState);
-      DocsAndPositionsEnum postings = termsEnum.docsAndPositions(liveDocs, null, false);
+      DocsAndPositionsEnum postings = termsEnum.docsAndPositions(liveDocs, null, 0);
       if (postings == null) {
         // term does exist, but has no positions
         throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.text() + ")");

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Tue Jul 31 20:58:32 2012
@@ -106,7 +106,7 @@ public class MultiTermQueryWrapperFilter
       do {
         // System.out.println("  iter termCount=" + termCount + " term=" +
         // enumerator.term().toBytesString());
-        docsEnum = termsEnum.docs(acceptDocs, docsEnum, false);
+        docsEnum = termsEnum.docs(acceptDocs, docsEnum, 0);
         int docid;
         while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
           bitSet.set(docid);

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java Tue Jul 31 20:58:32 2012
@@ -264,7 +264,7 @@ public class PhraseQuery extends Query {
           return null;
         }
         te.seekExact(t.bytes(), state);
-        DocsAndPositionsEnum postingsEnum = te.docsAndPositions(liveDocs, null, false);
+        DocsAndPositionsEnum postingsEnum = te.docsAndPositions(liveDocs, null, 0);
 
         // PhraseQuery on a field that did not index
         // positions.

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/TermQuery.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/TermQuery.java Tue Jul 31 20:58:32 2012
@@ -83,15 +83,9 @@ public class TermQuery extends Query {
       if (termsEnum == null) {
         return null;
       }
-      DocsEnum docs = termsEnum.docs(acceptDocs, null, true);
-      if (docs != null) {
-        return new TermScorer(this, docs, similarity.exactSimScorer(stats, context), termsEnum.docFreq());
-      } else {
-        // Index does not store freq info
-        docs = termsEnum.docs(acceptDocs, null, false);
-        assert docs != null;
-        return new MatchOnlyTermScorer(this, docs, similarity.exactSimScorer(stats, context), termsEnum.docFreq());
-      }
+      DocsEnum docs = termsEnum.docs(acceptDocs, null);
+      assert docs != null;
+      return new TermScorer(this, docs, similarity.exactSimScorer(stats, context), termsEnum.docFreq());
     }
     
     /**

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java Tue Jul 31 20:58:32 2012
@@ -1,6 +1,5 @@
 package org.apache.lucene.search.payloads;
 
-import org.apache.lucene.search.Explanation;
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -36,14 +35,6 @@ public class AveragePayloadFunction exte
   public float docScore(int docId, String field, int numPayloadsSeen, float payloadScore) {
     return numPayloadsSeen > 0 ? (payloadScore / numPayloadsSeen) : 1;
   }
-  @Override
-  public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) {
-      Explanation payloadBoost = new Explanation();
-      float avgPayloadScore = (numPayloadsSeen > 0 ? (payloadScore / numPayloadsSeen) : 1);
-      payloadBoost.setValue(avgPayloadScore);
-      payloadBoost.setDescription("AveragePayloadFunction(...)");
-      return payloadBoost;
-  } 
 
   @Override
   public int hashCode() {

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java Tue Jul 31 20:58:32 2012
@@ -1,6 +1,5 @@
 package org.apache.lucene.search.payloads;
 
-import org.apache.lucene.search.Explanation;
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -41,14 +40,6 @@ public class MaxPayloadFunction extends 
   }
   
   @Override
-  public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) {
-	    Explanation expl = new Explanation();
-	    float maxPayloadScore = (numPayloadsSeen > 0 ? payloadScore : 1);
-	    expl.setValue(maxPayloadScore);
-	    expl.setDescription("MaxPayloadFunction(...)");
-	    return expl;
-	  } 
-  @Override
   public int hashCode() {
     final int prime = 31;
     int result = 1;

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java Tue Jul 31 20:58:32 2012
@@ -1,6 +1,5 @@
 package org.apache.lucene.search.payloads;
 
-import org.apache.lucene.search.Explanation;
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -39,14 +38,6 @@ public class MinPayloadFunction extends 
   }
   
   @Override
-  public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) {
-	  Explanation expl = new Explanation();
-	  float minPayloadScore = (numPayloadsSeen > 0 ? payloadScore : 1);
-	  expl.setValue(minPayloadScore);
-	  expl.setDescription("MinPayloadFunction(...)");
-	  return expl;
-  }  
-  @Override
   public int hashCode() {
     final int prime = 31;
     int result = 1;

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java Tue Jul 31 20:58:32 2012
@@ -55,10 +55,10 @@ public abstract class PayloadFunction {
    */
   public abstract float docScore(int docId, String field, int numPayloadsSeen, float payloadScore);
   
-  public Explanation explain(int docId, int numPayloadsSeen, float payloadScore){
+  public Explanation explain(int docId, String field, int numPayloadsSeen, float payloadScore){
 	  Explanation result = new Explanation();
-	  result.setDescription("Unimpl Payload Function Explain");
-	  result.setValue(1);
+	  result.setDescription(getClass().getSimpleName() + ".docScore()");
+	  result.setValue(docScore(docId, field, numPayloadsSeen, payloadScore));
 	  return result;
   };
   

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java Tue Jul 31 20:58:32 2012
@@ -167,8 +167,9 @@ public class PayloadNearQuery extends Sp
           Explanation scoreExplanation = docScorer.explain(doc, new Explanation(freq, "phraseFreq=" + freq));
           expl.addDetail(scoreExplanation);
           expl.setValue(scoreExplanation.getValue());
+          String field = ((SpanQuery)getQuery()).getField();
           // now the payloads part
-          Explanation payloadExpl = function.explain(doc, scorer.payloadsSeen, scorer.payloadScore);
+          Explanation payloadExpl = function.explain(doc, field, scorer.payloadsSeen, scorer.payloadScore);
           // combined
           ComplexExplanation result = new ComplexExplanation();
           result.addDetail(expl);



Mime
View raw message