lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jpou...@apache.org
Subject [1/3] lucene-solr:branch_6x: LUCENE-7317: Remove auto-prefix terms.
Date Mon, 20 Jun 2016 08:16:07 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/branch_6x d3fcac779 -> 5afdc1d72


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bac521d1/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java
b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java
index 340b0c9..a4a150b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java
@@ -25,13 +25,11 @@ import org.apache.lucene.codecs.BlockTermState;
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.FieldsConsumer;
 import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.blocktree.AutoPrefixTermsWriter.PrefixTerm;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
@@ -90,15 +88,6 @@ import org.apache.lucene.util.packed.PackedInts;
  * between two index terms.
  * <p>
  *
- * If {@code minItemsInAutoPrefix} is not zero, then for
- * {@link IndexOptions#DOCS} fields we detect prefixes that match
- * "enough" terms and insert auto-prefix terms into the index, which are
- * used by {@link Terms#intersect}  at search time to speed up prefix
- * and range queries.  Besides {@link Terms#intersect}, these
- * auto-prefix terms are invisible to all other APIs (don't change terms
- * stats, don't show up in normal {@link TermsEnum}s, etc.).
- * <p>
- *
  * Files:
  * <ul>
  *   <li><tt>.tim</tt>: <a href="#Termdictionary">Term Dictionary</a></li>
@@ -124,7 +113,7 @@ import org.apache.lucene.util.packed.PackedInts;
  * and decoding the Postings Metadata and Term Metadata sections.</p>
  *
  * <ul>
- *    <li>TermsDict (.tim) --&gt; Header, HasAutoPrefixTerms, <i>PostingsHeader</i>,
NodeBlock<sup>NumBlocks</sup>,
+ *    <li>TermsDict (.tim) --&gt; Header, <i>PostingsHeader</i>, NodeBlock<sup>NumBlocks</sup>,
  *                               FieldSummary, DirOffset, Footer</li>
  *    <li>NodeBlock --&gt; (OuterNode | InnerNode)</li>
  *    <li>OuterNode --&gt; EntryCount, SuffixLength, Byte<sup>SuffixLength</sup>,
StatsLength, &lt; TermStats &gt;<sup>EntryCount</sup>, MetaLength, &lt;<i>TermMetadata</i>&gt;<sup>EntryCount</sup></li>
@@ -145,7 +134,6 @@ import org.apache.lucene.util.packed.PackedInts;
  * <ul>
  *    <li>Header is a {@link CodecUtil#writeHeader CodecHeader} storing the version
information
  *        for the BlockTree implementation.</li>
- *    <li>HasAutoPrefixTerms is a single byte; 1 means there may be auto-prefix terms
and 0 means there are none.
  *    <li>DirOffset is a pointer to the FieldSummary section.</li>
  *    <li>DocFreq is the count of documents which contain the term.</li>
  *    <li>TotalTermFreq is the total number of occurrences of the term. This is encoded
@@ -223,8 +211,6 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
   final int maxDoc;
   final int minItemsInBlock;
   final int maxItemsInBlock;
-  final int minItemsInAutoPrefix;
-  final int maxItemsInAutoPrefix;
 
   final PostingsWriterBase postingsWriter;
   final FieldInfos fieldInfos;
@@ -260,43 +246,14 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
 
   private final List<FieldMetaData> fields = new ArrayList<>();
 
-  // private final String segment;
-  final FixedBitSet prefixDocs;
-
-  /** Reused in getAutoPrefixTermsEnum: */
-  final BitSetTermsEnum prefixFixedBitsTermsEnum;
-
-  /** Reused in getAutoPrefixTermsEnum: */
-  private TermsEnum prefixTermsEnum;
-
-  /** Reused in getAutoPrefixTermsEnum: */
-  private PostingsEnum prefixDocsEnum;
-
-  /** Create a new writer, using default values for auto-prefix terms. */
-  public BlockTreeTermsWriter(SegmentWriteState state,
-                              PostingsWriterBase postingsWriter,
-                              int minItemsInBlock,
-                              int maxItemsInBlock) throws IOException {
-    this(state, postingsWriter, minItemsInBlock, maxItemsInBlock, 0, 0);
-  }
-
   /** Create a new writer.  The number of items (terms or
    *  sub-blocks) per block will aim to be between
    *  minItemsPerBlock and maxItemsPerBlock, though in some
-   *  cases the blocks may be smaller than the min.
-   *  For DOCS_ONLY fields, this terms dictionary will
-   *  insert automatically generated prefix terms for common
-   *  prefixes, as long as each prefix matches at least
-   *  {@code minItemsInAutoPrefix} other terms or prefixes,
-   *  and at most {@code maxItemsInAutoPrefix} other terms
-   *  or prefixes.  Set {@code minItemsInAutoPrefix} to 0
-   *  to disable auto-prefix terms. */
+   *  cases the blocks may be smaller than the min. */
   public BlockTreeTermsWriter(SegmentWriteState state,
                               PostingsWriterBase postingsWriter,
                               int minItemsInBlock,
-                              int maxItemsInBlock,
-                              int minItemsInAutoPrefix,
-                              int maxItemsInAutoPrefix)
+                              int maxItemsInBlock)
     throws IOException
   {
     validateSettings(minItemsInBlock,
@@ -305,21 +262,6 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
     this.minItemsInBlock = minItemsInBlock;
     this.maxItemsInBlock = maxItemsInBlock;
 
-    validateAutoPrefixSettings(minItemsInAutoPrefix,
-                               maxItemsInAutoPrefix);
-
-    if (minItemsInAutoPrefix != 0) {
-      // TODO: can we used compressed bitset instead?  that auto-upgrades if it's dense enough...
-      prefixDocs = new FixedBitSet(state.segmentInfo.maxDoc());
-      prefixFixedBitsTermsEnum = new BitSetTermsEnum(prefixDocs);
-    } else {
-      prefixDocs = null;
-      prefixFixedBitsTermsEnum = null;
-    }
-
-    this.minItemsInAutoPrefix = minItemsInAutoPrefix;
-    this.maxItemsInAutoPrefix = maxItemsInAutoPrefix;
-
     this.maxDoc = state.segmentInfo.maxDoc();
     this.fieldInfos = state.fieldInfos;
     this.postingsWriter = postingsWriter;
@@ -332,13 +274,6 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
       CodecUtil.writeIndexHeader(termsOut, BlockTreeTermsReader.TERMS_CODEC_NAME, BlockTreeTermsReader.VERSION_CURRENT,
                                  state.segmentInfo.getId(), state.segmentSuffix);
 
-      // So at read time we know, globally, that there will be no auto-prefix terms:
-      if (minItemsInAutoPrefix == 0) {
-        termsOut.writeByte((byte) 0);
-      } else {
-        termsOut.writeByte((byte) 1);
-      }
-
       final String indexName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix,
BlockTreeTermsReader.TERMS_INDEX_EXTENSION);
       indexOut = state.directory.createOutput(indexName, state.context);
       CodecUtil.writeIndexHeader(indexOut, BlockTreeTermsReader.TERMS_INDEX_CODEC_NAME, BlockTreeTermsReader.VERSION_CURRENT,
@@ -380,25 +315,6 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
     }
   }
 
-  /** Throws {@code IllegalArgumentException} if any of these settings
-   *  is invalid. */
-  public static void validateAutoPrefixSettings(int minItemsInAutoPrefix,
-                                                int maxItemsInAutoPrefix) {
-    if (minItemsInAutoPrefix != 0) {
-      if (minItemsInAutoPrefix < 2) {
-        throw new IllegalArgumentException("minItemsInAutoPrefix must be at least 2; got
minItemsInAutoPrefix=" + minItemsInAutoPrefix);
-      }
-      if (minItemsInAutoPrefix > maxItemsInAutoPrefix) {
-        throw new IllegalArgumentException("maxItemsInAutoPrefix must be >= minItemsInAutoPrefix;
got maxItemsInAutoPrefix=" + maxItemsInAutoPrefix + " minItemsInAutoPrefix=" + minItemsInAutoPrefix);
-      }
-      if (2*(minItemsInAutoPrefix-1) > maxItemsInAutoPrefix) {
-        throw new IllegalArgumentException("maxItemsInAutoPrefix must be at least 2*(minItemsInAutoPrefix-1);
got maxItemsInAutoPrefix=" + maxItemsInAutoPrefix + " minItemsInAutoPrefix=" + minItemsInAutoPrefix);
-      }
-    } else if (maxItemsInAutoPrefix != 0) {
-      throw new IllegalArgumentException("maxItemsInAutoPrefix must be 0 (disabled) when
minItemsInAutoPrefix is 0");
-    }
-  }
-
   @Override
   public void write(Fields fields) throws IOException {
     //if (DEBUG) System.out.println("\nBTTW.write seg=" + segment);
@@ -413,75 +329,26 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
       if (terms == null) {
         continue;
       }
-      FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
-
-      // First pass to find all prefix terms we should compile into the index:
-      List<PrefixTerm> prefixTerms;
-      if (minItemsInAutoPrefix != 0) {
-        if (fieldInfo.getIndexOptions() != IndexOptions.DOCS) {
-          throw new IllegalStateException("ranges can only be indexed with IndexOptions.DOCS
(field: " + fieldInfo.name + ")");
-        }
-        prefixTerms = new AutoPrefixTermsWriter(terms, minItemsInAutoPrefix, maxItemsInAutoPrefix).prefixes;
-        //if (DEBUG) {
-        //  for(PrefixTerm term : prefixTerms) {
-        //    System.out.println("field=" + fieldInfo.name + " PREFIX TERM: " + term);
-        //  }
-        //}
-      } else {
-        prefixTerms = null;
-      }
 
       TermsEnum termsEnum = terms.iterator();
       TermsWriter termsWriter = new TermsWriter(fieldInfos.fieldInfo(field));
-      int prefixTermUpto = 0;
       while (true) {
         BytesRef term = termsEnum.next();
         //if (DEBUG) System.out.println("BTTW: next term " + term);
 
-        // Insert (merge sort) next prefix term(s):
-        if (prefixTerms != null) {
-          while (prefixTermUpto < prefixTerms.size() && (term == null || prefixTerms.get(prefixTermUpto).compareTo(term)
<= 0)) {
-            PrefixTerm prefixTerm = prefixTerms.get(prefixTermUpto);
-            //if (DEBUG) System.out.println("seg=" + segment + " field=" + fieldInfo.name
+ " NOW INSERT prefix=" + prefixTerm);
-            termsWriter.write(prefixTerm.term, getAutoPrefixTermsEnum(terms, prefixTerm),
prefixTerm);
-            prefixTermUpto++;
-          }
-        }
-
         if (term == null) {
           break;
         }
 
         //if (DEBUG) System.out.println("write field=" + fieldInfo.name + " term=" + brToString(term));
-        termsWriter.write(term, termsEnum, null);
+        termsWriter.write(term, termsEnum);
       }
 
-      assert prefixTerms == null || prefixTermUpto == prefixTerms.size();
-
       termsWriter.finish();
 
       //if (DEBUG) System.out.println("\nBTTW.write done seg=" + segment + " field=" + field);
     }
   }
-
-  private TermsEnum getAutoPrefixTermsEnum(Terms terms, final PrefixTerm prefix) throws IOException
{
-    assert prefixDocs != null;
-    prefixDocs.clear(0, prefixDocs.length());
-
-    prefixTermsEnum = prefix.getTermsEnum(terms.iterator());
-
-    //System.out.println("BTTW.getAutoPrefixTE: prefix=" + prefix);
-    while (prefixTermsEnum.next() != null) {
-      //System.out.println("    got term=" + prefixTermsEnum.term().utf8ToString());
-      //termCount++;
-      prefixDocsEnum = prefixTermsEnum.postings(prefixDocsEnum, 0);
-      //System.out.println("      " + prefixDocsEnum + " doc=" + prefixDocsEnum.docID());
-      prefixDocs.or(prefixDocsEnum);
-    }
-
-    //System.out.println("  done terms: " + prefixDocs.cardinality() + " doc seen; " + termCount
+ " terms seen");
-    return prefixFixedBitsTermsEnum;
-  }
   
   static long encodeOutput(long fp, boolean hasTerms, boolean isFloor) {
     assert fp < (1L << 62);
@@ -500,16 +367,12 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
     public final byte[] termBytes;
     // stats + metadata
     public final BlockTermState state;
-    // Non-null if this is an auto-prefix-term:
-    public final PrefixTerm prefixTerm;
-    public PendingTerm other;
 
-    public PendingTerm(BytesRef term, BlockTermState state, PrefixTerm prefixTerm) {
+    public PendingTerm(BytesRef term, BlockTermState state) {
       super(true);
       this.termBytes = new byte[term.length];
       System.arraycopy(term.bytes, term.offset, termBytes, 0, term.length);
       this.state = state;
-      this.prefixTerm = prefixTerm;
     }
 
     @Override
@@ -698,7 +561,6 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
       // only points to sub-blocks in the terms index so we can avoid seeking
       // to it when we are looking for a term):
       boolean hasTerms = false;
-      boolean hasPrefixTerms = false;
       boolean hasSubBlocks = false;
 
       int start = pending.size()-count;
@@ -739,11 +601,10 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
             // block as soon as we have at least minItemsInBlock.  This is not always best:
it often produces
             // a too-small block as the final block:
             boolean isFloor = itemsInBlock < count;
-            newBlocks.add(writeBlock(prefixLength, isFloor, nextFloorLeadLabel, nextBlockStart,
i, hasTerms, hasPrefixTerms, hasSubBlocks));
+            newBlocks.add(writeBlock(prefixLength, isFloor, nextFloorLeadLabel, nextBlockStart,
i, hasTerms, hasSubBlocks));
 
             hasTerms = false;
             hasSubBlocks = false;
-            hasPrefixTerms = false;
             nextFloorLeadLabel = suffixLeadLabel;
             nextBlockStart = i;
           }
@@ -753,7 +614,6 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
 
         if (ent.isTerm) {
           hasTerms = true;
-          hasPrefixTerms |= ((PendingTerm) ent).prefixTerm != null;
         } else {
           hasSubBlocks = true;
         }
@@ -763,7 +623,7 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
       if (nextBlockStart < end) {
         int itemsInBlock = end - nextBlockStart;
         boolean isFloor = itemsInBlock < count;
-        newBlocks.add(writeBlock(prefixLength, isFloor, nextFloorLeadLabel, nextBlockStart,
end, hasTerms, hasPrefixTerms, hasSubBlocks));
+        newBlocks.add(writeBlock(prefixLength, isFloor, nextFloorLeadLabel, nextBlockStart,
end, hasTerms, hasSubBlocks));
       }
 
       assert newBlocks.isEmpty() == false;
@@ -789,7 +649,7 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
      *  same prefix, and so we broke it into multiple floor blocks where
      *  we record the starting label of the suffix of each floor block. */
     private PendingBlock writeBlock(int prefixLength, boolean isFloor, int floorLeadLabel,
int start, int end,
-                                    boolean hasTerms, boolean hasPrefixTerms, boolean hasSubBlocks)
throws IOException {
+                                    boolean hasTerms, boolean hasSubBlocks) throws IOException
{
 
       assert end > start;
 
@@ -823,7 +683,7 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
 
       // We optimize the leaf block case (block has only terms), writing a more
       // compact format in this case:
-      boolean isLeafBlock = hasSubBlocks == false && hasPrefixTerms == false;
+      boolean isLeafBlock = hasSubBlocks == false;
 
       //System.out.println("  isLeaf=" + isLeafBlock);
 
@@ -839,7 +699,6 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
           assert ent.isTerm: "i=" + i;
 
           PendingTerm term = (PendingTerm) ent;
-          assert term.prefixTerm == null;
 
           assert StringHelper.startsWith(term.termBytes, prefix): "term.term=" + term.termBytes
+ " prefix=" + prefix;
           BlockTermState state = term.state;
@@ -876,7 +735,6 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
       } else {
         // Block has at least one prefix term or a sub block:
         subIndices = new ArrayList<>();
-        boolean sawAutoPrefixTerm = false;
         for (int i=start;i<end;i++) {
           PendingEntry ent = pending.get(i);
           if (ent.isTerm) {
@@ -890,9 +748,6 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
             //  System.arraycopy(term.termBytes, prefixLength, suffixBytes.bytes, 0, suffix);
             //  suffixBytes.length = suffix;
             //  System.out.println("      write term suffix=" + brToString(suffixBytes));
-            //  if (term.prefixTerm != null) {
-            //    System.out.println("        ** auto-prefix term: " + term.prefixTerm);
-            //  }
             //}
 
             // For non-leaf block we borrow 1 bit to record
@@ -900,33 +755,8 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
             // it's a prefix term.  Terms cannot be larger than ~32 KB
             // so we won't run out of bits:
 
-            if (minItemsInAutoPrefix == 0) {
-              suffixWriter.writeVInt(suffix << 1);
-              suffixWriter.writeBytes(term.termBytes, prefixLength, suffix);
-            } else {
-              code = suffix<<2;
-              int floorLeadEnd = -1;
-              if (term.prefixTerm != null) {
-                assert minItemsInAutoPrefix > 0;
-                sawAutoPrefixTerm = true;
-                PrefixTerm prefixTerm = term.prefixTerm;
-                floorLeadEnd = prefixTerm.floorLeadEnd;
-                assert floorLeadEnd != -1;
-
-                if (prefixTerm.floorLeadStart == -2) {
-                  // Starts with empty string
-                  code |= 2;
-                } else {
-                  code |= 3;
-                }
-              }
-              suffixWriter.writeVInt(code);
-              suffixWriter.writeBytes(term.termBytes, prefixLength, suffix);
-              if (floorLeadEnd != -1) {
-                suffixWriter.writeByte((byte) floorLeadEnd);
-              }
-              assert floorLeadLabel == -1 || (term.termBytes[prefixLength] & 0xff) >=
floorLeadLabel;
-            }
+            suffixWriter.writeVInt(suffix << 1);
+            suffixWriter.writeBytes(term.termBytes, prefixLength, suffix);
 
             // Write term stats, to separate byte[] blob:
             statsWriter.writeVInt(state.docFreq);
@@ -961,13 +791,8 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
             assert suffix > 0;
 
             // For non-leaf block we borrow 1 bit to record
-            // if entry is term or sub-block, and 1 bit (unset here) to
-            // record if it's a prefix term:
-            if (minItemsInAutoPrefix == 0) {
-              suffixWriter.writeVInt((suffix<<1)|1);
-            } else {
-              suffixWriter.writeVInt((suffix<<2)|1);
-            }
+            // if entry is term or sub-block:f
+            suffixWriter.writeVInt((suffix<<1)|1);
             suffixWriter.writeBytes(block.prefix.bytes, prefixLength, suffix);
 
             //if (DEBUG2) {
@@ -985,7 +810,7 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
           }
         }
 
-        assert subIndices.size() != 0 || sawAutoPrefixTerm;
+        assert subIndices.size() != 0;
       }
 
       // TODO: we could block-write the term suffix pointers;
@@ -1029,7 +854,7 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
     }
     
     /** Writes one term's worth of postings. */
-    public void write(BytesRef text, TermsEnum termsEnum, PrefixTerm prefixTerm) throws IOException
{
+    public void write(BytesRef text, TermsEnum termsEnum) throws IOException {
       /*
       if (DEBUG) {
         int[] tmp = new int[lastTerm.length];
@@ -1045,20 +870,17 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
         assert fieldInfo.getIndexOptions() == IndexOptions.DOCS || state.totalTermFreq >=
state.docFreq: "postingsWriter=" + postingsWriter;
         pushTerm(text);
        
-        PendingTerm term = new PendingTerm(text, state, prefixTerm);
+        PendingTerm term = new PendingTerm(text, state);
         pending.add(term);
         //if (DEBUG) System.out.println("    add pending term = " + text + " pending.size()="
+ pending.size());
 
-        if (prefixTerm == null) {
-          // Only increment stats for real terms:
-          sumDocFreq += state.docFreq;
-          sumTotalTermFreq += state.totalTermFreq;
-          numTerms++;
-          if (firstPendingTerm == null) {
-            firstPendingTerm = term;
-          }
-          lastPendingTerm = term;
+        sumDocFreq += state.docFreq;
+        sumTotalTermFreq += state.totalTermFreq;
+        numTerms++;
+        if (firstPendingTerm == null) {
+          firstPendingTerm = term;
         }
+        lastPendingTerm = term;
       }
     }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bac521d1/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java
b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java
index 4d5b4ce..3934c55 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java
@@ -348,7 +348,6 @@ final class IntersectTermsEnum extends TermsEnum {
   private boolean skipPastLastAutoPrefixTerm() throws IOException {
     assert currentFrame.isAutoPrefixTerm;
     useAutoPrefixTerm = false;
-    currentFrame.termState.isRealTerm = true;
 
     // If we last returned an auto-prefix term, we must now skip all
     // actual terms sharing that prefix.  At most, that skipping
@@ -694,7 +693,6 @@ final class IntersectTermsEnum extends TermsEnum {
           if (useAutoPrefixTerm) {
             // All suffixes of this auto-prefix term are accepted by the automaton, so we
can use it:
             copyTerm();
-            currentFrame.termState.isRealTerm = false;
             return term;
           } else {
             // We move onto the next term

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bac521d1/lucene/core/src/java/org/apache/lucene/index/TermContext.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/TermContext.java b/lucene/core/src/java/org/apache/lucene/index/TermContext.java
index ee5163e..e55aeba 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TermContext.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TermContext.java
@@ -177,18 +177,6 @@ public final class TermContext {
     return totalTermFreq;
   }
 
-  /** Returns true if all terms stored here are real (e.g., not auto-prefix terms).
-   *
-   *  @lucene.internal */
-  public boolean hasOnlyRealTerms() {
-    for (TermState termState : states) {
-      if (termState != null && termState.isRealTerm() == false) {
-        return false;
-      }
-    }
-    return true;
-  }
-
   @Override
   public String toString() {
     StringBuilder sb = new StringBuilder();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bac521d1/lucene/core/src/java/org/apache/lucene/index/TermState.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/TermState.java b/lucene/core/src/java/org/apache/lucene/index/TermState.java
index d8e9be4..7d3f133 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TermState.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TermState.java
@@ -48,12 +48,6 @@ public abstract class TermState implements Cloneable {
       // should not happen
       throw new RuntimeException(cnse);
     }
-  } 
-
-  /** Returns true if this term is real (e.g., not an auto-prefix term).
-   *  @lucene.internal */
-  public boolean isRealTerm() {
-    return true;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bac521d1/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java
b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java
index 6657757..ed4c430 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java
@@ -122,10 +122,6 @@ final class MultiTermQueryConstantScoreWrapper<Q extends MultiTermQuery>
extends
             return true;
           }
           TermState state = termsEnum.termState();
-          if (state.isRealTerm() == false) {
-            // TermQuery does not accept fake terms for now
-            return false;
-          }
           terms.add(new TermAndState(BytesRef.deepCopyOf(term), state, termsEnum.docFreq(),
termsEnum.totalTermFreq()));
         }
         return termsEnum.next() == null;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bac521d1/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java b/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
index 3a62e15..38345b3 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
@@ -115,7 +115,7 @@ public abstract class ScoringRewrite<B> extends TermCollectingRewrite<B>
{
       for (int i = 0; i < size; i++) {
         final int pos = sort[i];
         final Term term = new Term(query.getField(), col.terms.get(pos, new BytesRef()));
-        assert termStates[pos].hasOnlyRealTerms() == false || reader.docFreq(term) == termStates[pos].docFreq();
+        assert reader.docFreq(term) == termStates[pos].docFreq();
         addClause(builder, term, termStates[pos].docFreq(), boost[pos], termStates[pos]);
       }
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bac521d1/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
index e815ff6..6547b10 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
@@ -53,8 +53,6 @@ public class TermQuery extends Query {
       super(TermQuery.this);
       this.needsScores = needsScores;
       assert termStates != null : "TermContext must not be null";
-      // checked with a real exception in TermQuery constructor
-      assert termStates.hasOnlyRealTerms();
       this.termStates = termStates;
       this.similarity = searcher.getSimilarity(needsScores);
 
@@ -166,12 +164,6 @@ public class TermQuery extends Query {
   public TermQuery(Term t, TermContext states) {
     assert states != null;
     term = Objects.requireNonNull(t);
-    if (states.hasOnlyRealTerms() == false) {
-      // The reason for this is that fake terms might have the same bytes as
-      // real terms, and this confuses query caching because they don't match
-      // the same documents
-      throw new IllegalArgumentException("Term queries must be created on real terms");
-    }
     perReaderTermState = Objects.requireNonNull(states);
   }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bac521d1/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java
index 132dbce..ee6340d 100644
--- a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java
+++ b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java
@@ -345,10 +345,6 @@ public final class SolrRangeQuery extends ExtendedQueryBase implements
DocSetPro
           return -count - 1;
         }
         TermState state = termsEnum.termState();
-        if (state.isRealTerm() == false) {
-          // TermQuery does not accept fake terms for now
-          return count;
-        }
         int df = termsEnum.docFreq();
         count += df;
         terms.add(new TermAndState(BytesRef.deepCopyOf(term), state, df, termsEnum.totalTermFreq()));


Mime
View raw message