lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r904750 [3/4] - in /lucene/java/branches/flex_1458: ./ contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/ contrib/misc/src/java/org/apache/lucene/index/ contrib/queries/src/java/org/apache/lucene/search/ src/java/org/apache/luce...
Date Sat, 30 Jan 2010 10:18:25 GMT
Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java?rev=904750&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java Sat Jan 30 10:16:35 2010
@@ -0,0 +1,53 @@
+package org.apache.lucene.index.codecs.standard;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Closeable;
+
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Bits;
+
+/** StandardTermsDictReader interacts with a single instance
+ *  of this to manage creation of docs enum instances.  It
+ *  provides an IndexInput (termsIn) where this class may
+ *  read any previously stored data that it had written in
+ *  its corresponding StandardDocsConsumer at indexing
+ *  time. */
+public abstract class StandardPostingsReader implements Closeable {
+
+  public abstract void init(IndexInput termsIn) throws IOException;
+
+  /** Return a newly created empty TermState */
+  public abstract TermState newTermState() throws IOException;
+
+  public abstract void readTerm(IndexInput termsIn, FieldInfo fieldInfo, TermState state, boolean isIndexTerm) throws IOException;
+
+  /** Must fully consume state, since after this call that
+   *  TermState may be reused. */
+  public abstract DocsEnum docs(FieldInfo fieldInfo, TermState state, Bits skipDocs, DocsEnum reuse) throws IOException;
+
+  /** Must fully consume state, since after this call that
+   *  TermState may be reused. */
+  public abstract DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, TermState state, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException;
+
+  public abstract void close() throws IOException;
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java?rev=904750&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java Sat Jan 30 10:16:35 2010
@@ -0,0 +1,712 @@
+package org.apache.lucene.index.codecs.standard;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+/** Concrete class that reads the current doc/freq/skip
+ *  postings format. */
+
+// nocommit -- should we switch "hasProx" higher up?  and
+// create two separate docs readers, one that also reads
+// prox and one that doesn't?
+
+public class StandardPostingsReaderImpl extends StandardPostingsReader {
+
+  private final IndexInput freqIn;
+  private final IndexInput proxIn;
+
+  int skipInterval;
+  int maxSkipLevels;
+
+  public StandardPostingsReaderImpl(Directory dir, SegmentInfo segmentInfo, int readBufferSize) throws IOException {
+    freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, StandardCodec.FREQ_EXTENSION),
+                           readBufferSize);
+    if (segmentInfo.getHasProx()) {
+      boolean success = false;
+      try {
+        proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, StandardCodec.PROX_EXTENSION),
+                               readBufferSize);
+        success = true;
+      } finally {
+        if (!success) {
+          freqIn.close();
+        }
+      }
+    } else {
+      proxIn = null;
+    }
+  }
+
+  public static void files(Directory dir, SegmentInfo segmentInfo, Collection<String> files) throws IOException {
+    files.add(IndexFileNames.segmentFileName(segmentInfo.name, StandardCodec.FREQ_EXTENSION));
+    if (segmentInfo.getHasProx()) {
+      files.add(IndexFileNames.segmentFileName(segmentInfo.name, StandardCodec.PROX_EXTENSION));
+    }
+  }
+
+  @Override
+  public void init(IndexInput termsIn) throws IOException {
+
+    // Make sure we are talking to the matching past writer
+    Codec.checkHeader(termsIn, StandardPostingsWriterImpl.CODEC, StandardPostingsWriterImpl.VERSION_START);
+
+    skipInterval = termsIn.readInt();
+    maxSkipLevels = termsIn.readInt();
+  }
+
+  private static class DocTermState extends TermState {
+    long freqOffset;
+    long proxOffset;
+    int skipOffset;
+
+    public Object clone() {
+      DocTermState other = (DocTermState) super.clone();
+      other.freqOffset = freqOffset;
+      other.proxOffset = proxOffset;
+      other.skipOffset = skipOffset;
+      return other;
+    }
+
+    public void copy(TermState _other) {
+      super.copy(_other);
+      DocTermState other = (DocTermState) _other;
+      freqOffset = other.freqOffset;
+      proxOffset = other.proxOffset;
+      skipOffset = other.skipOffset;
+    }
+
+    public String toString() {
+      return super.toString() + " freqFP=" + freqOffset + " proxFP=" + proxOffset + " skipOffset=" + skipOffset;
+    }
+  }
+
+  @Override
+  public TermState newTermState() {
+    return new DocTermState();
+  }
+
+  @Override
+  public void close() throws IOException {
+    try {
+      if (freqIn != null) {
+        freqIn.close();
+      }
+    } finally {
+      if (proxIn != null) {
+        proxIn.close();
+      }
+    }
+  }
+
+  @Override
+  public void readTerm(IndexInput termsIn, FieldInfo fieldInfo, TermState termState, boolean isIndexTerm)
+    throws IOException {
+
+    final DocTermState docTermState = (DocTermState) termState;
+
+    if (Codec.DEBUG) {
+      Codec.debug("  sdr.readTerm tis.fp=" + termsIn.getFilePointer() + " df=" + termState.docFreq + " isIndex?=" + isIndexTerm + " tis=" + termsIn);
+    }
+
+    if (isIndexTerm) {
+      docTermState.freqOffset = termsIn.readVLong();
+    } else {
+      docTermState.freqOffset += termsIn.readVLong();
+    }
+
+    if (Codec.DEBUG) {
+      Codec.debug("    frq.fp=" + docTermState.freqOffset + " vs len=" + freqIn.length());
+    }
+
+    if (docTermState.docFreq >= skipInterval) {
+      docTermState.skipOffset = termsIn.readVInt();
+    } else {
+      docTermState.skipOffset = 0;
+    }
+
+    if (!fieldInfo.omitTermFreqAndPositions) {
+      if (isIndexTerm) {
+        docTermState.proxOffset = termsIn.readVLong();
+      } else {
+        docTermState.proxOffset += termsIn.readVLong();
+      }
+    }
+  }
+    
+  @Override
+  public DocsEnum docs(FieldInfo fieldInfo, TermState termState, Bits skipDocs, DocsEnum reuse) throws IOException {
+    final SegmentDocsEnum docsEnum;
+    if (reuse == null) {
+      docsEnum = new SegmentDocsEnum(freqIn);
+    } else {
+      docsEnum = (SegmentDocsEnum) reuse;
+    }
+    return docsEnum.reset(fieldInfo, (DocTermState) termState, skipDocs);
+  }
+
+  @Override
+  public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, TermState termState, Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException {
+    if (fieldInfo.omitTermFreqAndPositions) {
+      return null;
+    }
+    final SegmentDocsAndPositionsEnum docsEnum;
+    if (reuse == null) {
+      docsEnum = new SegmentDocsAndPositionsEnum(freqIn, proxIn);
+    } else {
+      docsEnum = (SegmentDocsAndPositionsEnum) reuse;
+    }
+    return docsEnum.reset(fieldInfo, (DocTermState) termState, skipDocs);
+  }
+
+  // Decodes only docs
+  private class SegmentDocsEnum extends DocsEnum {
+    final IndexInput freqIn;
+
+    boolean omitTF;                               // does current field omit term freq?
+    boolean storePayloads;                        // does current field store payloads?
+
+    int limit;                                    // number of docs in this posting
+    int ord;                                      // how many docs we've read
+    int doc;                                      // doc we last read
+    int freq;                                     // freq we last read
+
+    Bits skipDocs;
+
+    long freqOffset;
+    int skipOffset;
+
+    boolean skipped;
+    DefaultSkipListReader skipper;
+
+    public SegmentDocsEnum(IndexInput freqIn) throws IOException {
+      if (Codec.DEBUG) {
+        System.out.println("new docs enum");
+      }
+      this.freqIn = (IndexInput) freqIn.clone();
+    }
+
+    public SegmentDocsEnum reset(FieldInfo fieldInfo, DocTermState termState, Bits skipDocs) throws IOException {
+      if (Codec.DEBUG) {
+        System.out.println("[" + desc + "] dr.reset freqIn seek " + termState.freqOffset + " docCount=" + termState.docFreq);
+      }
+      omitTF = fieldInfo.omitTermFreqAndPositions;
+      if (omitTF) {
+        freq = 1;
+      }
+      storePayloads = fieldInfo.storePayloads;
+      this.skipDocs = skipDocs;
+      freqOffset = termState.freqOffset;
+      skipOffset = termState.skipOffset;
+
+      // nocommit this seek frequently isn't needed, when
+      // we enum terms and all docs for each term (MTQ,
+      // or, merging).  is this seek costing us anything?
+      // we should avoid it so...
+      freqIn.seek(termState.freqOffset);
+      limit = termState.docFreq;
+      ord = 0;
+      doc = 0;
+
+      skipped = false;
+
+      return this;
+    }
+
+    @Override
+    public int nextDoc() throws IOException {
+      if (Codec.DEBUG) {
+        Codec.debug("sdr.next [" + desc + "] ord=" + ord + " vs df=" + limit + " freq.fp=" + freqIn.getFilePointer() + " + has skip docs=" + (skipDocs != null));
+      }
+
+      while(true) {
+        if (ord == limit) {
+          return doc = NO_MORE_DOCS;
+        }
+
+        ord++;
+
+        // Decode next doc/freq pair
+        final int code = freqIn.readVInt();
+        if (Codec.DEBUG) {
+          System.out.println("  read code=" + code);
+        }
+        if (omitTF) {
+          doc += code;
+        } else {
+          doc += code >>> 1;              // shift off low bit
+          if ((code & 1) != 0) {          // if low bit is set
+            freq = 1;                     // freq is one
+          } else {
+            freq = freqIn.readVInt();     // else read freq
+          }
+        }
+
+        if (skipDocs == null || !skipDocs.get(doc)) {
+          break;
+        } else if (Codec.DEBUG) {
+          System.out.println("  doc=" + doc + " is skipped");
+        }
+      }
+
+      if (Codec.DEBUG) {
+        System.out.println("  result doc=" + doc + " freq=" + freq);
+      }
+
+      return doc;
+    }
+
+    @Override
+    public int read(int[] docs, int[] freqs) throws IOException {
+      if (Codec.DEBUG) {
+        Codec.debug("sdr.bulk read: ord=" + ord + " df=" + limit + " omitTF=" + omitTF + " ord=" + ord + " of " + limit + " freq.fp=" + freqIn.getFilePointer(), desc);
+      }
+      int i = 0;
+      final int length = docs.length;
+      while (i < length && ord < limit) {
+        ord++;
+        // manually inlined call to next() for speed
+        final int code = freqIn.readVInt();
+        if (omitTF) {
+          doc += code;
+        } else {
+          doc += code >>> 1;              // shift off low bit
+          if ((code & 1) != 0) {          // if low bit is set
+            freq = 1;                     // freq is one
+          } else {
+            freq = freqIn.readVInt();     // else read freq
+          }
+        }
+
+        if (skipDocs == null || !skipDocs.get(doc)) {
+          if (Codec.DEBUG) {
+            Codec.debug("  " + i + ": doc=" + doc + " freq=" + freq, desc);
+          }
+          docs[i] = doc;
+          freqs[i] = freq;
+          ++i;
+        }
+      }
+      if (Codec.DEBUG) {
+        System.out.println("  return " + i);
+      }
+
+      return i;
+    }
+
+    @Override
+    public int docID() {
+      return doc;
+    }
+
+    @Override
+    public int freq() {
+      return freq;
+    }
+
+    @Override
+    public int advance(int target) throws IOException {
+
+      // TODO: jump right to next() if target is < X away
+      // from where we are now?
+
+      if (Codec.DEBUG) {
+        System.out.println("dr [" + desc + "]: skip to target=" + target);
+      }
+
+      if (skipOffset > 0) {
+
+        // There are enough docs in the posting to have
+        // skip data
+
+        if (skipper == null) {
+          // This is the first time this enum has ever been used for skipping -- do lazy init
+          skipper = new DefaultSkipListReader((IndexInput) freqIn.clone(), maxSkipLevels, skipInterval);
+        }
+
+        if (!skipped) {
+
+          // This is the first time this posting has
+          // skipped since reset() was called, so now we
+          // load the skip data for this posting
+
+          skipper.init(freqOffset + skipOffset,
+                       freqOffset, 0,
+                       limit, storePayloads);
+
+          if (Codec.DEBUG) {
+            System.out.println("    skipper init  skipFP=" + (freqOffset+skipOffset) + " freqFP=" + freqOffset);
+          }
+
+          skipped = true;
+        }
+
+        final int newOrd = skipper.skipTo(target); 
+
+        if (newOrd > ord) {
+          // Skipper moved
+
+          ord = newOrd;
+          doc = skipper.getDoc();
+          freqIn.seek(skipper.getFreqPointer());
+
+          if (Codec.DEBUG) {
+            System.out.println("dr [" + desc + "]: skipper moved to newOrd=" + newOrd + " freqFP=" + skipper.getFreqPointer() + " doc=" + doc + "; now scan...");
+          }
+
+        } else if (Codec.DEBUG) {
+          System.out.println("  no skipping to be done");
+        }
+      } else if (Codec.DEBUG) {
+        System.out.println("  no skip data (#docs is too low)");
+      }
+        
+      // scan for the rest:
+      do {
+        nextDoc();
+      } while (target > doc);
+
+      return doc;
+    }
+  }
+
+  // Decodes docs & positions
+  private class SegmentDocsAndPositionsEnum extends DocsAndPositionsEnum {
+    private final IndexInput freqIn;
+    private final IndexInput proxIn;
+
+    boolean storePayloads;                        // does current field store payloads?
+
+    int limit;                                    // number of docs in this posting
+    int ord;                                      // how many docs we've read
+    int doc;                                      // doc we last read
+    int freq;                                     // freq we last read
+    int position;
+
+    Bits skipDocs;
+
+    long freqOffset;
+    int skipOffset;
+    long proxOffset;
+
+    int posPendingCount;
+    int payloadLength;
+    boolean payloadPending;
+
+    boolean skipped;
+    DefaultSkipListReader skipper;
+    private BytesRef payload;
+    private long lazyProxPointer;
+
+    public SegmentDocsAndPositionsEnum(IndexInput freqIn, IndexInput proxIn) throws IOException {
+      if (Codec.DEBUG) {
+        System.out.println("new docs enum");
+      }
+      this.freqIn = (IndexInput) freqIn.clone();
+      this.proxIn = (IndexInput) proxIn.clone();
+    }
+
+    public SegmentDocsAndPositionsEnum reset(FieldInfo fieldInfo, DocTermState termState, Bits skipDocs) throws IOException {
+      if (Codec.DEBUG) {
+        System.out.println("[" + desc + "] dr.init freqIn seek freq.fp=" + termState.freqOffset + " prox.fp=" + termState.proxOffset + " docCount=" + termState.docFreq);
+      }
+      assert !fieldInfo.omitTermFreqAndPositions;
+      storePayloads = fieldInfo.storePayloads;
+      if (storePayloads && payload == null) {
+        payload = new BytesRef();
+        payload.bytes = new byte[1];
+      }
+
+      this.skipDocs = skipDocs;
+
+      // nocommit this seek frequently isn't needed, when
+      // we enum terms and all docs for each term (MTQ,
+      // or, merging).  is this seek costing us anything?
+      // we should avoid it so...
+      freqIn.seek(termState.freqOffset);
+      lazyProxPointer = termState.proxOffset;
+
+      limit = termState.docFreq;
+      ord = 0;
+      doc = 0;
+      position = 0;
+
+      skipped = false;
+      posPendingCount = 0;
+      payloadPending = false;
+
+      freqOffset = termState.freqOffset;
+      proxOffset = termState.proxOffset;
+      skipOffset = termState.skipOffset;
+
+      return this;
+    }
+
+    @Override
+    public int nextDoc() throws IOException {
+      if (Codec.DEBUG) {
+        Codec.debug("sdr.next [" + desc + "] ord=" + ord + " vs df=" + limit + " freq.fp=" + freqIn.getFilePointer() + " + has skip docs=" + (skipDocs != null));
+      }
+
+      while(true) {
+        if (ord == limit) {
+          return doc = NO_MORE_DOCS;
+        }
+
+        ord++;
+
+        // Decode next doc/freq pair
+        final int code = freqIn.readVInt();
+        if (Codec.DEBUG) {
+          System.out.println("  read code=" + code);
+        }
+        doc += code >>> 1;              // shift off low bit
+        if ((code & 1) != 0) {          // if low bit is set
+          freq = 1;                     // freq is one
+        } else {
+          freq = freqIn.readVInt();     // else read freq
+        }
+        posPendingCount += freq;
+
+        if (skipDocs == null || !skipDocs.get(doc)) {
+          break;
+        } else if (Codec.DEBUG) {
+          System.out.println("  doc=" + doc + " is skipped");
+        }
+      }
+
+      if (Codec.DEBUG) {
+        System.out.println("  result doc=" + doc + " freq=" + freq);
+      }
+      position = 0;
+
+      return doc;
+    }
+
+    @Override
+    public int docID() {
+      return doc;
+    }
+
+    @Override
+    public int freq() {
+      return freq;
+    }
+
+    @Override
+    public int advance(int target) throws IOException {
+
+      // TODO: jump right to next() if target is < X away
+      // from where we are now?
+
+      if (Codec.DEBUG) {
+        System.out.println("dr [" + desc + "]: skip to target=" + target);
+      }
+
+      if (skipOffset > 0) {
+
+        // There are enough docs in the posting to have
+        // skip data
+
+        if (skipper == null) {
+          // This is the first time this enum has ever been used for skipping -- do lazy init
+          skipper = new DefaultSkipListReader((IndexInput) freqIn.clone(), maxSkipLevels, skipInterval);
+        }
+
+        if (!skipped) {
+
+          // This is the first time this posting has
+          // skipped, since reset() was called, so now we
+          // load the skip data for this posting
+
+          skipper.init(freqOffset+skipOffset,
+                       freqOffset, proxOffset,
+                       limit, storePayloads);
+
+          if (Codec.DEBUG) {
+            Codec.debug("    skip reader base freqFP=" + (freqOffset+skipOffset) + " freqFP=" + freqOffset + " prox.fp=" + proxOffset);
+          }
+
+          skipped = true;
+        }
+
+        final int newOrd = skipper.skipTo(target); 
+
+        if (newOrd > ord) {
+          // Skipper moved
+          ord = newOrd;
+          doc = skipper.getDoc();
+          freqIn.seek(skipper.getFreqPointer());
+          lazyProxPointer = skipper.getProxPointer();
+          posPendingCount = 0;
+          position = 0;
+          payloadPending = false;
+          payloadLength = skipper.getPayloadLength();
+
+          if (Codec.DEBUG) {
+            Codec.debug("dr [" + desc + "]: skipper moved to newOrd=" + newOrd + " freq.fp=" + skipper.getFreqPointer() + " prox.fp=" + skipper.getProxPointer() + " doc=" + doc);
+          }
+
+        } else if (Codec.DEBUG) {
+          System.out.println("  no skipping to be done");
+        }
+      } else if (Codec.DEBUG) {
+        System.out.println("  no skip data (#docs is too low)");
+      }
+        
+      // Now, linear scan for the rest:
+      do {
+        nextDoc();
+      } while (target > doc);
+
+      return doc;
+    }
+
+    public int nextPosition() throws IOException {
+
+      if (lazyProxPointer != -1) {
+        proxIn.seek(lazyProxPointer);
+        lazyProxPointer = -1;
+      }
+      
+      if (Codec.DEBUG) {
+        System.out.println("nextPos [" + desc + "] payloadPending=" + payloadPending + " payloadLen=" + payloadLength + " posPendingCount=" + posPendingCount + " freq=" + freq);
+      }
+
+      if (payloadPending && payloadLength > 0) {
+        // payload of last position as never retrieved -- skip it
+        if (Codec.DEBUG) {
+          System.out.println("      skip payload len=" + payloadLength);
+        }
+        proxIn.seek(proxIn.getFilePointer() + payloadLength);
+        payloadPending = false;
+      }
+
+      // scan over any docs that were iterated without their positions
+      while(posPendingCount > freq) {
+
+        if (Codec.DEBUG) {
+          System.out.println("      skip position");
+        }
+
+        final int code = proxIn.readVInt();
+
+        if (storePayloads) {
+          if ((code & 1) != 0) {
+            // new payload length
+            payloadLength = proxIn.readVInt();
+            assert payloadLength >= 0;
+            if (Codec.DEBUG) {
+              System.out.println("        new payloadLen=" + payloadLength);
+            }
+          }
+          assert payloadLength != -1;
+          proxIn.seek(proxIn.getFilePointer() + payloadLength);
+          if (Codec.DEBUG) {
+            System.out.println("        skip payloadLen=" + payloadLength + " bytes");
+          }
+        }
+
+        posPendingCount--;
+        position = 0;
+        payloadPending = false;
+      }
+
+      // read next position
+      if (storePayloads) {
+
+        if (payloadPending && payloadLength > 0) {
+          // payload wasn't retrieved for last position
+          if (Codec.DEBUG) {
+            System.out.println("      payload pending: skip " + payloadLength + " bytes");
+          }
+          proxIn.seek(proxIn.getFilePointer()+payloadLength);
+        }
+
+        final int code = proxIn.readVInt();
+        if ((code & 1) != 0) {
+          // new payload length
+          payloadLength = proxIn.readVInt();
+          assert payloadLength >= 0;
+          if (Codec.DEBUG) {
+            System.out.println("      new payloadLen=" + payloadLength);
+          }
+        }
+        assert payloadLength != -1;
+          
+        payloadPending = true;
+        position += code >>> 1;
+      } else {
+        position += proxIn.readVInt();
+      }
+
+      posPendingCount--;
+
+      assert posPendingCount >= 0: "nextPosition() was called too many times (more than freq() times) posPendingCount=" + posPendingCount;
+
+      if (Codec.DEBUG) {
+        System.out.println("   proxFP=" + proxIn.getFilePointer() + " return pos=" + position);
+      }
+      return position;
+    }
+
+    /** Returns length of payload at current position */
+    public int getPayloadLength() {
+      assert lazyProxPointer == -1;
+      assert posPendingCount < freq;
+      return payloadLength;
+    }
+
+    /** Returns the payload at this position, or null if no
+     *  payload was indexed. */
+    public BytesRef getPayload() throws IOException {
+      assert lazyProxPointer == -1;
+      assert posPendingCount < freq;
+      if (Codec.DEBUG) {
+        System.out.println("      read payload: " + payloadLength);
+      }
+      if (!payloadPending) {
+        throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once.");
+      }
+      if (payloadLength > payload.bytes.length) {
+        payload.grow(payloadLength);
+      }
+      proxIn.readBytes(payload.bytes, 0, payloadLength);
+      payload.length = payloadLength;
+      payloadPending = false;
+
+      return payload;
+    }
+
+    public boolean hasPayload() {
+      return payloadPending && payloadLength > 0;
+    }
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java?rev=904750&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java Sat Jan 30 10:16:35 2010
@@ -0,0 +1,43 @@
+package org.apache.lucene.index.codecs.standard;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Closeable;
+
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.codecs.PostingsConsumer;
+
+/**
+ * NOTE: this API is experimental and will likely change
+ */
+
+public abstract class StandardPostingsWriter extends PostingsConsumer implements Closeable {
+
+  public abstract void start(IndexOutput termsOut) throws IOException;
+
+  public abstract void startTerm() throws IOException;
+
+  /** Finishes the current term */
+  public abstract void finishTerm(int numDocs, boolean isIndexTerm) throws IOException;
+
+  public abstract void setField(FieldInfo fieldInfo);
+
+  public abstract void close() throws IOException;
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriterImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriterImpl.java?rev=904750&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriterImpl.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriterImpl.java Sat Jan 30 10:16:35 2010
@@ -0,0 +1,267 @@
+package org.apache.lucene.index.codecs.standard;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Consumes doc & freq, writing them using the current
+ *  index file format */
+
+import java.io.IOException;
+
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.util.BytesRef;
+
+public final class StandardPostingsWriterImpl extends StandardPostingsWriter {
+  final static String CODEC = "StandardPostingsWriterImpl";
+  
+  // Increment version to change it:
+  final static int VERSION_START = 0;
+  final static int VERSION_CURRENT = VERSION_START;
+
+  final IndexOutput freqOut;
+  final IndexOutput proxOut;
+  final DefaultSkipListWriter skipListWriter;
+  final int skipInterval;
+  final int maxSkipLevels;
+  final int totalNumDocs;
+  IndexOutput termsOut;
+
+  boolean omitTermFreqAndPositions;
+  boolean storePayloads;
+  // Starts a new term
+  long lastFreqStart;
+  long freqStart;
+  long lastProxStart;
+  long proxStart;
+  FieldInfo fieldInfo;
+  int lastPayloadLength;
+  int lastPosition;
+
+  public StandardPostingsWriterImpl(SegmentWriteState state) throws IOException {
+    super();
+    String fileName = IndexFileNames.segmentFileName(state.segmentName, StandardCodec.FREQ_EXTENSION);
+    state.flushedFiles.add(fileName);
+    freqOut = state.directory.createOutput(fileName);
+
+    if (state.fieldInfos.hasProx()) {
+      // At least one field does not omit TF, so create the
+      // prox file
+      fileName = IndexFileNames.segmentFileName(state.segmentName, StandardCodec.PROX_EXTENSION);
+      state.flushedFiles.add(fileName);
+      proxOut = state.directory.createOutput(fileName);
+    } else {
+      // Every field omits TF so we will write no prox file
+      proxOut = null;
+    }
+
+    totalNumDocs = state.numDocs;
+
+    skipListWriter = new DefaultSkipListWriter(state.skipInterval,
+                                               state.maxSkipLevels,
+                                               state.numDocs,
+                                               freqOut,
+                                               proxOut);
+     
+    skipInterval = state.skipInterval;
+    maxSkipLevels = state.maxSkipLevels;
+  }
+
+  @Override
+  public void start(IndexOutput termsOut) throws IOException {
+    this.termsOut = termsOut;
+    Codec.writeHeader(termsOut, CODEC, VERSION_CURRENT);
+    termsOut.writeInt(skipInterval);                // write skipInterval
+    termsOut.writeInt(maxSkipLevels);               // write maxSkipLevels
+  }
+
+  @Override
+  public void startTerm() {
+    freqStart = freqOut.getFilePointer();
+    if (proxOut != null) {
+      proxStart = proxOut.getFilePointer();
+      // force first payload to write its length
+      lastPayloadLength = -1;
+    }
+    skipListWriter.resetSkip();
+  }
+
+  // nocommit -- should we NOT reuse across fields?  would
+  // be cleaner
+
+  // Currently, this instance is re-used across fields, so
+  // our parent calls setField whenever the field changes
+  @Override
+  public void setField(FieldInfo fieldInfo) {
+    this.fieldInfo = fieldInfo;
+    omitTermFreqAndPositions = fieldInfo.omitTermFreqAndPositions;
+    storePayloads = fieldInfo.storePayloads;
+  }
+
+  int lastDocID;
+  int df;
+  
+  int count;
+
+  /** Adds a new doc in this term.  If this returns null
+   *  then we just skip consuming positions/payloads. */
+  @Override
+  public void addDoc(int docID, int termDocFreq) throws IOException {
+
+    final int delta = docID - lastDocID;
+    
+    if (Codec.DEBUG) {
+      Codec.debug("  addDoc [" + desc + "] count=" + (count++) + " docID=" + docID + " lastDocID=" + lastDocID + " delta=" + delta + " omitTF=" + omitTermFreqAndPositions + " freq=" + termDocFreq + " freq.fp=" + freqOut.getFilePointer());
+    }
+
+    if (docID < 0 || (df > 0 && delta <= 0)) {
+      throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " )");
+    }
+
+    if ((++df % skipInterval) == 0) {
+      skipListWriter.setSkipData(lastDocID, storePayloads, lastPayloadLength);
+      skipListWriter.bufferSkip(df);
+      if (Codec.DEBUG) {
+        System.out.println("    bufferSkip lastDocID=" + lastDocID + " df=" + df + " freqFP=" + freqOut.getFilePointer() + " proxFP=" + skipListWriter.proxOutput.getFilePointer());
+      }
+    }
+
+    assert docID < totalNumDocs: "docID=" + docID + " totalNumDocs=" + totalNumDocs;
+
+    lastDocID = docID;
+    if (omitTermFreqAndPositions) {
+      freqOut.writeVInt(delta);
+    } else if (1 == termDocFreq) {
+      freqOut.writeVInt((delta<<1) | 1);
+    } else {
+      freqOut.writeVInt(delta<<1);
+      freqOut.writeVInt(termDocFreq);
+    }
+
+    lastPosition = 0;
+  }
+
+  /** Add a new position & payload */
+  @Override
+  public void addPosition(int position, BytesRef payload) throws IOException {
+    assert !omitTermFreqAndPositions: "omitTermFreqAndPositions is true";
+    assert proxOut != null;
+
+    if (Codec.DEBUG) {
+      if (payload != null) {
+        Codec.debug("    addPos [" + desc + "]: pos=" + position + " prox.fp=" + proxOut.getFilePointer() + " payload=" + payload.length + " bytes");
+      } else {
+        Codec.debug("    addPos [" + desc + "]: pos=" + position + " prox.fp=" + proxOut.getFilePointer());
+      }
+    }
+    
+    final int delta = position - lastPosition;
+    
+    assert delta > 0 || position == 0 || position == -1: "position=" + position + " lastPosition=" + lastPosition;            // not quite right (if pos=0 is repeated twice we don't catch it)
+
+    lastPosition = position;
+
+    if (storePayloads) {
+      if (Codec.DEBUG) {
+        System.out.println("  store payloads");
+      }
+      final int payloadLength = payload == null ? 0 : payload.length;
+
+      if (payloadLength != lastPayloadLength) {
+        if (Codec.DEBUG) {
+          System.out.println("  payload len change old=" + lastPayloadLength + " new=" + payloadLength);
+        }
+        lastPayloadLength = payloadLength;
+        proxOut.writeVInt((delta<<1)|1);
+        proxOut.writeVInt(payloadLength);
+      } else {
+        proxOut.writeVInt(delta << 1);
+      }
+
+      if (payloadLength > 0) {
+        proxOut.writeBytes(payload.bytes, payload.offset, payloadLength);
+      }
+    } else {
+      proxOut.writeVInt(delta);
+    }
+  }
+
+  @Override
+  public void finishDoc() {
+  }
+
+  /** Called when we are done adding docs to this term */
+  @Override
+  public void finishTerm(int docCount, boolean isIndexTerm) throws IOException {
+    // nocommit -- wasteful we are counting this in two places?
+    assert docCount == df;
+    // mxx
+    if (Codec.DEBUG) {
+      Codec.debug("dw.finishTerm termsOut.fp=" + termsOut.getFilePointer() + " freqStart=" + freqStart + " df=" + df + " isIndex?=" + isIndexTerm);
+    }
+
+    if (isIndexTerm) {
+      // Write absolute at seek points
+      termsOut.writeVLong(freqStart);
+    } else {
+      // Write delta between seek points
+      termsOut.writeVLong(freqStart - lastFreqStart);
+    }
+
+    lastFreqStart = freqStart;
+
+    if (df >= skipInterval) {
+      // mxx
+      if (Codec.DEBUG) {
+        System.out.println(Thread.currentThread().getName() + ":  writeSkip @ freqFP=" + freqOut.getFilePointer() + " freqStartFP=" + freqStart);
+      }
+      termsOut.writeVInt((int) (skipListWriter.writeSkip(freqOut)-freqStart));
+    }
+     
+    if (!omitTermFreqAndPositions) {
+      if (isIndexTerm) {
+        // Write absolute at seek points
+        termsOut.writeVLong(proxStart);
+      } else {
+        // Write delta between seek points
+        termsOut.writeVLong(proxStart - lastProxStart);
+      }
+      lastProxStart = proxStart;
+    }
+
+    lastDocID = 0;
+    df = 0;
+    
+    // nocommit
+    count = 0;
+  }
+
+  @Override
+  public void close() throws IOException {
+    try {
+      freqOut.close();
+    } finally {
+      if (proxOut != null) {
+        proxOut.close();
+      }
+    }
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriterImpl.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java Sat Jan 30 10:16:35 2010
@@ -18,11 +18,13 @@
  */
 
 import java.io.IOException;
+import java.io.Closeable;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.TreeMap;
 
 import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.FieldsEnum;
@@ -32,50 +34,80 @@
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.FieldsProducer;
-import org.apache.lucene.index.codecs.pulsing.PulsingDocsWriter.Document;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.CloseableThreadLocal;
 import org.apache.lucene.util.cache.Cache;
 import org.apache.lucene.util.cache.DoubleBarrelLRUCache;
 import org.apache.lucene.util.BytesRef;
 
-/** Handles a terms dict, but defers all details of postings
- *  reading to an instance of {@TermsDictDocsReader}. This
- *  terms dict codec is meant to be shared between
- *  different postings codecs, but, it's certainly possible
- *  to make a codec that has its own terms dict writer/reader. */
+/** Handles a terms dict, but decouples all details of
+ *  doc/freqs/positions reading to an instance of {@link
+ *  StandardPostingsReader}.  This class is reusable for
+ *  codecs that use a different format for
+ *  docs/freqs/positions (though codecs are also free to
+ *  make their own terms dict impl).
+ *
+ * <p>This class also interacts with an instance of {@link
+ * StandardTermsIndexReader}, to abstract away the specific
+ * implementation of the terms dict index. */
 
 public class StandardTermsDictReader extends FieldsProducer {
+  // Open input to the main terms dict file (_X.tis)
   private final IndexInput in;
 
-  private final StandardDocsProducer docs;
+  // Reads the terms dict entries, to gather state to
+  // produce DocsEnum on demand
+  private final StandardPostingsReader postingsReader;
 
-  final TreeMap<String,FieldReader> fields = new TreeMap<String,FieldReader>();
+  private final TreeMap<String,FieldReader> fields = new TreeMap<String,FieldReader>();
 
   private final String segment;
-  private StandardTermsIndexReader indexReader;
 
+  // Comparator that orders our terms
   private final BytesRef.Comparator termComp;
+
+  // Caches the most recently looked-up Terms:
+  private final Cache<FieldAndTerm,TermState> termsCache;
+
+  // Reads the terms index
+  private StandardTermsIndexReader indexReader;
+
+  // Used as key for the terms cache
+  private static class FieldAndTerm {
+    String field;
+    BytesRef term;
+
+    public FieldAndTerm() {
+    }
+
+    public FieldAndTerm(FieldAndTerm other) {
+      field = other.field;
+      term = new BytesRef(other.term);
+    }
+
+    public boolean equals(Object _other) {
+      FieldAndTerm other = (FieldAndTerm) _other;
+      return other.field == field && term.bytesEquals(other.term);
+    }
+
+    public int hashCode() {
+      return field.hashCode() * 31 + term.hashCode();
+    }
+  }
   
-  public StandardTermsDictReader(StandardTermsIndexReader indexReader, Directory dir, FieldInfos fieldInfos, String segment, StandardDocsProducer docs, int readBufferSize,
-                                 BytesRef.Comparator termComp)
+  public StandardTermsDictReader(StandardTermsIndexReader indexReader, Directory dir, FieldInfos fieldInfos, String segment, StandardPostingsReader postingsReader, int readBufferSize,
+                                 BytesRef.Comparator termComp, int termsCacheSize)
     throws IOException {
     
     this.segment = segment;
-    this.docs = docs;
+    this.postingsReader = postingsReader;
+    termsCache = new DoubleBarrelLRUCache<FieldAndTerm,TermState>(termsCacheSize);
 
     this.termComp = termComp;
     
-    String file = IndexFileNames.segmentFileName(segment, StandardCodec.TERMS_EXTENSION);
-    //nocommit
-    if(!dir.fileExists(file)) {
-      in = null;
-      return;
-    }
-    in = dir.openInput(file, readBufferSize);
-
+    in = dir.openInput(IndexFileNames.segmentFileName(segment, StandardCodec.TERMS_EXTENSION),
+                       readBufferSize);
 
     boolean success = false;
     try {
@@ -83,16 +115,14 @@
 
       final long dirOffset = in.readLong();
 
-
-      // Have DocsProducer init itself
-      docs.start(in);
+      // Have PostingsReader init itself
+      postingsReader.init(in);
 
       // Read per-field details
       in.seek(dirOffset);
 
       final int numFields = in.readInt();
 
-      // mxx
       if (Codec.DEBUG) {
         System.out.println(Thread.currentThread().getName() + ": stdr create seg=" + segment + " numFields=" + numFields + " hasProx?=" + fieldInfos.hasProx());
       }
@@ -106,11 +136,7 @@
         if (Codec.DEBUG) {
           System.out.println("  stdr: load field=" + fieldInfo.name + " numTerms=" + numTerms);
         }
-        if (indexReader != null) {
-          fieldIndexReader = indexReader.getField(fieldInfo);
-        } else {
-          fieldIndexReader = null;
-        }
+        fieldIndexReader = indexReader.getField(fieldInfo);
         if (numTerms > 0) {
           assert !fields.containsKey(fieldInfo.name);
           fields.put(fieldInfo.name, new FieldReader(fieldIndexReader, fieldInfo, numTerms, termsStartPointer));
@@ -128,9 +154,6 @@
 
   @Override
   public void loadTermsIndex(int indexDivisor) throws IOException {
-    // nocommit -- must handle case where segment has become
-    // a CFS since we originall opened; maybe Directory
-    // should be passed in?
     indexReader.loadTermsIndex(indexDivisor);
   }
 
@@ -142,7 +165,9 @@
           indexReader.close();
         }
       } finally {
-        // null so if an app hangs on to us we still free most ram
+        // null so if an app hangs on to us (ie, we are not
+        // GCable, despite being closed) we still free most
+        // ram
         indexReader = null;
         if (in != null) {
           in.close();
@@ -150,8 +175,8 @@
       }
     } finally {
       try {
-        if (docs != null) {
-          docs.close();
+        if (postingsReader != null) {
+          postingsReader.close();
         }
       } finally {
         for(FieldReader field : fields.values()) {
@@ -182,7 +207,7 @@
     return fields.get(field);
   }
 
-  // Iterates through all known fields
+  // Iterates through all fields
   private class TermFieldsEnum extends FieldsEnum {
     final Iterator<FieldReader> it;
     FieldReader current;
@@ -214,16 +239,12 @@
       return current.iterator();
     }
   }
-  
-  private class FieldReader extends Terms {
-    private final CloseableThreadLocal<ThreadResources> threadResources = new CloseableThreadLocal<ThreadResources>();
+
+  private class FieldReader extends Terms implements Closeable {
     final long numTerms;
     final FieldInfo fieldInfo;
     final long termsStartPointer;
     final StandardTermsIndexReader.FieldReader indexReader;
-    private final static int DEFAULT_CACHE_SIZE = 1024;
-    // Used for caching the least recently looked-up Terms
-    private final Cache<BytesRef,CacheEntry> termsCache = new DoubleBarrelLRUCache<BytesRef,CacheEntry>(DEFAULT_CACHE_SIZE);
 
     FieldReader(StandardTermsIndexReader.FieldReader fieldIndexReader, FieldInfo fieldInfo, long numTerms, long termsStartPointer) {
       assert numTerms > 0;
@@ -234,50 +255,12 @@
     }
 
     @Override
-    public int docFreq(BytesRef text) throws IOException {
-      ThreadResources resources = getThreadResources();
-      if (resources.termsEnum.seek(text) == TermsEnum.SeekStatus.FOUND) {
-        return resources.termsEnum.docFreq();
-      } else {
-        return 0;
-      }
-    }
-
-    @Override
     public BytesRef.Comparator getComparator() {
       return termComp;
     }
 
-    // nocommit -- figure out how to do this one: we want to
-    // reuse the thread private TermsEnum, but, get a
-    // clone'd docs, somehow.  This way if code is using the
-    // API sequentially, we match performance of current
-    // trunk (though, really, such code ought to get their
-    // own terms enum and use its seek...)
-    /*
-    @Override
-    public DocsEnum docs(Bits skipDocs, BytesRef text) throws IOException {
-      ThreadResources resources = getThreadResources();
-      if (resources.termsEnum.seek(text) == TermsEnum.SeekStatus.FOUND) {
-        return resources.termsEnum.docs(skipDocs);
-      } else {
-        return null;
-      }
-    }
-    */
-    
     public void close() {
-      threadResources.close();
-    }
-    
-    protected ThreadResources getThreadResources() throws IOException {
-      ThreadResources resources = (ThreadResources) threadResources.get();
-      if (resources == null) {
-        // Cache does not have to be thread-safe, it is only used by one thread at the same time
-        resources = new ThreadResources(new SegmentTermsEnum());
-        threadResources.set(resources);
-      }
-      return resources;
+      super.close();
     }
     
     @Override
@@ -294,12 +277,11 @@
     private class SegmentTermsEnum extends TermsEnum {
       private final IndexInput in;
       private final DeltaBytesReader bytesReader;
-      // nocommit: long?
-      private int termUpto = -1;
-      private final StandardDocsProducer.Reader docs;
-      private int docFreq;
+      private final TermState state;
+      private boolean seekPending;
       private final StandardTermsIndexReader.TermsIndexResult indexResult = new StandardTermsIndexReader.TermsIndexResult();
-      
+      private final FieldAndTerm fieldTerm = new FieldAndTerm();
+
       SegmentTermsEnum() throws IOException {
         if (Codec.DEBUG) {
           System.out.println("tdr " + this + ": CREATE TermsEnum field=" + fieldInfo.name + " startPos=" + termsStartPointer + " seg=" + segment);
@@ -307,10 +289,9 @@
         in = (IndexInput) StandardTermsDictReader.this.in.clone();
         in.seek(termsStartPointer);
         bytesReader = new DeltaBytesReader(in);
-        if (Codec.DEBUG) {
-          System.out.println("  bytesReader=" + bytesReader);
-        }
-        docs = StandardTermsDictReader.this.docs.reader(fieldInfo, in);
+        fieldTerm.field = fieldInfo.name;
+        state = postingsReader.newTermState();
+        state.ord = -1;
       }
 
       @Override
@@ -325,178 +306,158 @@
       @Override
       public SeekStatus seek(BytesRef term) throws IOException {
 
-        CacheEntry entry = null;
-        BytesRef entryKey = null;
+        if (Codec.DEBUG) {
+          Codec.debug("stdr.seek(text=" + fieldInfo.name + ":" + term + ") seg=" + segment);
+          new Throwable().printStackTrace(System.out);
+        }
 
-        // Consult terms cache first:
-        if (docs.canCaptureState()) {
-          entry = termsCache.get(term);
-          if (entry != null) {
-            docFreq = entry.freq;
-            bytesReader.term.copy(term);
-            docs.setState(entry, docFreq);
-            termUpto = entry.termUpTo;
-            // nocommit -- would be better to do this lazy?
-            in.seek(entry.filePointer);
-            return SeekStatus.FOUND;
+        // Check cache
+        fieldTerm.term = term;
+        TermState cachedState = termsCache.get(fieldTerm);
+        
+        if (cachedState != null) {
+
+          state.copy(cachedState);
+
+          seekPending = true;
+          bytesReader.term.copy(term);
+
+          if (Codec.DEBUG) {
+            Codec.debug("  cache hit!  term=" + bytesReader.term + " " + cachedState);
           }
+
+          return SeekStatus.FOUND;
         }
-        
-        // mxx
+
         if (Codec.DEBUG) {
-          System.out.println(Thread.currentThread().getName() + ":stdr.seek(text=" + fieldInfo.name + ":" + term + ") seg=" + segment);
+          Codec.debug("  cache miss!");
         }
 
         boolean doSeek = true;
 
-        if (termUpto != -1 && termUpto < numTerms) {
+        if (state.ord != -1) {
+          // we are positioned
 
-          // See if we are already positioned at the requested term
           final int cmp = termComp.compare(bytesReader.term, term);
 
           if (cmp == 0) {
-            // nocommit -- should we really bother special
-            // casing this?  how often does it really
-            // happen?
-            // We are already on the requested term
-            // nocommit -- not right if text is ""?
+            // already at the requested term
             if (Codec.DEBUG) {
-              System.out.println(Thread.currentThread().getName() + ":  already here!");
+              Codec.debug("  already here!");
             }
 
-            // nocommit -- cache this?
             return SeekStatus.FOUND;
           }
 
-          if (cmp < 0) {
-
-            // Requested term is after our current term --
-            // read next index term:
-            if (indexReader.nextIndexTerm(termUpto, indexResult)) {
-              final int cmpNext = termComp.compare(indexResult.term, term);
-
-              if (cmpNext > 0) {
-                // Requested term is within the same index
-                // block we are in; skip seeking
-                doSeek = false;
-              }
-            }
+          if (cmp < 0 &&
+              indexReader.nextIndexTerm(state.ord, indexResult) &&
+              termComp.compare(indexResult.term, term) > 0) {
+            // Optimization: requested term is within the
+            // same index block we are now in; skip seeking
+            // (but do scanning):
+            doSeek = false;
           }
         }
 
         if (doSeek) {
-          // nocommit -- also, not sure it'll help, but, we
-          // can bound this binary search, since we know the
-          // term ord we are now on, and we can compare how
-          // this new term compars to our current term
 
-          // Find latest index term that's <= our text:
+          // As index to find biggest index term that's <=
+          // our text:
           indexReader.getIndexOffset(term, indexResult);
 
-          // mxx
           if (Codec.DEBUG) {
-            System.out.println(Thread.currentThread().getName() + ": index pos=" + indexResult.position + " termFP=" + indexResult.offset + " term=" + indexResult.term + " this=" + this);
+            Codec.debug(" index pos=" + indexResult.position + " termFP=" + indexResult.offset + " term=" + indexResult.term + " this=" + this);
           }
 
           in.seek(indexResult.offset);
+          seekPending = false;
 
           // NOTE: the first next() after an index seek is
           // wasteful, since it redundantly reads the same
-          // bytes into the buffer
+          // bytes into the buffer.  We could avoid storing
+          // those bytes in the primary file, but then when
+          // scanning over an index term we'd have to
+          // special case it:
           bytesReader.reset(indexResult.term);
 
-          termUpto = (int) indexResult.position-1;
-          assert termUpto >= -1: "termUpto=" + termUpto;
+          state.ord = (int) indexResult.position-1;
+          assert state.ord >= -1: "ord=" + state.ord;
 
-          // mxx
           if (Codec.DEBUG) {
-            System.out.println(Thread.currentThread().getName() + ":  set termUpto=" + termUpto);
+            Codec.debug("  set ord=" + state.ord);
           }
-
         } else if (Codec.DEBUG) {
-          System.out.println(Thread.currentThread().getName() + ": use scanning only (no seek)");
+          Codec.debug(": use scanning only (no seek)");
         }
 
-        // Now, scan:
-
-        //int indexCount = 0;
-        //int lastIndexCount = 0;
-        //int scanCnt = 0;
+        // Now scan:
         while(next() != null) {
-          //scanCnt++;
           final int cmp = termComp.compare(bytesReader.term, term);
           if (cmp == 0) {
-            // mxx
             if (Codec.DEBUG) {
-              System.out.println(Thread.currentThread().getName() + ":  seek done found term=" + bytesReader.term);
-              //new Throwable().printStackTrace(System.out);
+              Codec.debug("  seek done found term=" + bytesReader.term);
             }
 
-            // nocommit -- see how often an already
-            // NOT_FOUND is then sent back here?  silly for
-            // apps to do so... but we should see if Lucene
-            // does 
-            // nocommit -- maybe we sometimes want to cache,
-            // with doSeek?
-            if (docs.canCaptureState() && doSeek) {
+            if (doSeek) {
               // Store in cache
-              entry = docs.captureState();
-              entryKey = (BytesRef) bytesReader.term.clone();
-              entry.freq = docFreq;
-              entry.termUpTo = termUpto;
-              entry.filePointer = in.getFilePointer();
-            
-              termsCache.put(entryKey, entry);
+              FieldAndTerm entryKey = new FieldAndTerm(fieldTerm);
+              cachedState = (TermState) state.clone();
+              // this is fp after current term
+              cachedState.filePointer = in.getFilePointer();
+              termsCache.put(entryKey, cachedState);
+              if (Codec.DEBUG) {
+                Codec.debug("  save to cache  term=" + fieldTerm.term + " " + cachedState);
+              }
             }
+            if (Codec.DEBUG) {
+              Codec.debug("  found term=" + fieldTerm.term);
+            }
+              
             return SeekStatus.FOUND;
           } else if (cmp > 0) {
-            // mxx
             if (Codec.DEBUG) {
-              System.out.println(Thread.currentThread().getName() + ":  seek done did not find term=" + term + " found instead: " + bytesReader.term);
+              Codec.debug("  seek done did not find term=" + term + " found instead: " + bytesReader.term);
             }
             return SeekStatus.NOT_FOUND;
           }
 
-          // We should not cross another indexed term while
-          // scanning:
-
-          // nocommit -- not correct that we call
-          // isIndexTerm, twice
-          //indexCount += indexReader.isIndexTerm(termUpto, docFreq) ? 1:0;
-          //assert lastIndexCount < indexDivisor: " indexCount=" + lastIndexCount + " indexDivisor=" + indexDivisor;
-          //lastIndexCount = indexCount;
-
-          // mxx
-          //System.out.println(Thread.currentThread().getName() + ":  cycle");
+          // nocommit -- put back assert that we don't cross
+          // another index term while scanning, here
         }
 
-        // mxx
         if (Codec.DEBUG) {
-          System.out.println(Thread.currentThread().getName() + ": seek done did not find term=" + term + ": hit EOF");
+          Codec.debug("  seek done did not find term=" + term + ": hit EOF");
         }
+
         return SeekStatus.END;
       }
 
       @Override
-      public SeekStatus seek(long pos) throws IOException {
-        if (pos >= numTerms) {
+      public SeekStatus seek(long ord) throws IOException {
+
+        // TODO: should we cache term lookup by ord as well...?
+
+        if (ord >= numTerms) {
+          state.ord = numTerms-1;
           return SeekStatus.END;
         }
-        indexReader.getIndexOffset(pos, indexResult);
+
+        indexReader.getIndexOffset(ord, indexResult);
         in.seek(indexResult.offset);
+        seekPending = false;
 
         // NOTE: the first next() after an index seek is
         // wasteful, since it redundantly reads the same
         // bytes into the buffer
         bytesReader.reset(indexResult.term);
 
-        termUpto = (int) indexResult.position-1;
-        assert termUpto >= -1: "termUpto=" + termUpto;
+        state.ord = indexResult.position-1;
+        assert state.ord >= -1: "ord=" + state.ord;
 
         // Now, scan:
-        int left = (int) (pos - termUpto);
+        int left = (int) (ord - state.ord);
         while(left > 0) {
-          BytesRef term = next();
+          final BytesRef term = next();
           assert term != null;
           left--;
         }
@@ -512,81 +473,92 @@
 
       @Override
       public long ord() {
-        return termUpto;
+        return state.ord;
       }
 
       @Override
       public BytesRef next() throws IOException {
-        if (termUpto >= numTerms-1) {
-          return null;
-        }
+
         if (Codec.DEBUG) {
-          System.out.println("tdr.next: field=" + fieldInfo.name + " termsInPointer=" + in.getFilePointer() + " vs len=" + in.length() + " seg=" + segment);
-          //new Throwable().printStackTrace(System.out);
+          Codec.debug("tdr.next: field=" + fieldInfo.name + " tis.fp=" + in.getFilePointer() + " vs len=" + in.length() + " seg=" + segment);
+        }
+
+        if (seekPending) {
+          if (Codec.DEBUG) {
+            Codec.debug("  do pending seek " + state);
+          }
+          seekPending = false;
+          in.seek(state.filePointer);
         }
+        
+        if (state.ord >= numTerms-1) {
+          return null;
+        }
+
         bytesReader.read();
-        docFreq = in.readVInt();
+        state.docFreq = in.readVInt();
+
         if (Codec.DEBUG) {
-          System.out.println("  text=" + bytesReader.term + " freq=" + docFreq);
+          Codec.debug("  text=" + bytesReader.term + " freq=" + state.docFreq + " tis=" + in);
         }
+
         // TODO: would be cleaner, but space-wasting, to
         // simply record a bit into each index entry as to
-        // whether it's an index entry or not... or,
-        // possibly store a "how many terms until next index
-        // entry" in each index entry, but that'd require
-        // some tricky lookahead work when writing the index
-        final boolean isIndex = indexReader.isIndexTerm(1+termUpto, docFreq);
+        // whether it's an index entry or not, rather than
+        // re-compute that information... or, possibly store
+        // a "how many terms until next index entry" in each
+        // index entry, but that'd require some tricky
+        // lookahead work when writing the index
+        postingsReader.readTerm(in,
+                                fieldInfo, state,
+                                indexReader.isIndexTerm(1+state.ord, state.docFreq));
 
-        // mxx
-        // System.out.println(Thread.currentThread().getName() + ": isIndex=" + isIndex);
+        state.ord++;
 
-        docs.readTerm(docFreq, isIndex);
-        termUpto++;
         if (Codec.DEBUG) {
-          System.out.println("  termUpto=" + termUpto + " vs numTerms=" + numTerms + " fp=" + in.getFilePointer());
+          Codec.debug("  ord=" + state.ord + " vs numTerms=" + numTerms + " fp=" + in.getFilePointer());
         }
+
         return bytesReader.term;
       }
 
       @Override
       public int docFreq() {
-        return docFreq;
+        return state.docFreq;
       }
 
       @Override
-      public DocsEnum docs(Bits skipDocs) throws IOException {
-        // nocommit
+      public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
         if (Codec.DEBUG) {
           System.out.println("stdr.docs");
         }
-        DocsEnum docsEnum = docs.docs(skipDocs);
+        DocsEnum docsEnum = postingsReader.docs(fieldInfo, state, skipDocs, reuse);
         if (Codec.DEBUG) {
-          docsEnum.desc = fieldInfo.name + ":" + bytesReader.term;
+          docsEnum.desc = fieldInfo.name + ":" + bytesReader.term.toString();
         }
         return docsEnum;
       }
-    }
-  }
-
-  // nocommit -- scrutinize API
-  public static class CacheEntry {
-    int termUpTo;                                 // ord for this term
-    long filePointer;                             // fp into the terms dict primary file (_X.tis)
-
-    // nocommit -- belongs in Pulsing's CacheEntry class:
-    public int freq;
-    public Document docs[];
-    public boolean pendingIndexTerm;
-  }
-  
-  /**
-   * Per-thread resources managed by ThreadLocal
-   */
-  private static final class ThreadResources {
-    final TermsEnum termsEnum;
 
-    ThreadResources(TermsEnum termsEnum) {
-      this.termsEnum = termsEnum;
+      @Override
+      public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException {
+        if (Codec.DEBUG) {
+          System.out.println("stdr.docsAndPositions omitTF=" + fieldInfo.omitTermFreqAndPositions);
+        }
+        if (fieldInfo.omitTermFreqAndPositions) {
+          return null;
+        } else {
+          DocsAndPositionsEnum postingsEnum = postingsReader.docsAndPositions(fieldInfo, state, skipDocs, reuse);
+          if (Codec.DEBUG) {
+            if (postingsEnum != null) {
+              postingsEnum.desc = fieldInfo.name + ":" + bytesReader.term.toString();
+            }
+          }
+          if (Codec.DEBUG) {
+            Codec.debug("  return enum=" + postingsEnum);
+          }
+          return postingsEnum;
+        }
+      }
     }
   }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java Sat Jan 30 10:16:35 2010
@@ -28,7 +28,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.FieldsConsumer;
-import org.apache.lucene.index.codecs.DocsConsumer;
+import org.apache.lucene.index.codecs.PostingsConsumer;
 import org.apache.lucene.index.codecs.TermsConsumer;
 import org.apache.lucene.store.IndexOutput;
 
@@ -54,7 +54,7 @@
   private final DeltaBytesWriter termWriter;
 
   final IndexOutput out;
-  final StandardDocsConsumer consumer;
+  final StandardPostingsWriter postingsWriter;
   final FieldInfos fieldInfos;
   FieldInfo currentField;
   private final StandardTermsIndexWriter indexWriter;
@@ -64,7 +64,7 @@
   // nocommit
   private String segment;
 
-  public StandardTermsDictWriter(StandardTermsIndexWriter indexWriter, SegmentWriteState state, StandardDocsConsumer consumer, BytesRef.Comparator termComp) throws IOException {
+  public StandardTermsDictWriter(StandardTermsIndexWriter indexWriter, SegmentWriteState state, StandardPostingsWriter postingsWriter, BytesRef.Comparator termComp) throws IOException {
     final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, StandardCodec.TERMS_EXTENSION);
     this.indexWriter = indexWriter;
     this.termComp = termComp;
@@ -86,9 +86,9 @@
 
     termWriter = new DeltaBytesWriter(out);
     currentField = null;
-    this.consumer = consumer;
+    this.postingsWriter = postingsWriter;
 
-    consumer.start(out);                          // have consumer write its format/header
+    postingsWriter.start(out);                          // have consumer write its format/header
   }
 
   @Override
@@ -99,7 +99,7 @@
     assert currentField == null || currentField.name.compareTo(field.name) < 0;
     currentField = field;
     StandardTermsIndexWriter.FieldWriter fieldIndexWriter = indexWriter.addField(field);
-    TermsConsumer terms = new TermsWriter(fieldIndexWriter, field, consumer);
+    TermsConsumer terms = new TermsWriter(fieldIndexWriter, field, postingsWriter);
     fields.add(terms);
     return terms;
   }
@@ -134,7 +134,7 @@
         out.close();
       } finally {
         try {
-          consumer.close();
+          postingsWriter.close();
         } finally {
           indexWriter.close();
         }
@@ -146,20 +146,20 @@
 
   class TermsWriter extends TermsConsumer {
     final FieldInfo fieldInfo;
-    final StandardDocsConsumer consumer;
+    final StandardPostingsWriter postingsWriter;
     final long termsStartPointer;
     int numTerms;
     final StandardTermsIndexWriter.FieldWriter fieldIndexWriter;
 
-    TermsWriter(StandardTermsIndexWriter.FieldWriter fieldIndexWriter, FieldInfo fieldInfo, StandardDocsConsumer consumer) {
+    TermsWriter(StandardTermsIndexWriter.FieldWriter fieldIndexWriter, FieldInfo fieldInfo, StandardPostingsWriter postingsWriter) {
       this.fieldInfo = fieldInfo;
-      this.consumer = consumer;
       this.fieldIndexWriter = fieldIndexWriter;
 
       termWriter.reset();
       termsStartPointer = out.getFilePointer();
-      consumer.setField(fieldInfo);
+      postingsWriter.setField(fieldInfo);
       lastIndexPointer = termsStartPointer;
+      this.postingsWriter = postingsWriter;
 
       if (Codec.DEBUG) {
         System.out.println("stdw: now write field=" + fieldInfo.name);
@@ -172,13 +172,13 @@
     }
 
     @Override
-    public DocsConsumer startTerm(BytesRef text) throws IOException {
-      consumer.startTerm();
+    public PostingsConsumer startTerm(BytesRef text) throws IOException {
+      postingsWriter.startTerm();
       if (Codec.DEBUG) {
-        consumer.desc = fieldInfo.name + ":" + text;
-        System.out.println("stdw.startTerm term=" + fieldInfo.name + ":" + text + " seg=" + segment);
+        postingsWriter.desc = fieldInfo.name + ":" + text.toBytesString();
+        System.out.println("stdw.startTerm term=" + fieldInfo.name + ":" + text.toBytesString() + " seg=" + segment);
       }
-      return consumer;
+      return postingsWriter;
     }
 
     @Override
@@ -186,8 +186,8 @@
 
       // mxx
       if (Codec.DEBUG) {
-        // nocommit
-        System.out.println(Thread.currentThread().getName() + ": stdw.finishTerm seg=" + segment + " text=" + fieldInfo.name + ":" + text + " numDocs=" + numDocs + " numTerms=" + numTerms);
+        // nocommit     
+        Codec.debug("finishTerm seg=" + segment + " text=" + fieldInfo.name + ":" + text.toBytesString() + " numDocs=" + numDocs + " numTerms=" + numTerms);
       }
 
       if (numDocs > 0) {
@@ -195,13 +195,13 @@
 
         // mxx
         if (Codec.DEBUG) {
-          System.out.println(Thread.currentThread().getName() + ":  filePointer=" + out.getFilePointer() + " isIndexTerm?=" + isIndexTerm);
+          Codec.debug("  tis.fp=" + out.getFilePointer() + " isIndexTerm?=" + isIndexTerm);
           System.out.println("  term bytes=" + text.toBytesString());
         }
         termWriter.write(text);
         out.writeVInt(numDocs);
 
-        consumer.finishTerm(numDocs, isIndexTerm);
+        postingsWriter.finishTerm(numDocs, isIndexTerm);
         numTerms++;
       }
     }

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/TermState.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/TermState.java?rev=904750&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/TermState.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/TermState.java Sat Jan 30 10:16:35 2010
@@ -0,0 +1,51 @@
+package org.apache.lucene.index.codecs.standard;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Holds all state required for {@link StandardDocsProducer}
+ * to produce a {@link DocsEnum} without re-seeking the
+ * terms dict.
+ */
+
+public class TermState implements Cloneable {
+  public long ord;                                     // ord for this term
+  public long filePointer;                             // fp into the terms dict primary file (_X.tis)
+  public int docFreq;                                  // how many docs have this term
+
+  public void copy(TermState other) {
+    ord = other.ord;
+    filePointer = other.filePointer;
+    docFreq = other.docFreq;
+  }
+
+  @Override
+  public Object clone() {
+    try {
+      return super.clone();
+    } catch (CloneNotSupportedException cnse) {
+      // should not happen
+      throw new RuntimeException(cnse);
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "tis.fp=" + filePointer + " docFreq=" + docFreq + " ord=" + ord;
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/TermState.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/ExactPhraseScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/ExactPhraseScorer.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/ExactPhraseScorer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/ExactPhraseScorer.java Sat Jan 30 10:16:35 2010
@@ -22,9 +22,9 @@
 
 final class ExactPhraseScorer extends PhraseScorer {
 
-  ExactPhraseScorer(Weight weight, DocsEnum[] docs, int[] offsets,
+  ExactPhraseScorer(Weight weight, DocsAndPositionsEnum[] postings, int[] offsets,
       Similarity similarity, byte[] norms) {
-    super(weight, docs, offsets, similarity, norms);
+    super(weight, postings, offsets, similarity, norms);
   }
 
   @Override

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java Sat Jan 30 10:16:35 2010
@@ -284,6 +284,7 @@
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
         final Bits delDocs = reader.getDeletedDocs();
+        DocsEnum docs = null;
         try {
           while(true) {
             final BytesRef term = termsEnum.next();
@@ -291,7 +292,7 @@
               break;
             }
             final byte termval = parser.parseByte(term);
-            final DocsEnum docs = termsEnum.docs(delDocs);
+            docs = termsEnum.docs(delDocs, docs);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -337,6 +338,7 @@
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
         final Bits delDocs = reader.getDeletedDocs();
+        DocsEnum docs = null;
         try {
           while(true) {
             final BytesRef term = termsEnum.next();
@@ -344,7 +346,7 @@
               break;
             }
             final short termval = parser.parseShort(term);
-            final DocsEnum docs = termsEnum.docs(delDocs);
+            docs = termsEnum.docs(delDocs, docs);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -395,6 +397,7 @@
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
         final Bits delDocs = reader.getDeletedDocs();
+        DocsEnum docs = null;
         try {
           while(true) {
             final BytesRef term = termsEnum.next();
@@ -407,7 +410,7 @@
               retArray = new int[reader.maxDoc()];
             }
 
-            final DocsEnum docs = termsEnum.docs(delDocs);
+            docs = termsEnum.docs(delDocs, docs);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -466,6 +469,7 @@
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
         final Bits delDocs = reader.getDeletedDocs();
+        DocsEnum docs = null;
         try {
           while(true) {
             final BytesRef term = termsEnum.next();
@@ -477,8 +481,8 @@
               // late init so numeric fields don't double allocate
               retArray = new float[reader.maxDoc()];
             }
-
-            final DocsEnum docs = termsEnum.docs(delDocs);
+            
+            docs = termsEnum.docs(delDocs, docs);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -532,6 +536,7 @@
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
         final Bits delDocs = reader.getDeletedDocs();
+        DocsEnum docs = null;
         try {
           while(true) {
             final BytesRef term = termsEnum.next();
@@ -544,7 +549,7 @@
               retArray = new long[reader.maxDoc()];
             }
 
-            final DocsEnum docs = termsEnum.docs(delDocs);
+            docs = termsEnum.docs(delDocs, docs);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -600,6 +605,7 @@
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
         final Bits delDocs = reader.getDeletedDocs();
+        DocsEnum docs = null;
         try {
           while(true) {
             final BytesRef term = termsEnum.next();
@@ -612,7 +618,7 @@
               retArray = new double[reader.maxDoc()];
             }
 
-            final DocsEnum docs = termsEnum.docs(delDocs);
+            docs = termsEnum.docs(delDocs, docs);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -651,12 +657,13 @@
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
         final Bits delDocs = reader.getDeletedDocs();
+        DocsEnum docs = null;
         while(true) {
           final BytesRef term = termsEnum.next();
           if (term == null) {
             break;
           }
-          final DocsEnum docs = termsEnum.docs(delDocs);
+          docs = termsEnum.docs(delDocs, docs);
           final String termval = term.toString();
           while (true) {
             final int docID = docs.nextDoc();
@@ -689,6 +696,7 @@
       final int[] retArray = new int[reader.maxDoc()];
       String[] mterms = new String[reader.maxDoc()+1];
 
+      //System.out.println("FC: getStringIndex field=" + field);
       Terms terms = reader.fields().terms(field);
 
       int t = 0;  // current term number
@@ -702,6 +710,7 @@
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
         final Bits delDocs = reader.getDeletedDocs();
+        DocsEnum docs = null;
         while(true) {
           final BytesRef term = termsEnum.next();
           if (term == null) {
@@ -710,13 +719,15 @@
 
           // store term text
           mterms[t] = term.toString();
+          //System.out.println("FC:  ord=" + t + " term=" + term.toBytesString());
 
-          final DocsEnum docs = termsEnum.docs(delDocs);
+          docs = termsEnum.docs(delDocs, docs);
           while (true) {
             final int docID = docs.nextDoc();
             if (docID == DocsEnum.NO_MORE_DOCS) {
               break;
             }
+            //System.out.println("FC:    docID=" + docID);
             retArray[docID] = t;
           }
           t++;
@@ -736,6 +747,7 @@
       }
 
       StringIndex value = new StringIndex (retArray, mterms);
+      //System.out.println("FC: done\n");
       return value;
     }
   };

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermsEnum.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermsEnum.java Sat Jan 30 10:16:35 2010
@@ -23,6 +23,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.Bits;
 
@@ -157,8 +158,13 @@
   }
 
   @Override
-  public DocsEnum docs(Bits bits) throws IOException {
-    return (tenum == null) ? null : tenum.docs(bits);
+  public DocsEnum docs(Bits bits, DocsEnum reuse) throws IOException {
+    return (tenum == null) ? null : tenum.docs(bits, reuse);
+  }
+    
+  @Override
+  public DocsAndPositionsEnum docsAndPositions(Bits bits, DocsAndPositionsEnum reuse) throws IOException {
+    return (tenum == null) ? null : tenum.docsAndPositions(bits, reuse);
   }
     
   @Override

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/IndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/IndexSearcher.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/IndexSearcher.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/IndexSearcher.java Sat Jan 30 10:16:35 2010
@@ -247,7 +247,7 @@
     }
     int filterDoc = filterIter.nextDoc();
     int scorerDoc = scorer.advance(filterDoc);
-    
+
     collector.setScorer(scorer);
     while (true) {
       if (scorerDoc == filterDoc) {



Mime
View raw message