lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r824918 [5/11] - in /lucene/java/branches/flex_1458: contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/ contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/ contrib/benchmark/src/test/org/apache/lucene/benc...
Date Tue, 13 Oct 2009 20:44:59 GMT
Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/MultiLevelSkipListWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/MultiLevelSkipListWriter.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/MultiLevelSkipListWriter.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/MultiLevelSkipListWriter.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,156 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.RAMOutputStream;
+
+/**
+ * This abstract class writes skip lists with multiple levels.
+ * 
+ * Example for skipInterval = 3:
+ *                                                     c            (skip level 2)
+ *                 c                 c                 c            (skip level 1) 
+ *     x     x     x     x     x     x     x     x     x     x      (skip level 0)
+ * d d d d d d d d d d d d d d d d d d d d d d d d d d d d d d d d  (posting list)
+ *     3     6     9     12    15    18    21    24    27    30     (df)
+ * 
+ * d - document
+ * x - skip data
+ * c - skip data with child pointer
+ * 
+ * Skip level i contains every skipInterval-th entry from skip level i-1.
+ * Therefore the number of entries on level i is: floor(df / ((skipInterval ^ (i + 1))).
+ * 
+ * Each skip entry on a level i>0 contains a pointer to the corresponding skip entry in list i-1.
+ * This guarantees a logarithmic amount of skips to find the target document.
+ * 
+ * While this class takes care of writing the different skip levels,
+ * subclasses must define the actual format of the skip data.
+ * 
+ */
+
+// nocommit -- made public
+public abstract class MultiLevelSkipListWriter {
+  // number of levels in this skip list
+  protected int numberOfSkipLevels;
+  
+  // the skip interval in the list with level = 0
+  private int skipInterval;
+  
+  // for every skip level a different buffer is used 
+  private RAMOutputStream[] skipBuffer;
+
+  protected MultiLevelSkipListWriter(int skipInterval, int maxSkipLevels, int df) {
+    this.skipInterval = skipInterval;
+    
+    // calculate the maximum number of skip levels for this document frequency
+    numberOfSkipLevels = df == 0 ? 0 : (int) Math.floor(Math.log(df) / Math.log(skipInterval));
+    
+    // make sure it does not exceed maxSkipLevels
+    if (numberOfSkipLevels > maxSkipLevels) {
+      numberOfSkipLevels = maxSkipLevels;
+    }
+  }
+  
+  protected void init() {
+    skipBuffer = new RAMOutputStream[numberOfSkipLevels];
+    for (int i = 0; i < numberOfSkipLevels; i++) {
+      skipBuffer[i] = new RAMOutputStream();
+    }
+  }
+
+  protected void resetSkip() {
+    // creates new buffers or empties the existing ones
+    if (skipBuffer == null) {
+      init();
+    } else {
+      for (int i = 0; i < skipBuffer.length; i++) {
+        skipBuffer[i].reset();
+      }
+    }      
+  }
+
+  /**
+   * Subclasses must implement the actual skip data encoding in this method.
+   *  
+   * @param level the level skip data shall be writing for
+   * @param skipBuffer the skip buffer to write to
+   */
+  protected abstract void writeSkipData(int level, IndexOutput skipBuffer) throws IOException;
+  
+  /**
+   * Writes the current skip data to the buffers. The current document frequency determines
+   * the max level is skip data is to be written to. 
+   * 
+   * @param df the current document frequency 
+   * @throws IOException
+   */
+  // nocommit -- made public
+  public void bufferSkip(int df) throws IOException {
+    int numLevels;
+   
+    // determine max level
+    for (numLevels = 0; (df % skipInterval) == 0 && numLevels < numberOfSkipLevels; df /= skipInterval) {
+      numLevels++;
+    }
+    
+    long childPointer = 0;
+    
+    for (int level = 0; level < numLevels; level++) {
+      writeSkipData(level, skipBuffer[level]);
+      
+      long newChildPointer = skipBuffer[level].getFilePointer();
+      
+      if (level != 0) {
+        // store child pointers for all levels except the lowest
+        skipBuffer[level].writeVLong(childPointer);
+      }
+      
+      //remember the childPointer for the next level
+      childPointer = newChildPointer;
+    }
+  }
+
+  /**
+   * Writes the buffered skip lists to the given output.
+   * 
+   * @param output the IndexOutput the skip lists shall be written to 
+   * @return the pointer the skip list starts
+   */
+  // nocommit -- made public
+  public long writeSkip(IndexOutput output) throws IOException {
+    long skipPointer = output.getFilePointer();
+    //System.out.println("skipper.writeSkip fp=" + skipPointer);
+    if (skipBuffer == null || skipBuffer.length == 0) return skipPointer;
+    
+    for (int level = numberOfSkipLevels - 1; level > 0; level--) {
+      long length = skipBuffer[level].getFilePointer();
+      if (length > 0) {
+        output.writeVLong(length);
+        skipBuffer[level].writeTo(output);
+      }
+    }
+    skipBuffer[0].writeTo(output);
+    
+    return skipPointer;
+  }
+
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/MultiLevelSkipListWriter.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsConsumer.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsConsumer.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsConsumer.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,44 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.store.IndexOutput;
+
+// nocommit -- split into generic vs standardtermsdict
+public abstract class PositionsConsumer {
+
+  public abstract void start(IndexOutput termsOut) throws IOException;
+
+  public abstract void startTerm() throws IOException;
+
+  /** Add a new position & payload.  If payloadLength > 0
+   *  you must read those bytes from the IndexInput.  NOTE:
+   *  you must fully consume the byte[] payload, since
+   *  caller is free to reuse it on subsequent calls. */
+  public abstract void addPosition(int position, byte[] payload, int payloadOffset, int payloadLength) throws IOException;
+
+  /** Called when we are done adding positions & payloads
+   * for each doc */
+  public abstract void finishDoc() throws IOException;
+
+  public abstract void finishTerm(boolean isIndexTerm) throws IOException;
+  
+  public abstract void close() throws IOException;
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsConsumer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsProducer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsProducer.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsProducer.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsProducer.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,40 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.PositionsEnum;
+
+public abstract class PositionsProducer {
+
+  public abstract class Reader {
+    public abstract void readTerm(int docFreq, boolean isIndexTerm) throws IOException;
+
+    /** Returns a pos enum for the last term read */
+    public abstract PositionsEnum positions() throws IOException;
+  }
+
+  public abstract void start(IndexInput termsIn) throws IOException;
+
+  public abstract Reader reader(FieldInfo fieldInfo, IndexInput termsIn) throws IOException;
+
+  public abstract void close() throws IOException;
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsProducer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,38 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+/**
+ * NOTE: this API is experimental and will likely change
+ */
+
+public abstract class TermsConsumer {
+
+  // nocommit -- CharSequence?
+  /** Starts a new term in this field; term ends with U+FFFF
+   *  char */
+  public abstract DocsConsumer startTerm(char[] text, int start) throws IOException;
+
+  /** Finishes the current term */
+  public abstract void finishTerm(char[] text, int start, int numDocs) throws IOException;
+
+  /** Called when we are done adding terms to this field */
+  public abstract void finish() throws IOException;
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,173 @@
+package org.apache.lucene.index.codecs.intblock;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Naive int block API that writes vInts.  This is
+ *  expected to give poor performance; it's really only for
+ *  testing the pluggability.  One should typically use pfor instead. */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.codecs.sep.IntIndexInput;
+import org.apache.lucene.store.IndexInput;
+
+/** Abstract base class that reads fixed-size blocks of ints
+ *  from an IndexInput.  While this is a simple approach, a
+ *  more performant approach would directly create an impl
+ *  of IntIndexInput inside Directory.  Wrapping a generic
+ *  IndexInput will likely cost performance.  */
+public abstract class FixedIntBlockIndexInput extends IntIndexInput {
+
+  private IndexInput in;
+  protected int blockSize;
+
+  protected void init(IndexInput in) throws IOException {
+    this.in = in;
+    blockSize = in.readVInt();
+  }
+
+  public Reader reader() throws IOException {
+    int[] buffer = new int[blockSize];
+    IndexInput clone = (IndexInput) in.clone();
+    // nocommit -- awkward
+    return new Reader(clone, buffer, getBlockReader(clone, buffer));
+  }
+
+  public void close() throws IOException {
+    in.close();
+  }
+
+  public Index index() {
+    return new Index();
+  }
+
+  protected abstract BlockReader getBlockReader(IndexInput in, int[] buffer) throws IOException;
+
+  public interface BlockReader {
+    public void readBlock() throws IOException;
+  }
+
+  private static class Reader extends IntIndexInput.Reader {
+    private final IndexInput in;
+
+    protected final int[] pending;
+    int upto;
+
+    private boolean seekPending;
+    private long pendingFP;
+    private int pendingUpto;
+    private long lastBlockFP;
+    private final BlockReader blockReader;
+    private final int blockSize;
+
+    private final BulkReadResult result = new BulkReadResult();
+
+    public Reader(IndexInput in, int[] pending, BlockReader blockReader) {
+      this.in = in;
+      this.pending = pending;
+      this.blockSize = pending.length;
+      result.buffer = pending;
+      this.blockReader = blockReader;
+    }
+
+    void seek(long fp, int upto) {
+      pendingFP = fp;
+      pendingUpto = upto;
+      seekPending = true;
+    }
+
+    private void maybeSeek() throws IOException {
+      if (seekPending) {
+        if (pendingFP != lastBlockFP) {
+          // need new block
+          in.seek(pendingFP);
+          lastBlockFP = pendingFP;
+          blockReader.readBlock();
+        }
+        upto = pendingUpto;
+        seekPending = false;
+      }
+    }
+
+    public int next() throws IOException {
+      maybeSeek();
+      if (upto == blockSize) {
+        lastBlockFP = in.getFilePointer();
+        blockReader.readBlock();
+        upto = 0;
+      }
+
+      return pending[upto++];
+    }
+
+    public BulkReadResult read(int[] buffer, int count) throws IOException {
+      maybeSeek();
+      if (upto == blockSize) {
+        blockReader.readBlock();
+        upto = 0;
+      }
+      result.offset = upto;
+      if (upto + count < blockSize) {
+        result.len = count;
+        upto += count;
+      } else {
+        result.len = blockSize - upto;
+        upto = blockSize;
+      }
+
+      return result;
+    }
+
+    public String descFilePointer() {
+      return in.getFilePointer() + ":" + upto;
+    }
+  }
+
+  private class Index extends IntIndexInput.Index {
+    private long fp;
+    private int upto;
+
+    public void read(IndexInput indexIn, boolean absolute) throws IOException {
+      if (absolute) {
+        fp = indexIn.readVLong();
+        upto = indexIn.readVInt();
+      } else {
+        final long delta = indexIn.readVLong();
+        if (delta == 0) {
+          // same block
+          upto += indexIn.readVInt();
+        } else {
+          // new block
+          fp += delta;
+          upto = indexIn.readVInt();
+        }
+      }
+      assert upto < blockSize;
+    }
+
+    public void seek(IntIndexInput.Reader other) throws IOException {
+      ((Reader) other).seek(fp, upto);
+    }
+
+    public void set(IntIndexInput.Index other) {
+      Index idx = (Index) other;
+      fp = idx.fp;
+      upto = idx.upto;
+    }
+  }
+}
\ No newline at end of file

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexOutput.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexOutput.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexOutput.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,109 @@
+package org.apache.lucene.index.codecs.intblock;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Naive int block API that writes vInts.  This is
+ *  expected to give poor performance; it's really only for
+ *  testing the pluggability.  One should typically use pfor instead. */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.codecs.sep.IntIndexOutput;
+import org.apache.lucene.store.IndexOutput;
+
+public abstract class FixedIntBlockIndexOutput extends IntIndexOutput {
+
+  private IndexOutput out;
+  private int blockSize;
+  private int[] pending;
+  private int upto;
+  private long lastSavedFilePointer;
+  private int lastSavedUpto;
+
+  protected void init(IndexOutput out, int fixedBlockSize) throws IOException {
+    blockSize = fixedBlockSize;
+    out.writeVInt(blockSize);
+    this.out = out;
+    pending = new int[blockSize];
+  }
+
+  protected abstract void flushBlock(int[] buffer, IndexOutput out) throws IOException;
+
+  public Index index() throws IOException {
+    return new Index();
+  }
+
+  public String descFilePointer() {
+    return out.getFilePointer() + ":" + upto;
+  }
+
+  private class Index extends IntIndexOutput.Index {
+    long fp;
+    int upto;
+    long lastFP;
+    int lastUpto;
+
+    public void mark() throws IOException {
+      fp = out.getFilePointer();
+      upto = FixedIntBlockIndexOutput.this.upto;
+    }
+
+    public void set(IntIndexOutput.Index other) throws IOException {
+      Index idx = (Index) other;
+      lastFP = fp = idx.fp;
+      lastUpto = upto = idx.upto;
+    }
+
+    public void write(IndexOutput indexOut, boolean absolute) throws IOException {
+      if (absolute) {
+        indexOut.writeVLong(fp);
+        indexOut.writeVInt(upto);
+      } else if (fp == lastFP) {
+        // same block
+        indexOut.writeVLong(0);
+        assert upto >= lastUpto;
+        indexOut.writeVLong(upto - lastUpto);
+      } else {      
+        // new block
+        indexOut.writeVLong(fp - lastFP);
+        indexOut.writeVLong(upto);
+      }
+      lastUpto = upto;
+      lastFP = fp;
+    }
+  }
+
+  public void write(int v) throws IOException {
+    pending[upto++] = v;
+    if (upto == blockSize) {
+      flushBlock(pending, out);
+      upto = 0;
+    }
+  }
+
+  public void close() throws IOException {
+    // NOTE: entries in the block after current upto are
+    // invalid
+    // nocommit -- zero fill?
+    try {
+      flushBlock(pending, out);
+    } finally {
+      out.close();
+    }
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexOutput.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,131 @@
+package org.apache.lucene.index.codecs.intblock;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.DocsConsumer;
+import org.apache.lucene.index.codecs.DocsProducer;
+import org.apache.lucene.index.codecs.FieldsConsumer;
+import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.sep.SepCodec;
+import org.apache.lucene.index.codecs.sep.SepDocsReader;
+import org.apache.lucene.index.codecs.sep.SepDocsWriter;
+import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexReader;
+import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexWriter;
+import org.apache.lucene.index.codecs.standard.StandardTermsDictReader;
+import org.apache.lucene.index.codecs.standard.StandardTermsDictWriter;
+import org.apache.lucene.index.codecs.standard.StandardTermsIndexReader;
+import org.apache.lucene.index.codecs.standard.StandardTermsIndexWriter;
+import org.apache.lucene.store.Directory;
+
+public class IntBlockCodec extends Codec {
+
+  public IntBlockCodec() {
+    name = "IntBlock";
+  }
+
+  public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+    DocsConsumer docsWriter = new SepDocsWriter(state, new SimpleIntBlockFactory(1024));
+
+    boolean success = false;
+    StandardTermsIndexWriter indexWriter;
+    try {
+      indexWriter = new SimpleStandardTermsIndexWriter(state);
+      success = true;
+    } finally {
+      if (!success) {
+        docsWriter.close();
+      }
+    }
+
+    success = false;
+    try {
+      FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, docsWriter);
+      success = true;
+      return ret;
+    } finally {
+      if (!success) {
+        try {
+          docsWriter.close();
+        } finally {
+          indexWriter.close();
+        }
+      }
+    }
+  }
+
+  /*
+  final static String DOC_EXTENSION = "doc";
+  final static String SKIP_EXTENSION = "skp";
+  final static String FREQ_EXTENSION = "frq";
+  final static String PROX_EXTENSION = "prx";
+  final static String PAYLOAD_EXTENSION = "pyl";
+  */
+
+  public FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException {
+    DocsProducer docsReader = new SepDocsReader(dir, si, readBufferSize, new SimpleIntBlockFactory(1024));
+
+    StandardTermsIndexReader indexReader;
+    boolean success = false;
+    try {
+      indexReader = new SimpleStandardTermsIndexReader(dir,
+                                                       fieldInfos,
+                                                       si.name,
+                                                       indexDivisor);
+      success = true;
+    } finally {
+      if (!success) {
+        docsReader.close();
+      }
+    }
+
+    success = false;
+    try {
+      FieldsProducer ret = new StandardTermsDictReader(indexReader,
+                                                       dir, fieldInfos, si.name,
+                                                       docsReader,
+                                                       readBufferSize);
+      success = true;
+      return ret;
+    } finally {
+      if (!success) {
+        try {
+          docsReader.close();
+        } finally {
+          indexReader.close();
+        }
+      }
+    }
+  }
+
+  public void files(Directory dir, SegmentInfo segmentInfo, Collection files) {
+    SepDocsReader.files(segmentInfo, files);
+    StandardTermsDictReader.files(segmentInfo, files);
+    SimpleStandardTermsIndexReader.files(segmentInfo, files);
+  }
+
+  public void getExtensions(Collection extensions) {
+    SepCodec.getSepExtensions(extensions);
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockFactory.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockFactory.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockFactory.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockFactory.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,38 @@
+package org.apache.lucene.index.codecs.intblock;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.index.codecs.sep.IntStreamFactory;
+import org.apache.lucene.index.codecs.sep.IntIndexInput;
+import org.apache.lucene.index.codecs.sep.IntIndexOutput;
+
+import java.io.IOException;
+
+public class SimpleIntBlockFactory extends IntStreamFactory {
+  private final int blockSize;
+  public SimpleIntBlockFactory(int blockSize) {
+    this.blockSize = blockSize;
+  }
+  public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException {
+    return new SimpleIntBlockIndexInput(dir, fileName, readBufferSize);
+  }
+  public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException {
+    return new SimpleIntBlockIndexOutput(dir, fileName, blockSize);
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockFactory.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,62 @@
+package org.apache.lucene.index.codecs.intblock;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Naive int block API that writes vInts.  This is
+ *  expected to give poor performance; it's really only for
+ *  testing the pluggability.  One should typically use pfor instead. */
+
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+
+import java.io.IOException;
+
+/** Don't use this class!!  It naively encodes ints one vInt
+ *  at a time.  Use it only for testing.  */
+public class SimpleIntBlockIndexInput extends FixedIntBlockIndexInput {
+
+  public SimpleIntBlockIndexInput(Directory dir, String fileName, int readBufferSize) throws IOException {
+    IndexInput in = dir.openInput(fileName, readBufferSize);
+    Codec.checkHeader(in, SimpleIntBlockIndexOutput.CODEC, SimpleIntBlockIndexOutput.VERSION_START);
+    init(in);
+  }
+
+  private static class BlockReader implements FixedIntBlockIndexInput.BlockReader {
+
+    private final IndexInput in;
+    private final int[] buffer;
+
+    public BlockReader(IndexInput in, int[] buffer) {
+      this.in = in;
+      this.buffer = buffer;
+    }
+
+    public void readBlock() throws IOException {
+      // silly impl
+      for(int i=0;i<buffer.length;i++) {
+        buffer[i] = in.readVInt();
+      }
+    }
+  }
+
+  protected BlockReader getBlockReader(IndexInput in, int[] buffer) {
+    return new BlockReader(in, buffer);
+  }
+}
+

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexOutput.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexOutput.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexOutput.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,51 @@
+package org.apache.lucene.index.codecs.intblock;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Naive int block API that writes vInts.  This is
+ *  expected to give poor performance; it's really only for
+ *  testing the pluggability.  One should typically use pfor instead. */
+
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexOutput;
+
+import java.io.IOException;
+
+/** Don't use this class!!  It naively encodes ints one vInt
+ *  at a time.  Use it only for testing.  */
+public class SimpleIntBlockIndexOutput extends FixedIntBlockIndexOutput {
+
+  public final static String CODEC = "SIMPLE_INT_BLOCKS";
+  public final static int VERSION_START = 0;
+  public final static int VERSION_CURRENT = VERSION_START;
+
+  public SimpleIntBlockIndexOutput(Directory dir, String fileName, int blockSize) throws IOException {
+    IndexOutput out = dir.createOutput(fileName);
+    Codec.writeHeader(out, CODEC, VERSION_CURRENT);
+    init(out, blockSize);
+  }
+
+  protected void flushBlock(int[] buffer, IndexOutput out) throws IOException {
+    // silly impl
+    for(int i=0;i<buffer.length;i++) {
+      out.writeVInt(buffer[i]);
+    }
+  }
+}
+

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexOutput.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,70 @@
+package org.apache.lucene.index.codecs.preflex;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Collection;
+import java.io.IOException;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.codecs.FieldsConsumer;
+import org.apache.lucene.index.codecs.FieldsProducer;
+
+/** Codec that reads the pre-flex-indexing postings
+ *  format.  It does not provide a writer because newly
+ *  written segments should use StandardCodec. */
+public class PreFlexCodec extends Codec {
+
+  /** Extension of terms file */
+  static final String TERMS_EXTENSION = "tis";
+
+  /** Extension of terms index file */
+  static final String TERMS_INDEX_EXTENSION = "tii";
+
+  /** Extension of freq postings file */
+  static final String FREQ_EXTENSION = "frq";
+
+  /** Extension of prox postings file */
+  static final String PROX_EXTENSION = "prx";
+
+  public PreFlexCodec() {
+    name = "PreFlex";
+  }
+  
+  public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+    throw new IllegalArgumentException("this codec can only be used for reading");
+  }
+
+  public FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo info, int readBufferSize, int indexDivisor) throws IOException {
+    return new PreFlexFields(dir, fieldInfos, info, readBufferSize, indexDivisor);
+  }
+
+  public void files(Directory dir, SegmentInfo info, Collection files) throws IOException {
+    PreFlexFields.files(dir, info, files);
+  }
+
+  public void getExtensions(Collection extensions) {
+    extensions.add(FREQ_EXTENSION);
+    extensions.add(PROX_EXTENSION);
+    extensions.add(TERMS_EXTENSION);
+    extensions.add(TERMS_INDEX_EXTENSION);
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,351 @@
+package org.apache.lucene.index.codecs.preflex;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.TreeMap;
+
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.PositionsEnum;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermRef;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Bits;
+
+public class PreFlexFields extends FieldsProducer {
+
+  // nocommit -- needed public by SegmentReader
+  public final TermInfosReader tis;
+
+  // nocomit -- needed public by SR
+  public final IndexInput freqStream;
+  // nocomit -- needed public by SR
+  public final IndexInput proxStream;
+  final private FieldInfos fieldInfos;
+  final TreeMap fields = new TreeMap(); /*String -> FieldInfo */
+
+  PreFlexFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, int readBufferSize, int indexDivisor)
+    throws IOException {
+    tis = new TermInfosReader(dir, info.name, fieldInfos, readBufferSize, indexDivisor);    
+    this.fieldInfos = fieldInfos;
+
+    // make sure that all index files have been read or are kept open
+    // so that if an index update removes them we'll still have them
+    freqStream = dir.openInput(info.name + ".frq", readBufferSize);
+    boolean anyProx = false;
+    final int numFields = fieldInfos.size();
+    for(int i=0;i<numFields;i++) {
+      final FieldInfo fieldInfo = fieldInfos.fieldInfo(i);
+      if (fieldInfo.isIndexed) {
+        fields.put(fieldInfo.name, fieldInfo);
+        if (!fieldInfo.omitTermFreqAndPositions) {
+          anyProx = true;
+        }
+      }
+    }
+
+    if (anyProx) {
+      proxStream = dir.openInput(info.name + ".prx", readBufferSize);
+    } else {
+      proxStream = null;
+    }
+  }
+
+  static void files(Directory dir, SegmentInfo info, Collection files) throws IOException {
+    files.add(IndexFileNames.segmentFileName(info.name, PreFlexCodec.TERMS_EXTENSION));
+    files.add(IndexFileNames.segmentFileName(info.name, PreFlexCodec.TERMS_INDEX_EXTENSION));
+    files.add(IndexFileNames.segmentFileName(info.name, PreFlexCodec.FREQ_EXTENSION));
+    //System.out.println("seg=" + info.name + " hasProx?=" + info.getHasProx());
+    if (info.getHasProx()) {
+      // LUCENE-1739: for certain versions of 2.9-dev,
+      // hasProx would be incorrectly computed during
+      // indexing as true, and then stored into the segments
+      // file, when it should have been false.  So we do the
+      // extra check, here:
+      final String prx = IndexFileNames.segmentFileName(info.name, PreFlexCodec.PROX_EXTENSION);
+      if (dir.fileExists(prx)) {
+        files.add(prx);
+      }
+    }
+  }
+
+  public FieldsEnum iterator() {
+    return new Fields();
+  }
+
+  public Terms terms(String field) {
+    FieldInfo fi = fieldInfos.fieldInfo(field);
+    if (fi != null) {
+      return new PreTerms(fi);
+    } else {
+      return null;
+    }
+  }
+
+  public void loadTermsIndex() throws IOException {
+    // nocommit -- todo
+  }
+
+  public void close() throws IOException {
+    tis.close();
+  }
+
+  private class Fields extends FieldsEnum {
+    Iterator it;
+    FieldInfo current;
+    private PreTermsEnum lastTermsEnum;
+
+    public Fields() {
+      it = fields.values().iterator();
+    }
+
+    public String next() {
+      if (it.hasNext()) {
+        current = (FieldInfo) it.next();
+        return current.name;
+      } else {
+        return null;
+      }
+    }
+    
+    public TermsEnum terms() throws IOException {
+      final PreTermsEnum terms;
+      if (lastTermsEnum != null) {
+        // Carry over SegmentTermsEnum
+        terms = new PreTermsEnum(current, lastTermsEnum.terms);
+      } else {
+        terms = new PreTermsEnum(current);
+      }
+      lastTermsEnum = terms;
+      return terms;
+    }
+  }
+  
+  private class PreTerms extends Terms {
+    final FieldInfo fieldInfo;
+    PreTerms(FieldInfo fieldInfo) {
+      this.fieldInfo = fieldInfo;
+    }
+    public TermsEnum iterator() {
+      //System.out.println("pff.init create no context");
+      return new PreTermsEnum(fieldInfo);
+    }
+  }
+
+  private class PreTermsEnum extends TermsEnum {
+    private SegmentTermEnum terms;
+    private final FieldInfo fieldInfo;
+    private PreDocsEnum docsEnum;
+    private boolean skipNext;
+    private TermRef current;
+
+    PreTermsEnum(FieldInfo fieldInfo) {
+      this.fieldInfo = fieldInfo;
+      terms = tis.terms();
+    }
+
+    PreTermsEnum(FieldInfo fieldInfo, SegmentTermEnum terms) {
+      this.fieldInfo = fieldInfo;
+      this.terms = terms;
+      skipNext = true;
+      if (Codec.DEBUG) {
+        System.out.println("pff.terms.init field=" + fieldInfo.name);
+      }
+    }
+
+    public SeekStatus seek(long ord) throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    public long ord() throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    public SeekStatus seek(TermRef term) throws IOException {
+      if (Codec.DEBUG) {
+        System.out.println("pff.seek term=" + term);
+      }
+      terms = tis.terms(new Term(fieldInfo.name, term.toString()));
+      final Term t = terms.term();
+      //System.out.println("  got to term=" + t  + " field eq?=" + (t.field() == fieldInfo.name) + " term eq?=" +
+      //term.equals(new TermRef(t.text())));
+
+      // nocommit -- reuse TermRef instance
+      final TermRef tr;
+      if (t != null) {
+        tr = new TermRef(t.text());
+      } else {
+        tr = null;
+      }
+
+      if (t != null && t.field() == fieldInfo.name && term.termEquals(tr)) {
+        current = tr;
+        return SeekStatus.FOUND;
+      } else if (t == null || t.field() != fieldInfo.name) {
+        current = null;
+        return SeekStatus.END;
+      } else {
+        current = tr;
+        return SeekStatus.NOT_FOUND;
+      }
+    }
+
+    public TermRef next() throws IOException {
+      if (skipNext) {
+        // nocommit -- is there a cleaner way?
+        skipNext = false;
+        // nocommit -- reuse TermRef
+        current = new TermRef(terms.term().text());
+        return current;
+      }
+      if (terms.next()) {
+        final Term t = terms.term();
+        if (Codec.DEBUG) {
+          System.out.println("pff.next term=" + t);
+        }
+        if (t.field() == fieldInfo.name) {
+          // nocommit -- reuse TermRef instance
+          if (Codec.DEBUG) {
+            System.out.println("  ok");
+          }
+          current = new TermRef(t.text());
+          return current;
+        } else {
+          // Crossed into new field
+          if (Codec.DEBUG) {
+            System.out.println("  stop (new field " + t.field());
+          }
+          current = null;
+          return null;
+        }
+      } else {
+        current = null;
+        return null;
+      }
+    }
+
+    public TermRef term() {
+      return current;
+    }
+
+    public int docFreq() {
+      return terms.docFreq();
+    }
+
+    public DocsEnum docs(Bits skipDocs) throws IOException {
+      return new PreDocsEnum(skipDocs, terms);
+    }
+  }
+
+  private final class PreDocsEnum extends DocsEnum {
+    final private SegmentTermDocs docs;
+    final private SegmentTermPositions pos;
+    private SegmentTermDocs current;
+    final private PrePositionsEnum prePos;
+
+    PreDocsEnum(Bits skipDocs, Term t) throws IOException {
+      current = docs = new SegmentTermDocs(freqStream, skipDocs, tis, fieldInfos);
+      pos = new SegmentTermPositions(freqStream, proxStream, skipDocs, tis, fieldInfos);
+      prePos = new PrePositionsEnum(pos);
+      docs.seek(t);
+      pos.seek(t);
+    }
+
+    PreDocsEnum(Bits skipDocs, SegmentTermEnum te) throws IOException {
+      current = docs = new SegmentTermDocs(freqStream, skipDocs, tis, fieldInfos);
+      pos = new SegmentTermPositions(freqStream, proxStream, skipDocs, tis, fieldInfos);
+      prePos = new PrePositionsEnum(pos);
+      docs.seek(te);
+      pos.seek(te);
+    }
+
+    public int next() throws IOException {
+      if (Codec.DEBUG) {
+        System.out.println("pff.docs.next");
+      }
+      if (current.next()) {
+        return current.doc();
+      } else {
+        return NO_MORE_DOCS;
+      }
+    }
+
+    public int advance(int target) throws IOException {
+      if (current.skipTo(target)) {
+        return current.doc();
+      } else {
+        return NO_MORE_DOCS;
+      }
+    }
+
+    public int freq() {
+      return current.freq();
+    }
+
+    public int read(int[] docIDs, int[] freqs) throws IOException {
+      if (current != docs) {
+        docs.skipTo(current.doc());
+        current = docs;
+      }
+      return current.read(docIDs, freqs);
+    }
+
+    public PositionsEnum positions() throws IOException {
+      if (current != pos) {
+        pos.skipTo(docs.doc());
+        current = pos;
+      }
+      return prePos;
+    }
+  }
+
+  private final class PrePositionsEnum extends PositionsEnum {
+    final private SegmentTermPositions pos;
+    PrePositionsEnum(SegmentTermPositions pos) {
+      this.pos = pos;
+    }
+
+    public int next() throws IOException {
+      return pos.nextPosition();
+    }
+
+    public int getPayloadLength() {
+      return pos.getPayloadLength();
+    }
+
+    public boolean hasPayload() {
+      return pos.isPayloadAvailable();
+    }
+
+    public byte[] getPayload(byte[] data, int offset) throws IOException {
+      return pos.getPayload(data, offset);
+    }
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,235 @@
+package org.apache.lucene.index.codecs.preflex;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.codecs.standard.DefaultSkipListReader;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Bits;
+
+/** @deprecated */
+public class SegmentTermDocs implements TermDocs {
+  //protected SegmentReader parent;
+  protected final Bits skipDocs;
+  private final FieldInfos fieldInfos;
+  private final TermInfosReader tis;
+  protected IndexInput freqStream;
+  protected int count;
+  protected int df;
+  int doc = 0;
+  int freq;
+
+  private int skipInterval;
+  private int maxSkipLevels;
+  private DefaultSkipListReader skipListReader;
+  
+  private long freqBasePointer;
+  private long proxBasePointer;
+
+  private long skipPointer;
+  private boolean haveSkipped;
+  
+  protected boolean currentFieldStoresPayloads;
+  protected boolean currentFieldOmitTermFreqAndPositions;
+  
+  /*
+  protected SegmentTermDocs(SegmentReader parent) {
+    this.parent = parent;
+    this.freqStream = (IndexInput) parent.core.freqStream.clone();
+    synchronized (parent) {
+      this.deletedDocs = parent.deletedDocs;
+    }
+    this.skipInterval = parent.core.getTermsReader().getSkipInterval();
+    this.maxSkipLevels = parent.core.getTermsReader().getMaxSkipLevels();
+  }
+  */
+
+  // nocommit -- SR needs public
+  public SegmentTermDocs(IndexInput freqStream, Bits skipDocs, TermInfosReader tis, FieldInfos fieldInfos) {
+    this.freqStream = (IndexInput) freqStream.clone();
+    this.skipDocs = skipDocs;
+    this.tis = tis;
+    this.fieldInfos = fieldInfos;
+    skipInterval = tis.getSkipInterval();
+    maxSkipLevels = tis.getMaxSkipLevels();
+  }
+
+  public void seek(Term term) throws IOException {
+    TermInfo ti = tis.get(term);
+    seek(ti, term);
+  }
+
+  public void seek(TermEnum termEnum) throws IOException {
+    TermInfo ti;
+    Term term;
+    
+    // use comparison of fieldinfos to verify that termEnum belongs to the same segment as this SegmentTermDocs
+    if (termEnum instanceof SegmentTermEnum && ((SegmentTermEnum) termEnum).fieldInfos == fieldInfos) {        // optimized case
+      SegmentTermEnum segmentTermEnum = ((SegmentTermEnum) termEnum);
+      term = segmentTermEnum.term();
+      ti = segmentTermEnum.termInfo();
+    } else  {                                         // punt case
+      term = termEnum.term();
+      ti = tis.get(term); 
+    }
+    
+    seek(ti, term);
+  }
+
+  void seek(TermInfo ti, Term term) throws IOException {
+    count = 0;
+    FieldInfo fi = fieldInfos.fieldInfo(term.field());
+    currentFieldOmitTermFreqAndPositions = (fi != null) ? fi.omitTermFreqAndPositions : false;
+    currentFieldStoresPayloads = (fi != null) ? fi.storePayloads : false;
+    if (ti == null) {
+      df = 0;
+    } else {
+      df = ti.docFreq;
+      doc = 0;
+      freqBasePointer = ti.freqPointer;
+      proxBasePointer = ti.proxPointer;
+      skipPointer = freqBasePointer + ti.skipOffset;
+      freqStream.seek(freqBasePointer);
+      haveSkipped = false;
+    }
+  }
+
+  public void close() throws IOException {
+    freqStream.close();
+    if (skipListReader != null)
+      skipListReader.close();
+  }
+
+  public final int doc() { return doc; }
+  public final int freq() { return freq; }
+
+  protected void skippingDoc() throws IOException {
+  }
+
+  public boolean next() throws IOException {
+    while (true) {
+      if (count == df)
+        return false;
+      final int docCode = freqStream.readVInt();
+      
+      if (currentFieldOmitTermFreqAndPositions) {
+        doc += docCode;
+        freq = 1;
+      } else {
+        doc += docCode >>> 1;       // shift off low bit
+        if ((docCode & 1) != 0)       // if low bit is set
+          freq = 1;         // freq is one
+        else
+          freq = freqStream.readVInt();     // else read freq
+      }
+      
+      count++;
+
+      if (skipDocs == null || !skipDocs.get(doc)) {
+        break;
+      }
+      skippingDoc();
+    }
+    return true;
+  }
+
+  /** Optimized implementation. */
+  public int read(final int[] docs, final int[] freqs)
+          throws IOException {
+    final int length = docs.length;
+    if (currentFieldOmitTermFreqAndPositions) {
+      return readNoTf(docs, freqs, length);
+    } else {
+      int i = 0;
+      while (i < length && count < df) {
+        // manually inlined call to next() for speed
+        final int docCode = freqStream.readVInt();
+        doc += docCode >>> 1;       // shift off low bit
+        if ((docCode & 1) != 0)       // if low bit is set
+          freq = 1;         // freq is one
+        else
+          freq = freqStream.readVInt();     // else read freq
+        count++;
+
+        if (skipDocs == null || !skipDocs.get(doc)) {
+          docs[i] = doc;
+          freqs[i] = freq;
+          ++i;
+        }
+      }
+      return i;
+    }
+  }
+
+  private final int readNoTf(final int[] docs, final int[] freqs, final int length) throws IOException {
+    int i = 0;
+    while (i < length && count < df) {
+      // manually inlined call to next() for speed
+      doc += freqStream.readVInt();       
+      count++;
+
+      if (skipDocs == null || !skipDocs.get(doc)) {
+        docs[i] = doc;
+        // Hardware freq to 1 when term freqs were not
+        // stored in the index
+        freqs[i] = 1;
+        ++i;
+      }
+    }
+    return i;
+  }
+ 
+  
+  /** Overridden by SegmentTermPositions to skip in prox stream. */
+  protected void skipProx(long proxPointer, int payloadLength) throws IOException {}
+
+  /** Optimized implementation. */
+  public boolean skipTo(int target) throws IOException {
+    if (df >= skipInterval) {                      // optimized case
+      if (skipListReader == null)
+        skipListReader = new DefaultSkipListReader((IndexInput) freqStream.clone(), maxSkipLevels, skipInterval); // lazily clone
+
+      if (!haveSkipped) {                          // lazily initialize skip stream
+        skipListReader.init(skipPointer, freqBasePointer, proxBasePointer, df, currentFieldStoresPayloads);
+        haveSkipped = true;
+      }
+
+      int newCount = skipListReader.skipTo(target); 
+      if (newCount > count) {
+        freqStream.seek(skipListReader.getFreqPointer());
+        skipProx(skipListReader.getProxPointer(), skipListReader.getPayloadLength());
+
+        doc = skipListReader.getDoc();
+        count = newCount;
+      }      
+    }
+
+    // done skipping, now just scan
+    do {
+      if (!next())
+        return false;
+    } while (target > doc);
+    return true;
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,229 @@
+package org.apache.lucene.index.codecs.preflex;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.CorruptIndexException;
+
+/**
+ * @deprecated No longer used with flex indexing, except for
+ * reading old segments */
+
+public final class SegmentTermEnum extends TermEnum implements Cloneable {
+  private IndexInput input;
+  FieldInfos fieldInfos;
+  long size;
+  long position = -1;
+
+  /** The file format version, a negative number. */
+  public static final int FORMAT = -3;
+
+  // Changed strings to true utf8 with length-in-bytes not
+  // length-in-chars
+  public static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = -4;
+
+  // NOTE: always change this if you switch to a new format!
+  public static final int FORMAT_CURRENT = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES;
+
+  private TermBuffer termBuffer = new TermBuffer();
+  private TermBuffer prevBuffer = new TermBuffer();
+  private TermBuffer scanBuffer = new TermBuffer(); // used for scanning
+
+  private TermInfo termInfo = new TermInfo();
+
+  private int format;
+  private boolean isIndex = false;
+  long indexPointer = 0;
+  int indexInterval;
+  int skipInterval;
+  int maxSkipLevels;
+  private int formatM1SkipInterval;
+
+  SegmentTermEnum(IndexInput i, FieldInfos fis, boolean isi)
+          throws CorruptIndexException, IOException {
+    input = i;
+    fieldInfos = fis;
+    isIndex = isi;
+    maxSkipLevels = 1; // use single-level skip lists for formats > -3 
+    
+    int firstInt = input.readInt();
+    if (firstInt >= 0) {
+      // original-format file, without explicit format version number
+      format = 0;
+      size = firstInt;
+
+      // back-compatible settings
+      indexInterval = 128;
+      skipInterval = Integer.MAX_VALUE; // switch off skipTo optimization
+    } else {
+      // we have a format version number
+      format = firstInt;
+
+      // check that it is a format we can understand
+      if (format < FORMAT_CURRENT)
+        throw new CorruptIndexException("Unknown format version:" + format + " expected " + FORMAT_CURRENT + " or higher");
+
+      size = input.readLong();                    // read the size
+      
+      if(format == -1){
+        if (!isIndex) {
+          indexInterval = input.readInt();
+          formatM1SkipInterval = input.readInt();
+        }
+        // switch off skipTo optimization for file format prior to 1.4rc2 in order to avoid a bug in 
+        // skipTo implementation of these versions
+        skipInterval = Integer.MAX_VALUE;
+      } else {
+        indexInterval = input.readInt();
+        skipInterval = input.readInt();
+        if (format <= FORMAT) {
+          // this new format introduces multi-level skipping
+          maxSkipLevels = input.readInt();
+        }
+      }
+      assert indexInterval > 0: "indexInterval=" + indexInterval + " is negative; must be > 0";
+      assert skipInterval > 0: "skipInterval=" + skipInterval + " is negative; must be > 0";
+    }
+    if (format > FORMAT_VERSION_UTF8_LENGTH_IN_BYTES) {
+      termBuffer.setPreUTF8Strings();
+      scanBuffer.setPreUTF8Strings();
+      prevBuffer.setPreUTF8Strings();
+    }
+  }
+
+  protected Object clone() {
+    SegmentTermEnum clone = null;
+    try {
+      clone = (SegmentTermEnum) super.clone();
+    } catch (CloneNotSupportedException e) {}
+
+    clone.input = (IndexInput) input.clone();
+    clone.termInfo = new TermInfo(termInfo);
+
+    clone.termBuffer = (TermBuffer)termBuffer.clone();
+    clone.prevBuffer = (TermBuffer)prevBuffer.clone();
+    clone.scanBuffer = new TermBuffer();
+
+    return clone;
+  }
+
+  final void seek(long pointer, int p, Term t, TermInfo ti)
+          throws IOException {
+    input.seek(pointer);
+    position = p;
+    termBuffer.set(t);
+    prevBuffer.reset();
+    termInfo.set(ti);
+  }
+
+  /** Increments the enumeration to the next element.  True if one exists.*/
+  public final boolean next() throws IOException {
+    if (position++ >= size - 1) {
+      prevBuffer.set(termBuffer);
+      termBuffer.reset();
+      return false;
+    }
+
+    prevBuffer.set(termBuffer);
+    termBuffer.read(input, fieldInfos);
+
+    termInfo.docFreq = input.readVInt();	  // read doc freq
+    termInfo.freqPointer += input.readVLong();	  // read freq pointer
+    termInfo.proxPointer += input.readVLong();	  // read prox pointer
+    
+    if(format == -1){
+    //  just read skipOffset in order to increment  file pointer; 
+    // value is never used since skipTo is switched off
+      if (!isIndex) {
+        if (termInfo.docFreq > formatM1SkipInterval) {
+          termInfo.skipOffset = input.readVInt(); 
+        }
+      }
+    }
+    else{
+      if (termInfo.docFreq >= skipInterval) 
+        termInfo.skipOffset = input.readVInt();
+    }
+    
+    if (isIndex)
+      indexPointer += input.readVLong();	  // read index pointer
+
+    return true;
+  }
+
+  /** Optimized scan, without allocating new terms. 
+   *  Return number of invocations to next(). */
+  final int scanTo(Term term) throws IOException {
+    scanBuffer.set(term);
+    int count = 0;
+    while (scanBuffer.compareTo(termBuffer) > 0 && next()) {
+      count++;
+    }
+    return count;
+  }
+
+  /** Returns the current Term in the enumeration.
+   Initially invalid, valid after next() called for the first time.*/
+  public final Term term() {
+    return termBuffer.toTerm();
+  }
+
+  /** Returns the previous Term enumerated. Initially null.*/
+  final Term prev() {
+    return prevBuffer.toTerm();
+  }
+
+  /** Returns the current TermInfo in the enumeration.
+   Initially invalid, valid after next() called for the first time.*/
+  final TermInfo termInfo() {
+    return new TermInfo(termInfo);
+  }
+
+  /** Sets the argument to the current TermInfo in the enumeration.
+   Initially invalid, valid after next() called for the first time.*/
+  final void termInfo(TermInfo ti) {
+    ti.set(termInfo);
+  }
+
+  /** Returns the docFreq from the current TermInfo in the enumeration.
+   Initially invalid, valid after next() called for the first time.*/
+  public final int docFreq() {
+    return termInfo.docFreq;
+  }
+
+  /* Returns the freqPointer from the current TermInfo in the enumeration.
+    Initially invalid, valid after next() called for the first time.*/
+  final long freqPointer() {
+    return termInfo.freqPointer;
+  }
+
+  /* Returns the proxPointer from the current TermInfo in the enumeration.
+    Initially invalid, valid after next() called for the first time.*/
+  final long proxPointer() {
+    return termInfo.proxPointer;
+  }
+
+  /** Closes the enumeration to further activity, freeing resources. */
+  public final void close() throws IOException {
+    input.close();
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,210 @@
+package org.apache.lucene.index.codecs.preflex;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.FieldInfos;
+
+import java.io.IOException;
+
+public final class SegmentTermPositions
+extends SegmentTermDocs implements TermPositions {
+  private IndexInput proxStream;
+  private IndexInput proxStreamOrig;
+  private int proxCount;
+  private int position;
+  
+  // the current payload length
+  private int payloadLength;
+  // indicates whether the payload of the current position has
+  // been read from the proxStream yet
+  private boolean needToLoadPayload;
+  
+  // these variables are being used to remember information
+  // for a lazy skip
+  private long lazySkipPointer = -1;
+  private int lazySkipProxCount = 0;
+
+  /*
+  SegmentTermPositions(SegmentReader p) {
+    super(p);
+    this.proxStream = null;  // the proxStream will be cloned lazily when nextPosition() is called for the first time
+  }
+  */
+
+  // nocommit -- public
+  public SegmentTermPositions(IndexInput freqStream, IndexInput proxStream, Bits skipDocs, TermInfosReader tis, FieldInfos fieldInfos) {
+    super(freqStream, skipDocs, tis, fieldInfos);
+    this.proxStreamOrig = proxStream;  // the proxStream will be cloned lazily when nextPosition() is called for the first time
+  }
+
+  final void seek(TermInfo ti, Term term) throws IOException {
+    super.seek(ti, term);
+    if (ti != null)
+      lazySkipPointer = ti.proxPointer;
+    
+    lazySkipProxCount = 0;
+    proxCount = 0;
+    payloadLength = 0;
+    needToLoadPayload = false;
+  }
+
+  public final void close() throws IOException {
+    super.close();
+    if (proxStream != null) proxStream.close();
+  }
+
+  public final int nextPosition() throws IOException {
+    if (currentFieldOmitTermFreqAndPositions)
+      // This field does not store term freq, positions, payloads
+      return 0;
+    // perform lazy skips if necessary
+    lazySkip();
+    proxCount--;
+    return position += readDeltaPosition();
+  }
+
+  private final int readDeltaPosition() throws IOException {
+    int delta = proxStream.readVInt();
+    if (currentFieldStoresPayloads) {
+      // if the current field stores payloads then
+      // the position delta is shifted one bit to the left.
+      // if the LSB is set, then we have to read the current
+      // payload length
+      if ((delta & 1) != 0) {
+        payloadLength = proxStream.readVInt();
+      } 
+      delta >>>= 1;
+      needToLoadPayload = true;
+    }
+    return delta;
+  }
+  
+  protected final void skippingDoc() throws IOException {
+    // we remember to skip a document lazily
+    lazySkipProxCount += freq;
+  }
+
+  public final boolean next() throws IOException {
+    // we remember to skip the remaining positions of the current
+    // document lazily
+    lazySkipProxCount += proxCount;
+    
+    if (super.next()) {               // run super
+      proxCount = freq;               // note frequency
+      position = 0;               // reset position
+      return true;
+    }
+    return false;
+  }
+
+  public final int read(final int[] docs, final int[] freqs) {
+    throw new UnsupportedOperationException("TermPositions does not support processing multiple documents in one call. Use TermDocs instead.");
+  }
+
+
+  /** Called by super.skipTo(). */
+  protected void skipProx(long proxPointer, int payloadLength) throws IOException {
+    // we save the pointer, we might have to skip there lazily
+    lazySkipPointer = proxPointer;
+    lazySkipProxCount = 0;
+    proxCount = 0;
+    this.payloadLength = payloadLength;
+    needToLoadPayload = false;
+  }
+
+  private void skipPositions(int n) throws IOException {
+    assert !currentFieldOmitTermFreqAndPositions;
+    for (int f = n; f > 0; f--) {        // skip unread positions
+      readDeltaPosition();
+      skipPayload();
+    }      
+  }
+  
+  private void skipPayload() throws IOException {
+    if (needToLoadPayload && payloadLength > 0) {
+      proxStream.seek(proxStream.getFilePointer() + payloadLength);
+    }
+    needToLoadPayload = false;
+  }
+
+  // It is not always necessary to move the prox pointer
+  // to a new document after the freq pointer has been moved.
+  // Consider for example a phrase query with two terms:
+  // the freq pointer for term 1 has to move to document x
+  // to answer the question if the term occurs in that document. But
+  // only if term 2 also matches document x, the positions have to be
+  // read to figure out if term 1 and term 2 appear next
+  // to each other in document x and thus satisfy the query.
+  // So we move the prox pointer lazily to the document
+  // as soon as positions are requested.
+  private void lazySkip() throws IOException {
+    if (proxStream == null) {
+      // clone lazily
+      proxStream = (IndexInput)proxStreamOrig.clone();
+    }
+    
+    // we might have to skip the current payload
+    // if it was not read yet
+    skipPayload();
+      
+    if (lazySkipPointer != -1) {
+      proxStream.seek(lazySkipPointer);
+      lazySkipPointer = -1;
+    }
+     
+    if (lazySkipProxCount != 0) {
+      skipPositions(lazySkipProxCount);
+      lazySkipProxCount = 0;
+    }
+  }
+  
+  public int getPayloadLength() {
+    return payloadLength;
+  }
+
+  public byte[] getPayload(byte[] data, int offset) throws IOException {
+    if (!needToLoadPayload) {
+      throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once.");
+    }
+
+    // read payloads lazily
+    byte[] retArray;
+    int retOffset;
+    if (data == null || data.length - offset < payloadLength) {
+      // the array is too small to store the payload data,
+      // so we allocate a new one
+      retArray = new byte[payloadLength];
+      retOffset = 0;
+    } else {
+      retArray = data;
+      retOffset = offset;
+    }
+    proxStream.readBytes(retArray, retOffset, payloadLength);
+    needToLoadPayload = false;
+    return retArray;
+  }
+
+  public boolean isPayloadAvailable() {
+    return needToLoadPayload && payloadLength > 0;
+  }
+
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,141 @@
+package org.apache.lucene.index.codecs.preflex;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.UnicodeUtil;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.FieldInfos;
+
+final class TermBuffer implements Cloneable {
+
+  private String field;
+  private Term term;                            // cached
+  private boolean preUTF8Strings;                // true if strings are stored in modified UTF8 encoding (LUCENE-510)
+  private boolean dirty;                          // true if text was set externally (ie not read via UTF8 bytes)
+
+  private UnicodeUtil.UTF16Result text = new UnicodeUtil.UTF16Result();
+  private UnicodeUtil.UTF8Result bytes = new UnicodeUtil.UTF8Result();
+
+  public final int compareTo(TermBuffer other) {
+    if (field == other.field) 	  // fields are interned
+      return compareChars(text.result, text.length, other.text.result, other.text.length);
+    else
+      return field.compareTo(other.field);
+  }
+
+  private static final int compareChars(char[] chars1, int len1,
+                                        char[] chars2, int len2) {
+    final int end = len1 < len2 ? len1:len2;
+    for (int k = 0; k < end; k++) {
+      char c1 = chars1[k];
+      char c2 = chars2[k];
+      if (c1 != c2) {
+        return c1 - c2;
+      }
+    }
+    return len1 - len2;
+  }
+
+  /** Call this if the IndexInput passed to {@link #read}
+   *  stores terms in the "modified UTF8" (pre LUCENE-510)
+   *  format. */
+  void setPreUTF8Strings() {
+    preUTF8Strings = true;
+  }
+
+  public final void read(IndexInput input, FieldInfos fieldInfos)
+    throws IOException {
+    this.term = null;                           // invalidate cache
+    int start = input.readVInt();
+    int length = input.readVInt();
+    int totalLength = start + length;
+    if (preUTF8Strings) {
+      text.setLength(totalLength);
+      input.readChars(text.result, start, length);
+    } else {
+
+      if (dirty) {
+        // Fully convert all bytes since bytes is dirty
+        UnicodeUtil.UTF16toUTF8(text.result, 0, text.length, bytes);
+        bytes.setLength(totalLength);
+        input.readBytes(bytes.result, start, length);
+        UnicodeUtil.UTF8toUTF16(bytes.result, 0, totalLength, text);
+        dirty = false;
+      } else {
+        // Incrementally convert only the UTF8 bytes that are new:
+        bytes.setLength(totalLength);
+        input.readBytes(bytes.result, start, length);
+        UnicodeUtil.UTF8toUTF16(bytes.result, start, length, text);
+      }
+    }
+    this.field = fieldInfos.fieldName(input.readVInt());
+  }
+
+  public final void set(Term term) {
+    if (term == null) {
+      reset();
+      return;
+    }
+    final String termText = term.text();
+    final int termLen = termText.length();
+    text.setLength(termLen);
+    termText.getChars(0, termLen, text.result, 0);
+    dirty = true;
+    field = term.field();
+    this.term = term;
+  }
+
+  public final void set(TermBuffer other) {
+    text.copyText(other.text);
+    dirty = true;
+    field = other.field;
+    term = other.term;
+  }
+
+  public void reset() {
+    field = null;
+    text.setLength(0);
+    term = null;
+    dirty = true;
+  }
+
+  public Term toTerm() {
+    if (field == null)                            // unset
+      return null;
+
+    if (term == null)
+      term = new Term(field, new String(text.result, 0, text.length), false);
+
+    return term;
+  }
+
+  protected Object clone() {
+    TermBuffer clone = null;
+    try {
+      clone = (TermBuffer)super.clone();
+    } catch (CloneNotSupportedException e) {}
+
+    clone.dirty = true;
+    clone.bytes = new UnicodeUtil.UTF8Result();
+    clone.text = new UnicodeUtil.UTF16Result();
+    clone.text.copyText(text);
+    return clone;
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,62 @@
+package org.apache.lucene.index.codecs.preflex;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** A TermInfo is the record of information stored for a
+ * term
+ * @deprecated This class is no longer used in flexible
+ * indexing. */
+
+final class TermInfo {
+  /** The number of documents which contain the term. */
+  int docFreq = 0;
+
+  long freqPointer = 0;
+  long proxPointer = 0;
+  int skipOffset;
+
+  TermInfo() {}
+
+  TermInfo(int df, long fp, long pp) {
+    docFreq = df;
+    freqPointer = fp;
+    proxPointer = pp;
+  }
+
+  TermInfo(TermInfo ti) {
+    docFreq = ti.docFreq;
+    freqPointer = ti.freqPointer;
+    proxPointer = ti.proxPointer;
+    skipOffset = ti.skipOffset;
+  }
+
+  final void set(int docFreq,
+                 long freqPointer, long proxPointer, int skipOffset) {
+    this.docFreq = docFreq;
+    this.freqPointer = freqPointer;
+    this.proxPointer = proxPointer;
+    this.skipOffset = skipOffset;
+  }
+
+  final void set(TermInfo ti) {
+    docFreq = ti.docFreq;
+    freqPointer = ti.freqPointer;
+    proxPointer = ti.proxPointer;
+    skipOffset = ti.skipOffset;
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message