lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1367338 [2/2] - in /lucene/dev/branches/pforcodec_3892/lucene: core/src/java/org/apache/lucene/codecs/block/ core/src/java/org/apache/lucene/codecs/blockpacked/ core/src/java/org/apache/lucene/codecs/pfor/ core/src/resources/META-INF/servi...
Date Mon, 30 Jul 2012 22:15:07 GMT
Added: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java?rev=1367338&view=auto
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java
(added)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java
Mon Jul 30 22:15:06 2012
@@ -0,0 +1,205 @@
+package org.apache.lucene.codecs.blockpacked;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.lucene.codecs.MultiLevelSkipListReader;
+import org.apache.lucene.store.IndexInput;
+
+/**
+ * Implements the skip list reader for the 4.0 posting list format
+ * that stores positions and payloads.
+ * 
+ * @see Lucene40PostingsFormat
+ * @lucene.experimental
+ */
+final class BlockPackedSkipReader extends MultiLevelSkipListReader {
+  private boolean DEBUG = BlockPackedPostingsReader.DEBUG;
+
+  private long docPointer[];
+  private long posPointer[];
+  private long payPointer[];
+  private int posBufferUpto[];
+  private int endOffset[];
+  private int payloadByteUpto[];
+
+  private long lastPosPointer;
+  private long lastPayPointer;
+  private int lastEndOffset;
+  private int lastPayloadByteUpto;
+  private long lastDocPointer;
+  private int lastPosBufferUpto;
+
+  public BlockPackedSkipReader(IndexInput skipStream, int maxSkipLevels, int skipInterval,
boolean hasPos, boolean hasOffsets, boolean hasPayloads) {
+    super(skipStream, maxSkipLevels, skipInterval);
+    docPointer = new long[maxSkipLevels];
+    if (hasPos) {
+      posPointer = new long[maxSkipLevels];
+      posBufferUpto = new int[maxSkipLevels];
+      if (hasPayloads) {
+        payloadByteUpto = new int[maxSkipLevels];
+      } else {
+        payloadByteUpto = null;
+      }
+      if (hasOffsets) {
+        endOffset = new int[maxSkipLevels];
+      } else {
+        endOffset = null;
+      }
+      if (hasOffsets || hasPayloads) {
+        payPointer = new long[maxSkipLevels];
+      } else {
+        payPointer = null;
+      }
+    } else {
+      posPointer = null;
+    }
+  }
+
+  public void init(long skipPointer, long docBasePointer, long posBasePointer, long payBasePointer,
int df) {
+    super.init(skipPointer, df);
+    lastDocPointer = docBasePointer;
+    lastPosPointer = posBasePointer;
+    lastPayPointer = payBasePointer;
+
+    Arrays.fill(docPointer, docBasePointer);
+    if (posPointer != null) {
+      Arrays.fill(posPointer, posBasePointer);
+      if (payPointer != null) {
+        Arrays.fill(payPointer, payBasePointer);
+      }
+    } else {
+      assert posBasePointer == 0;
+    }
+  }
+
+  /** Returns the doc pointer of the doc to which the last call of 
+   * {@link MultiLevelSkipListReader#skipTo(int)} has skipped.  */
+  public long getDocPointer() {
+    return lastDocPointer;
+  }
+
+  public long getPosPointer() {
+    return lastPosPointer;
+  }
+
+  public int getPosBufferUpto() {
+    return lastPosBufferUpto;
+  }
+
+  public long getPayPointer() {
+    return lastPayPointer;
+  }
+
+  public int getEndOffset() {
+    return lastEndOffset;
+  }
+
+  public int getPayloadByteUpto() {
+    return lastPayloadByteUpto;
+  }
+
+  @Override
+  protected void seekChild(int level) throws IOException {
+    super.seekChild(level);
+    if (DEBUG) {
+      System.out.println("seekChild level=" + level);
+    }
+    docPointer[level] = lastDocPointer;
+    if (posPointer != null) {
+      posPointer[level] = lastPosPointer;
+      posBufferUpto[level] = lastPosBufferUpto;
+      if (endOffset != null) {
+        endOffset[level] = lastEndOffset;
+      }
+      if (payloadByteUpto != null) {
+        payloadByteUpto[level] = lastPayloadByteUpto;
+      }
+      if (payPointer != null) {
+        payPointer[level] = lastPayPointer;
+      }
+    }
+  }
+  
+  @Override
+  protected void setLastSkipData(int level) {
+    super.setLastSkipData(level);
+    lastDocPointer = docPointer[level];
+    if (DEBUG) {
+      System.out.println("setLastSkipData level=" + level);
+      System.out.println("  lastDocPointer=" + lastDocPointer);
+    }
+    if (posPointer != null) {
+      lastPosPointer = posPointer[level];
+      lastPosBufferUpto = posBufferUpto[level];
+      if (DEBUG) {
+        System.out.println("  lastPosPointer=" + lastPosPointer + " lastPosBUfferUpto=" +
lastPosBufferUpto);
+      }
+      if (payPointer != null) {
+        lastPayPointer = payPointer[level];
+      }
+      if (endOffset != null) {
+        lastEndOffset = endOffset[level];
+      }
+      if (payloadByteUpto != null) {
+        lastPayloadByteUpto = payloadByteUpto[level];
+      }
+    }
+  }
+
+  @Override
+  protected int readSkipData(int level, IndexInput skipStream) throws IOException {
+    if (DEBUG) {
+      System.out.println("readSkipData level=" + level);
+    }
+    int delta = skipStream.readVInt();
+    if (DEBUG) {
+      System.out.println("  delta=" + delta);
+    }
+    docPointer[level] += skipStream.readVInt();
+    if (DEBUG) {
+      System.out.println("  docFP=" + docPointer[level]);
+    }
+
+    if (posPointer != null) {
+      posPointer[level] += skipStream.readVInt();
+      if (DEBUG) {
+        System.out.println("  posFP=" + posPointer[level]);
+      }
+      posBufferUpto[level] = skipStream.readVInt();
+      if (DEBUG) {
+        System.out.println("  posBufferUpto=" + posBufferUpto[level]);
+      }
+
+      if (payloadByteUpto != null) {
+        payloadByteUpto[level] = skipStream.readVInt();
+      }
+
+      if (endOffset != null) {
+        endOffset[level] += skipStream.readVInt();
+      }
+
+      if (payPointer != null) {
+        payPointer[level] += skipStream.readVInt();
+      }
+    }
+    return delta;
+  }
+}

Added: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipWriter.java?rev=1367338&view=auto
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipWriter.java
(added)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipWriter.java
Mon Jul 30 22:15:06 2012
@@ -0,0 +1,147 @@
+package org.apache.lucene.codecs.blockpacked;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.codecs.MultiLevelSkipListWriter;
+
+// nocommit do we need more frequent skips at level > 0?
+// 128*128 is immense?  may need to decouple
+// baseSkipInterval & theRestSkipInterval?
+
+final class BlockPackedSkipWriter extends MultiLevelSkipListWriter {
+  private boolean DEBUG = BlockPackedPostingsReader.DEBUG;
+  
+  private int[] lastSkipDoc;
+  private long[] lastSkipDocPointer;
+  private long[] lastSkipPosPointer;
+  private long[] lastSkipPayPointer;
+  private int[] lastEndOffset;
+  private int[] lastPayloadByteUpto;
+
+  private final IndexOutput docOut;
+  private final IndexOutput posOut;
+  private final IndexOutput payOut;
+
+  private int curDoc;
+  private long curDocPointer;
+  private long curPosPointer;
+  private long curPayPointer;
+  private int curPosBufferUpto;
+  private int curEndOffset;
+  private int curPayloadByteUpto;
+  private boolean fieldHasPositions;
+  private boolean fieldHasOffsets;
+  private boolean fieldHasPayloads;
+
+  public BlockPackedSkipWriter(int skipInterval, int maxSkipLevels, int docCount, IndexOutput
docOut, IndexOutput posOut, IndexOutput payOut) {
+    super(skipInterval, maxSkipLevels, docCount);
+    this.docOut = docOut;
+    this.posOut = posOut;
+    this.payOut = payOut;
+    
+    lastSkipDoc = new int[maxSkipLevels];
+    lastSkipDocPointer = new long[maxSkipLevels];
+    if (posOut != null) {
+      lastSkipPosPointer = new long[maxSkipLevels];
+      if (payOut != null) {
+        lastSkipPayPointer = new long[maxSkipLevels];
+      }
+      lastEndOffset = new int[maxSkipLevels];
+      lastPayloadByteUpto = new int[maxSkipLevels];
+    }
+  }
+
+  public void setField(boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads)
{
+    this.fieldHasPositions = fieldHasPositions;
+    this.fieldHasOffsets = fieldHasOffsets;
+    this.fieldHasPayloads = fieldHasPayloads;
+  }
+
+  @Override
+  public void resetSkip() {
+    super.resetSkip();
+    Arrays.fill(lastSkipDoc, 0);
+    Arrays.fill(lastSkipDocPointer, docOut.getFilePointer());
+    if (fieldHasPositions) {
+      Arrays.fill(lastSkipPosPointer, posOut.getFilePointer());
+      if (fieldHasOffsets) {
+        Arrays.fill(lastEndOffset, 0);
+      }
+      if (fieldHasPayloads) {
+        Arrays.fill(lastPayloadByteUpto, 0);
+      }
+      if (fieldHasOffsets || fieldHasPayloads) {
+        Arrays.fill(lastSkipPayPointer, payOut.getFilePointer());
+      }
+    }
+  }
+
+  /**
+   * Sets the values for the current skip data. 
+   */
+  public void bufferSkip(int doc, int numDocs, long posFP, long payFP, int posBufferUpto,
int endOffset, int payloadByteUpto) throws IOException {
+    this.curDoc = doc;
+    this.curDocPointer = docOut.getFilePointer();
+    this.curPosPointer = posFP;
+    this.curPayPointer = payFP;
+    this.curPosBufferUpto = posBufferUpto;
+    this.curPayloadByteUpto = payloadByteUpto;
+    this.curEndOffset = endOffset;
+    bufferSkip(numDocs);
+  }
+  
+  @Override
+  protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException {
+    int delta = curDoc - lastSkipDoc[level];
+    if (DEBUG) {
+      System.out.println("writeSkipData level=" + level + " lastDoc=" + curDoc + " delta="
+ delta + " curDocPointer=" + curDocPointer);
+    }
+    skipBuffer.writeVInt(delta);
+    lastSkipDoc[level] = curDoc;
+
+    skipBuffer.writeVInt((int) (curDocPointer - lastSkipDocPointer[level]));
+    lastSkipDocPointer[level] = curDocPointer;
+
+    if (fieldHasPositions) {
+      if (DEBUG) {
+        System.out.println("  curPosPointer=" + curPosPointer + " curPosBufferUpto=" + curPosBufferUpto);
+      }
+      skipBuffer.writeVInt((int) (curPosPointer - lastSkipPosPointer[level]));
+      lastSkipPosPointer[level] = curPosPointer;
+      skipBuffer.writeVInt(curPosBufferUpto);
+
+      if (fieldHasPayloads) {
+        skipBuffer.writeVInt(curPayloadByteUpto);
+      }
+
+      if (fieldHasOffsets) {
+        skipBuffer.writeVInt(curEndOffset - lastEndOffset[level]);
+        lastEndOffset[level] = curEndOffset;
+      }
+
+      if (fieldHasOffsets || fieldHasPayloads) {
+        skipBuffer.writeVInt((int) (curPayPointer - lastSkipPayPointer[level]));
+        lastSkipPayPointer[level] = curPayPointer;
+      }
+    }
+  }
+}

Added: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/ForUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/ForUtil.java?rev=1367338&view=auto
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/ForUtil.java
(added)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/ForUtil.java
Mon Jul 30 22:15:06 2012
@@ -0,0 +1,130 @@
+package org.apache.lucene.codecs.blockpacked;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.nio.LongBuffer;
+import java.nio.IntBuffer;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PackedInts.Reader;
+import org.apache.lucene.util.packed.PackedInts.Writer;
+import org.apache.lucene.util.packed.PackedInts.Mutable;
+import org.apache.lucene.util.packed.PackedInts.Encoder;
+import org.apache.lucene.util.packed.PackedInts.Decoder;
+
+/**
+ * Encode all values in normal area with fixed bit width, 
+ * which is determined by the max value in this block.
+ */
+public class ForUtil {
+  protected static final int[] MASK = {   0x00000000,
+    0x00000001, 0x00000003, 0x00000007, 0x0000000f, 0x0000001f, 0x0000003f,
+    0x0000007f, 0x000000ff, 0x000001ff, 0x000003ff, 0x000007ff, 0x00000fff,
+    0x00001fff, 0x00003fff, 0x00007fff, 0x0000ffff, 0x0001ffff, 0x0003ffff,
+    0x0007ffff, 0x000fffff, 0x001fffff, 0x003fffff, 0x007fffff, 0x00ffffff,
+    0x01ffffff, 0x03ffffff, 0x07ffffff, 0x0fffffff, 0x1fffffff, 0x3fffffff,
+    0x7fffffff, 0xffffffff};
+
+  /** Compress given int[] into output stream, with For format
+   */
+  public static int compress(final LongBuffer data, LongBuffer packed) throws IOException
{
+    int numBits=getNumBits(data.array());
+
+    if (numBits == 0) { // when block is equal, save the value once
+      packed.put(0, data.get(0)<<32); // java uses big endian for LongBuffer impl 
+      return (getHeader(1,numBits));
+    }
+
+    PackedInts.Format format = PackedInts.fastestFormatAndBits(128, numBits, PackedInts.FASTEST).format;
+    PackedInts.Encoder encoder = PackedInts.getEncoder(format, PackedInts.VERSION_CURRENT,
numBits);
+    int perIter = encoder.values();
+    int iters = 128/perIter;
+    int nblocks = encoder.blocks()*iters;
+    assert 128 % perIter == 0;
+
+    packed.rewind();
+    data.rewind();
+
+    encoder.encode(data, packed, iters);
+
+    int encodedSize = nblocks*2;
+    return getHeader(encodedSize,numBits);
+  }
+
+  /** Decompress given ouput stream into int array.
+   */
+  public static void decompress(LongBuffer data, LongBuffer packed, int header) throws IOException
{
+    // nocommit assert header isn't "malformed", ie besides
+    // numBytes / bit-width there is nothing else!
+    
+    packed.rewind();
+    data.rewind();
+    int numBits = ((header >> 8) & MASK[6]);
+
+    if (numBits == 0) {
+      Arrays.fill(data.array(), (int)(packed.get(0)>>>32));
+      return;
+    }
+
+    PackedInts.Format format = PackedInts.fastestFormatAndBits(128, numBits, PackedInts.FASTEST).format;
+    PackedInts.Decoder decoder = PackedInts.getDecoder(format, PackedInts.VERSION_CURRENT,
numBits);
+    int perIter = decoder.values();
+    int iters = 128/perIter;
+    int nblocks = decoder.blocks()*iters;
+    assert 128 % perIter == 0;
+
+    decoder.decode(packed, data, iters);
+  }
+
+  static int getNumBits(final long[] data) {
+    if (isAllEqual(data)) {
+      return 0;
+    }
+    int size=data.length;
+    int optBits=1;
+    for (int i=0; i<size; ++i) {
+      while ((data[i] & ~MASK[optBits]) != 0) {
+        optBits++;
+      }
+    }
+    return optBits;
+  }
+
+  protected static boolean isAllEqual(final long[] data) {
+    int len = data.length;
+    long v = data[0];
+    for (int i=1; i<len; i++) {
+      if (data[i] != v) {
+        return false;
+      }
+    }
+    return true;
+  }
+  static int getHeader(int encodedSize, int numBits) {
+    return  (encodedSize)
+          | ((numBits) << 8);
+  }
+  public static int getEncodedSize(int header) {
+    return ((header & MASK[8]))*4;
+  }
+  public static int getNumBits(int header) {
+    return ((header >> 8) & MASK[6]);
+  }
+}

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1367338&r1=1367337&r2=1367338&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
(original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
Mon Jul 30 22:15:06 2012
@@ -17,8 +17,6 @@ org.apache.lucene.codecs.lucene40.Lucene
 org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat
 org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat
 org.apache.lucene.codecs.memory.MemoryPostingsFormat
-org.apache.lucene.codecs.pfor.ForPostingsFormat
-org.apache.lucene.codecs.pfor.PForPostingsFormat
 org.apache.lucene.codecs.bulkvint.BulkVIntPostingsFormat
 org.apache.lucene.codecs.block.BlockPostingsFormat
 org.apache.lucene.codecs.memory.DirectPostingsFormat

Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1367338&r1=1367337&r2=1367338&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
(original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
Mon Jul 30 22:15:06 2012
@@ -61,7 +61,6 @@ import org.apache.lucene.store.IndexOutp
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.codecs.pfor.*;
 
 /**
  * Randomly combines terms index impl w/ postings impls.
@@ -103,8 +102,6 @@ public class MockRandomPostingsFormat ex
       final int baseBlockSize = _TestUtil.nextInt(random, 1, 127);
       delegates.add(new MockVariableIntBlockPostingsFormat.MockIntFactory(baseBlockSize));
       // TODO: others
-      delegates.add(new ForFactory());
-      delegates.add(new PForFactory());
     }
 
     private static String getExtension(String fileName) {

Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1367338&r1=1367337&r2=1367338&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
(original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
Mon Jul 30 22:15:06 2012
@@ -282,9 +282,7 @@ public abstract class LuceneTestCase ext
     "MockFixedIntBlock",
     "MockVariableIntBlock",
     "MockSep",
-    "MockRandom",
-    "For",
-    "PFor"
+    "MockRandom"
   ));
   
   // -----------------------------------------------------------------



Mime
View raw message