lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1049480 [1/3] - in /lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene: index/codecs/ index/codecs/pfordelta/ store/ util/pfor/
Date Wed, 15 Dec 2010 10:44:07 GMT
Author: mikemccand
Date: Wed Dec 15 10:44:06 2010
New Revision: 1049480

URL: http://svn.apache.org/viewvc?rev=1049480&view=rev
Log:
LUCENE-1410: commit prototype FOR/PFOR codec on bulk postings branch

Added:
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORFactory.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexInput.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexOutput.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FrameOfRefCodec.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaFactory.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexInput.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexOutput.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PatchedFrameOfRefCodec.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For10Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For11Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For12Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For13Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For14Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For15Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For16Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For17Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For18Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For19Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For1Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For20Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For21Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For22Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For23Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For24Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For25Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For26Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For27Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For28Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For29Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For2Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For30Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For31Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For32Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For3Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For4Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For5Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For6Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For7Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For8Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For9Decompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/ForDecompress.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/FrameOfRef.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/PFor.java   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/gendecompress.py   (with props)
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/package.html   (with props)
Modified:
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java
    lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/store/MMapDirectory.java

Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java?rev=1049480&r1=1049479&r2=1049480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java Wed Dec 15 10:44:06 2010
@@ -27,6 +27,8 @@ import org.apache.lucene.index.codecs.pr
 import org.apache.lucene.index.codecs.pulsing.PulsingCodec;
 import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec;
 import org.apache.lucene.index.codecs.standard.StandardCodec;
+import org.apache.lucene.index.codecs.pfordelta.PatchedFrameOfRefCodec;
+import org.apache.lucene.index.codecs.pfordelta.FrameOfRefCodec;
 
 /** Holds a set of codecs, keyed by name.  You subclass
  *  this, instantiate it, and register your codecs, then
@@ -47,8 +49,7 @@ public class CodecProvider {
 
   private final Set<String> knownExtensions = new HashSet<String>();
 
-
-  public final static String[] CORE_CODECS = new String[] {"Standard", "Pulsing", "PreFlex", "SimpleText"};
+  public final static String[] CORE_CODECS = new String[] {"Standard", "Pulsing", "PreFlex", "SimpleText", "PatchedFrameOfRef", "FrameOfRef"};
 
   public synchronized void register(Codec codec) {
     if (codec.name == null) {
@@ -171,5 +172,7 @@ class DefaultCodecProvider extends Codec
     register(new PreFlexCodec());
     register(new PulsingCodec(1));
     register(new SimpleTextCodec());
+    register(new PatchedFrameOfRefCodec());
+    register(new FrameOfRefCodec());
   }
 }

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORFactory.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORFactory.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORFactory.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,43 @@
+package org.apache.lucene.index.codecs.pfordelta;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.index.codecs.sep.IntStreamFactory;
+import org.apache.lucene.index.codecs.sep.IntIndexInput;
+import org.apache.lucene.index.codecs.sep.IntIndexOutput;
+
+import java.io.IOException;
+
+public class FORFactory extends IntStreamFactory {
+  private final int blockSize;
+
+  /** blockSize is only used when creating the
+   *  IntIndexOutput */
+  public FORFactory(int blockSize) {
+    this.blockSize = blockSize;
+  }
+
+  public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException {
+    return new FORIndexInput(dir, fileName, readBufferSize);
+  }
+
+  public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException {
+    return new FORIndexOutput(dir, fileName, blockSize);
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexInput.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexInput.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexInput.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,175 @@
+package org.apache.lucene.index.codecs.pfordelta;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.MMapDirectory.MMapIndexInput;
+import org.apache.lucene.store.MMapDirectory.MultiMMapIndexInput;
+import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput;
+import org.apache.lucene.util.pfor.FrameOfRef;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.IntBuffer;
+
+public class FORIndexInput extends FixedIntBlockIndexInput {
+
+  public FORIndexInput(Directory dir, String fileName, int readBufferSize) throws IOException {
+    super(dir.openInput(fileName, readBufferSize));
+  }
+
+  private static class BlockReader implements FixedIntBlockIndexInput.BlockReader {
+    private final IndexInput in;
+    private final int[] buffer;
+    private final FrameOfRef decompressor;
+    private final byte[] input;
+    private final IntBuffer intInput;
+
+    public BlockReader(IndexInput in, int[] buffer) {
+      this.in = in;
+      this.buffer = buffer;
+
+      decompressor = new FrameOfRef();
+      // nocommit -- can't hardwire 1024; it's a function of blockSize
+      ByteBuffer byteBuffer = ByteBuffer.allocate(1024);
+      input = byteBuffer.array();
+      intInput = byteBuffer.asIntBuffer();
+      decompressor.setCompressedBuffer(intInput);
+      decompressor.setUnCompressedData(buffer, 0, buffer.length);
+    }
+
+    public void seek(long pos) throws IOException {
+      //System.out.println("for: seek pos=" + pos);
+    }
+
+    public void readBlock() throws IOException {
+      // nocommit -- we don't need this numBytes header --
+      // it's a waste.  we need something like the zip
+      // interface -- the decompressor asks for more bytes
+      // if it needs it
+      //System.out.println("for: read @ fp=" + in.getFilePointer());
+      int numBytes = in.readInt();
+      // nocommit -- how to avoid this copy?  plus, the copy
+      // inside BII.  if mmapdir how can we directly access
+      // underlying ram w/ no copy?
+      in.readBytes(input, 0, numBytes);
+      intInput.rewind();
+      decompressor.decompress();
+      //System.out.println("  FOR.readBlock");
+    }
+  }
+
+  // nocommit -- hacked up -- maybe move
+  // IntIndexInput/Output into store?
+  private static class MMapBlockReader implements FixedIntBlockIndexInput.BlockReader {
+    private final MMapIndexInput in;
+    private final int[] buffer;
+    private final FrameOfRef decompressor;
+    private final ByteBuffer byteBufferIn;
+    private final IntBuffer intBufferIn;
+
+    public MMapBlockReader(MMapIndexInput in, int[] buffer) {
+      this.in = in;
+      this.buffer = buffer;
+
+      // nocommit -- we seem to be creating this class
+      // alot... how come?
+      decompressor = new FrameOfRef();
+      
+      byteBufferIn = in.getBuffer();
+      //System.out.println("pos start=" + byteBufferIn.position());
+      // nocommit -- this sets pos relative to current pos
+      // of byteBuffer (confusing) -- this is why we -1 in
+      // seek (pos is always 4 here)
+      intBufferIn = byteBufferIn.asIntBuffer();
+      
+      decompressor.setCompressedBuffer(intBufferIn);
+      decompressor.setUnCompressedData(buffer, 0, buffer.length);
+      //System.out.println("mmmap");
+    }
+
+    public void seek(long pos) throws IOException {
+      assert pos%4 == 0;
+      //System.out.println("this=" + this + " seek=" + pos);
+      intBufferIn.position((int) (pos/4)-1);
+    }
+
+    public void readBlock() throws IOException {
+      // nocommit -- we don't need this numBytes header --
+      // it's a waste.  we need something like the zip
+      // interface -- the decompressor asks for more bytes
+      // if it needs it
+      //int numBytes = in.readInt();
+      int numBytes = intBufferIn.get();
+      //System.out.println("NB=" + numBytes);
+      // nocommit -- how to avoid this copy?  plus, the copy
+      // inside BII.  if mmapdir how can we directly access
+      // underlying ram w/ no copy?
+      //in.readBytes(input, 0, numBytes);
+      decompressor.decompress();
+    }
+  }
+
+  // nocommit -- hacked up -- maybe move
+  // IntIndexInput/Output into store?
+  private static class MultiMMapBlockReader implements FixedIntBlockIndexInput.BlockReader {
+    private final MultiMMapIndexInput in;
+    private final int[] buffer;
+    private final FrameOfRef decompressor;
+    private final ByteBuffer byteBufferIn;
+    private final IntBuffer intBufferIn;
+
+    public MultiMMapBlockReader(MultiMMapIndexInput in, int[] buffer) {
+      this.in = in;
+      this.buffer = buffer;
+
+      decompressor = new FrameOfRef();
+      
+      byteBufferIn = in.getBuffer();
+      intBufferIn = byteBufferIn.asIntBuffer();
+      
+      decompressor.setCompressedBuffer(intBufferIn);
+      decompressor.setUnCompressedData(buffer, 0, buffer.length);
+    }
+
+    public void seek(long pos) throws IOException {
+      //
+      
+    }
+
+    public void readBlock() throws IOException {
+      int numBytes = in.readInt();
+      // nocommit -- how to avoid this copy?  plus, the copy
+      // inside BII.  if mmapdir how can we directly access
+      // underlying ram w/ no copy?
+      //in.readBytes(input, 0, numBytes);
+      //decompressor.decompress();
+    }
+  }
+
+  protected FixedIntBlockIndexInput.BlockReader getBlockReader(IndexInput in, int[] buffer) {
+    if (in instanceof MMapIndexInput) {
+      return new MMapBlockReader((MMapIndexInput) in, buffer);
+    } else if (false && in instanceof MultiMMapIndexInput) {
+      return new MultiMMapBlockReader((MultiMMapIndexInput) in, buffer);
+    } else {
+      return new BlockReader(in, buffer);
+    }
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexOutput.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexOutput.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexOutput.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,66 @@
+package org.apache.lucene.index.codecs.pfordelta;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Naive int block API that writes vInts.  This is
+ *  expected to give poor performance; it's really only for
+ *  testing the pluggability.  One should typically use pfor instead. */
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput;
+import org.apache.lucene.util.pfor.FrameOfRef;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/** Don't use this class!!  It naively encodes ints one vInt
+ *  at a time.  Use it only for testing.  */
+public class FORIndexOutput extends FixedIntBlockIndexOutput {
+
+  public final static String CODEC = "P_FOR_DELTA";
+  public final static int VERSION_START = 0;
+  public final static int VERSION_CURRENT = VERSION_START;
+  private final FrameOfRef compressor;
+  private final byte[] output;
+
+  // nocommit
+  private final String fileName;
+
+  public FORIndexOutput(Directory dir, String fileName, int blockSize) throws IOException {
+    super(dir.createOutput(fileName), blockSize);
+
+    // nocommit
+    this.fileName = fileName;
+    compressor = new FrameOfRef();
+    // nocommit -- can't hardwire 1024; it's a function of blockSize
+    ByteBuffer byteBuffer = ByteBuffer.allocate(1024);
+    output = byteBuffer.array();
+    compressor.setCompressedBuffer(byteBuffer.asIntBuffer());
+  }
+
+  @Override
+  protected void flushBlock() throws IOException {
+    compressor.setUnCompressedData(buffer, 0, buffer.length);
+    final int numFrameBits = compressor.frameBitsForCompression();
+    compressor.compress();
+    final int numBytes = compressor.compressedSize() * 4;
+    assert numBytes <= 1024;
+    out.writeInt(numBytes);
+    out.writeBytes(output, numBytes);
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FrameOfRefCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FrameOfRefCodec.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FrameOfRefCodec.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FrameOfRefCodec.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,141 @@
+package org.apache.lucene.index.codecs.pfordelta;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Set;
+import java.io.IOException;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.index.codecs.FieldsConsumer;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
+import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
+import org.apache.lucene.index.codecs.standard.StandardCodec;
+import org.apache.lucene.index.codecs.PrefixCodedTermsWriter;
+import org.apache.lucene.index.codecs.PrefixCodedTermsReader;
+import org.apache.lucene.index.codecs.TermsIndexWriterBase;
+import org.apache.lucene.index.codecs.TermsIndexReaderBase;
+import org.apache.lucene.index.codecs.PostingsReaderBase;
+import org.apache.lucene.index.codecs.PostingsWriterBase;
+import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
+
+public class FrameOfRefCodec extends Codec {
+
+  public FrameOfRefCodec() {
+    name = "FrameOfRef";
+  }
+
+  @Override
+  public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+    PostingsWriterBase postingsWriter = new SepPostingsWriterImpl(state, new FORFactory(128));
+
+    boolean success = false;
+    TermsIndexWriterBase indexWriter;
+    try {
+      indexWriter = new FixedGapTermsIndexWriter(state);
+      success = true;
+    } finally {
+      if (!success) {
+        postingsWriter.close();
+      }
+    }
+
+    success = false;
+    try {
+      FieldsConsumer ret = new PrefixCodedTermsWriter(indexWriter, state, postingsWriter, BytesRef.getUTF8SortedAsUnicodeComparator());
+      success = true;
+      return ret;
+    } finally {
+      if (!success) {
+        try {
+          postingsWriter.close();
+        } finally {
+          indexWriter.close();
+        }
+      }
+    }
+  }
+
+  @Override
+  public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+    PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir,
+                                                                  state.segmentInfo,
+                                                                  state.readBufferSize,
+                                                                  new FORFactory(128),
+                                                                  state.codecId);
+
+    TermsIndexReaderBase indexReader;
+    boolean success = false;
+    try {
+      indexReader = new FixedGapTermsIndexReader(state.dir,
+                                                 state.fieldInfos,
+                                                 state.segmentInfo.name,
+                                                 state.termsIndexDivisor,
+                                                 BytesRef.getUTF8SortedAsUnicodeComparator(),
+                                                 state.codecId);
+      success = true;
+    } finally {
+      if (!success) {
+        postingsReader.close();
+      }
+    }
+
+    success = false;
+    try {
+      FieldsProducer ret = new PrefixCodedTermsReader(indexReader,
+                                                      state.dir,
+                                                      state.fieldInfos,
+                                                      state.segmentInfo.name,
+                                                      postingsReader,
+                                                      state.readBufferSize,
+                                                      BytesRef.getUTF8SortedAsUnicodeComparator(),
+                                                      StandardCodec.TERMS_CACHE_SIZE,
+                                                      state.codecId);
+      success = true;
+      return ret;
+    } finally {
+      if (!success) {
+        try {
+          postingsReader.close();
+        } finally {
+          indexReader.close();
+        }
+      }
+    }
+  }
+
+  @Override
+  public void files(Directory dir, SegmentInfo segmentInfo, String id, Set<String> files) {
+    SepPostingsReaderImpl.files(segmentInfo, id, files);
+    PrefixCodedTermsReader.files(dir, segmentInfo, id, files);
+    FixedGapTermsIndexReader.files(dir, segmentInfo, id, files);
+  }
+
+  @Override
+  public void getExtensions(Set<String> extensions) {
+    SepPostingsWriterImpl.getExtensions(extensions);
+    PrefixCodedTermsReader.getExtensions(extensions);
+    FixedGapTermsIndexReader.getIndexExtensions(extensions);
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaFactory.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaFactory.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaFactory.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,43 @@
+package org.apache.lucene.index.codecs.pfordelta;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.index.codecs.sep.IntStreamFactory;
+import org.apache.lucene.index.codecs.sep.IntIndexInput;
+import org.apache.lucene.index.codecs.sep.IntIndexOutput;
+
+import java.io.IOException;
+
+public class PForDeltaFactory extends IntStreamFactory {
+  private final int blockSize;
+
+  /** blockSize is only used when creating the
+   *  IntIndexOutput */
+  public PForDeltaFactory(int blockSize) {
+    this.blockSize = blockSize;
+  }
+
+  public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException {
+    return new PForDeltaIndexInput(dir, fileName, readBufferSize);
+  }
+
+  public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException {
+    return new PForDeltaIndexOutput(dir, fileName, blockSize);
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexInput.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexInput.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexInput.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,73 @@
+package org.apache.lucene.index.codecs.pfordelta;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput;
+import org.apache.lucene.util.pfor.PFor;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.IntBuffer;
+
+public class PForDeltaIndexInput extends FixedIntBlockIndexInput {
+
+  public PForDeltaIndexInput(Directory dir, String fileName, int readBufferSize) throws IOException {
+    super(dir.openInput(fileName, readBufferSize));
+  }
+
+  private static class BlockReader implements FixedIntBlockIndexInput.BlockReader {
+    private final IndexInput in;
+    private final int[] buffer;
+    private final PFor decompressor;
+    private final byte[] input;
+    private final IntBuffer intInput;
+
+    public BlockReader(IndexInput in, int[] buffer) {
+      this.in = in;
+      this.buffer = buffer;
+
+      decompressor = new PFor();
+      // nocommit -- can't hardwire 1024; it's a function of blockSize
+      ByteBuffer byteBuffer = ByteBuffer.allocate(1024);
+      input = byteBuffer.array();
+      intInput = byteBuffer.asIntBuffer();
+      decompressor.setCompressedBuffer(intInput);
+      decompressor.setUnCompressedData(buffer, 0, buffer.length);
+    }
+
+    public void seek(long pos) throws IOException {
+      //
+    }
+
+    public void readBlock() throws IOException {
+      int numBytes = in.readInt();
+      //System.out.println("nb=" + numBytes);
+      // nocommit -- how to avoid this copy?
+      in.readBytes(input, 0, numBytes);
+      intInput.rewind();
+      decompressor.decompress();
+    }
+  }
+
+  protected BlockReader getBlockReader(IndexInput in, int[] buffer) {
+    return new BlockReader(in, buffer);
+  }
+}
+

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexOutput.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexOutput.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexOutput.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,56 @@
+package org.apache.lucene.index.codecs.pfordelta;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput;
+import org.apache.lucene.util.pfor.PFor;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class PForDeltaIndexOutput extends FixedIntBlockIndexOutput {
+
+  public final static String CODEC = "P_FOR_DELTA";
+  public final static int VERSION_START = 0;
+  public final static int VERSION_CURRENT = VERSION_START;
+  private final PFor compressor;
+  private final byte[] output;
+
+  public PForDeltaIndexOutput(Directory dir, String fileName, int blockSize) throws IOException {
+    super(dir.createOutput(fileName), blockSize);
+
+    compressor = new PFor();
+    // nocommit -- can't hardwire 1024; it's a function of blockSize
+    ByteBuffer byteBuffer = ByteBuffer.allocate(1024);
+    output = byteBuffer.array();
+    compressor.setCompressedBuffer(byteBuffer.asIntBuffer());
+  }
+
+  @Override
+  protected void flushBlock() throws IOException {
+    compressor.setUnCompressedData(buffer, 0, buffer.length);
+    final int numFrameBits = compressor.frameBitsForCompression();
+    compressor.compress();
+    final int numBytes = compressor.compressedSize() * 4;
+    assert numBytes <= 1024;
+    out.writeInt(numBytes);
+    out.writeBytes(output, numBytes);
+  }
+}
+

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PatchedFrameOfRefCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PatchedFrameOfRefCodec.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PatchedFrameOfRefCodec.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PatchedFrameOfRefCodec.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,141 @@
+package org.apache.lucene.index.codecs.pfordelta;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Set;
+import java.io.IOException;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.index.codecs.FieldsConsumer;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
+import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
+import org.apache.lucene.index.codecs.standard.StandardCodec;
+import org.apache.lucene.index.codecs.PrefixCodedTermsWriter;
+import org.apache.lucene.index.codecs.PrefixCodedTermsReader;
+import org.apache.lucene.index.codecs.TermsIndexWriterBase;
+import org.apache.lucene.index.codecs.TermsIndexReaderBase;
+import org.apache.lucene.index.codecs.PostingsReaderBase;
+import org.apache.lucene.index.codecs.PostingsWriterBase;
+import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
+
+public class PatchedFrameOfRefCodec extends Codec {
+
+  public PatchedFrameOfRefCodec() {
+    name = "PatchedFrameOfRef";
+  }
+
+  @Override
+  public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+    PostingsWriterBase postingsWriter = new SepPostingsWriterImpl(state, new PForDeltaFactory(128));
+
+    boolean success = false;
+    TermsIndexWriterBase indexWriter;
+    try {
+      indexWriter = new FixedGapTermsIndexWriter(state);
+      success = true;
+    } finally {
+      if (!success) {
+        postingsWriter.close();
+      }
+    }
+
+    success = false;
+    try {
+      FieldsConsumer ret = new PrefixCodedTermsWriter(indexWriter, state, postingsWriter, BytesRef.getUTF8SortedAsUnicodeComparator());
+      success = true;
+      return ret;
+    } finally {
+      if (!success) {
+        try {
+          postingsWriter.close();
+        } finally {
+          indexWriter.close();
+        }
+      }
+    }
+  }
+
+  @Override
+  public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+    PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir,
+                                                                  state.segmentInfo,
+                                                                  state.readBufferSize,
+                                                                  new PForDeltaFactory(128),
+                                                                  state.codecId);
+
+    TermsIndexReaderBase indexReader;
+    boolean success = false;
+    try {
+      indexReader = new FixedGapTermsIndexReader(state.dir,
+                                                 state.fieldInfos,
+                                                 state.segmentInfo.name,
+                                                 state.termsIndexDivisor,
+                                                 BytesRef.getUTF8SortedAsUnicodeComparator(),
+                                                 state.codecId);
+      success = true;
+    } finally {
+      if (!success) {
+        postingsReader.close();
+      }
+    }
+
+    success = false;
+    try {
+      FieldsProducer ret = new PrefixCodedTermsReader(indexReader,
+                                                      state.dir,
+                                                      state.fieldInfos,
+                                                      state.segmentInfo.name,
+                                                      postingsReader,
+                                                      state.readBufferSize,
+                                                      BytesRef.getUTF8SortedAsUnicodeComparator(),
+                                                      StandardCodec.TERMS_CACHE_SIZE,
+                                                      state.codecId);
+      success = true;
+      return ret;
+    } finally {
+      if (!success) {
+        try {
+          postingsReader.close();
+        } finally {
+          indexReader.close();
+        }
+      }
+    }
+  }
+
+  @Override
+  public void files(Directory dir, SegmentInfo segmentInfo, String id, Set<String> files) {
+    SepPostingsReaderImpl.files(segmentInfo, id, files);
+    PrefixCodedTermsReader.files(dir, segmentInfo, id, files);
+    FixedGapTermsIndexReader.files(dir, segmentInfo, id, files);
+  }
+
+  @Override
+  public void getExtensions(Set<String> extensions) {
+    SepPostingsWriterImpl.getExtensions(extensions);
+    PrefixCodedTermsReader.getExtensions(extensions);
+    FixedGapTermsIndexReader.getIndexExtensions(extensions);
+  }
+}

Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/store/MMapDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/store/MMapDirectory.java?rev=1049480&r1=1049479&r2=1049480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/store/MMapDirectory.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/store/MMapDirectory.java Wed Dec 15 10:44:06 2010
@@ -214,7 +214,8 @@ public class MMapDirectory extends FSDir
     }
   }
 
-  private class MMapIndexInput extends IndexInput {
+  // nocommit make private again
+  public class MMapIndexInput extends IndexInput {
 
     private ByteBuffer buffer;
     private final long length;
@@ -225,6 +226,11 @@ public class MMapDirectory extends FSDir
         this.buffer = raf.getChannel().map(MapMode.READ_ONLY, 0, length);
     }
 
+    // nocommit
+    public ByteBuffer getBuffer() {
+      return buffer;
+    }
+
     @Override
     public byte readByte() throws IOException {
       try {
@@ -283,7 +289,8 @@ public class MMapDirectory extends FSDir
   // Because Java's ByteBuffer uses an int to address the
   // values, it's necessary to access a file >
   // Integer.MAX_VALUE in size using multiple byte buffers.
-  private class MultiMMapIndexInput extends IndexInput {
+  // nocommit make private again
+  public class MultiMMapIndexInput extends IndexInput {
   
     private ByteBuffer[] buffers;
     private int[] bufSizes; // keep here, ByteBuffer.size() method is optional
@@ -331,6 +338,11 @@ public class MMapDirectory extends FSDir
       seek(0L);
     }
   
+    public ByteBuffer getBuffer() {
+      // nocommit fixup
+      return null;
+    }
+
     @Override
     public byte readByte() throws IOException {
       // Performance might be improved by reading ahead into an array of

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For10Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For10Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For10Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For10Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,81 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For10Decompress extends ForDecompress {
+  static final int numFrameBits = 10;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      int intValue5 = compressedBuffer.get();
+      int intValue6 = compressedBuffer.get();
+      int intValue7 = compressedBuffer.get();
+      int intValue8 = compressedBuffer.get();
+      int intValue9 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = (intValue0 >>> 10) & mask;
+      output[2 + outputOffset] = (intValue0 >>> 20) & mask;
+      output[3 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask;
+      output[4 + outputOffset] = (intValue1 >>> 8) & mask;
+      output[5 + outputOffset] = (intValue1 >>> 18) & mask;
+      output[6 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask;
+      output[7 + outputOffset] = (intValue2 >>> 6) & mask;
+      output[8 + outputOffset] = (intValue2 >>> 16) & mask;
+      output[9 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask;
+      output[10 + outputOffset] = (intValue3 >>> 4) & mask;
+      output[11 + outputOffset] = (intValue3 >>> 14) & mask;
+      output[12 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask;
+      output[13 + outputOffset] = (intValue4 >>> 2) & mask;
+      output[14 + outputOffset] = (intValue4 >>> 12) & mask;
+      output[15 + outputOffset] = intValue4 >>> 22;
+      output[16 + outputOffset] = intValue5 & mask;
+      output[17 + outputOffset] = (intValue5 >>> 10) & mask;
+      output[18 + outputOffset] = (intValue5 >>> 20) & mask;
+      output[19 + outputOffset] = ((intValue5 >>> 30) | (intValue6 << 2)) & mask;
+      output[20 + outputOffset] = (intValue6 >>> 8) & mask;
+      output[21 + outputOffset] = (intValue6 >>> 18) & mask;
+      output[22 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask;
+      output[23 + outputOffset] = (intValue7 >>> 6) & mask;
+      output[24 + outputOffset] = (intValue7 >>> 16) & mask;
+      output[25 + outputOffset] = ((intValue7 >>> 26) | (intValue8 << 6)) & mask;
+      output[26 + outputOffset] = (intValue8 >>> 4) & mask;
+      output[27 + outputOffset] = (intValue8 >>> 14) & mask;
+      output[28 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask;
+      output[29 + outputOffset] = (intValue9 >>> 2) & mask;
+      output[30 + outputOffset] = (intValue9 >>> 12) & mask;
+      output[31 + outputOffset] = intValue9 >>> 22;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For11Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For11Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For11Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For11Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,82 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For11Decompress extends ForDecompress {
+  static final int numFrameBits = 11;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      int intValue5 = compressedBuffer.get();
+      int intValue6 = compressedBuffer.get();
+      int intValue7 = compressedBuffer.get();
+      int intValue8 = compressedBuffer.get();
+      int intValue9 = compressedBuffer.get();
+      int intValue10 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = (intValue0 >>> 11) & mask;
+      output[2 + outputOffset] = ((intValue0 >>> 22) | (intValue1 << 10)) & mask;
+      output[3 + outputOffset] = (intValue1 >>> 1) & mask;
+      output[4 + outputOffset] = (intValue1 >>> 12) & mask;
+      output[5 + outputOffset] = ((intValue1 >>> 23) | (intValue2 << 9)) & mask;
+      output[6 + outputOffset] = (intValue2 >>> 2) & mask;
+      output[7 + outputOffset] = (intValue2 >>> 13) & mask;
+      output[8 + outputOffset] = ((intValue2 >>> 24) | (intValue3 << 8)) & mask;
+      output[9 + outputOffset] = (intValue3 >>> 3) & mask;
+      output[10 + outputOffset] = (intValue3 >>> 14) & mask;
+      output[11 + outputOffset] = ((intValue3 >>> 25) | (intValue4 << 7)) & mask;
+      output[12 + outputOffset] = (intValue4 >>> 4) & mask;
+      output[13 + outputOffset] = (intValue4 >>> 15) & mask;
+      output[14 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask;
+      output[15 + outputOffset] = (intValue5 >>> 5) & mask;
+      output[16 + outputOffset] = (intValue5 >>> 16) & mask;
+      output[17 + outputOffset] = ((intValue5 >>> 27) | (intValue6 << 5)) & mask;
+      output[18 + outputOffset] = (intValue6 >>> 6) & mask;
+      output[19 + outputOffset] = (intValue6 >>> 17) & mask;
+      output[20 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask;
+      output[21 + outputOffset] = (intValue7 >>> 7) & mask;
+      output[22 + outputOffset] = (intValue7 >>> 18) & mask;
+      output[23 + outputOffset] = ((intValue7 >>> 29) | (intValue8 << 3)) & mask;
+      output[24 + outputOffset] = (intValue8 >>> 8) & mask;
+      output[25 + outputOffset] = (intValue8 >>> 19) & mask;
+      output[26 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask;
+      output[27 + outputOffset] = (intValue9 >>> 9) & mask;
+      output[28 + outputOffset] = (intValue9 >>> 20) & mask;
+      output[29 + outputOffset] = ((intValue9 >>> 31) | (intValue10 << 1)) & mask;
+      output[30 + outputOffset] = (intValue10 >>> 10) & mask;
+      output[31 + outputOffset] = intValue10 >>> 21;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For12Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For12Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For12Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For12Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,83 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For12Decompress extends ForDecompress {
+  static final int numFrameBits = 12;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      int intValue5 = compressedBuffer.get();
+      int intValue6 = compressedBuffer.get();
+      int intValue7 = compressedBuffer.get();
+      int intValue8 = compressedBuffer.get();
+      int intValue9 = compressedBuffer.get();
+      int intValue10 = compressedBuffer.get();
+      int intValue11 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = (intValue0 >>> 12) & mask;
+      output[2 + outputOffset] = ((intValue0 >>> 24) | (intValue1 << 8)) & mask;
+      output[3 + outputOffset] = (intValue1 >>> 4) & mask;
+      output[4 + outputOffset] = (intValue1 >>> 16) & mask;
+      output[5 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask;
+      output[6 + outputOffset] = (intValue2 >>> 8) & mask;
+      output[7 + outputOffset] = intValue2 >>> 20;
+      output[8 + outputOffset] = intValue3 & mask;
+      output[9 + outputOffset] = (intValue3 >>> 12) & mask;
+      output[10 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask;
+      output[11 + outputOffset] = (intValue4 >>> 4) & mask;
+      output[12 + outputOffset] = (intValue4 >>> 16) & mask;
+      output[13 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask;
+      output[14 + outputOffset] = (intValue5 >>> 8) & mask;
+      output[15 + outputOffset] = intValue5 >>> 20;
+      output[16 + outputOffset] = intValue6 & mask;
+      output[17 + outputOffset] = (intValue6 >>> 12) & mask;
+      output[18 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask;
+      output[19 + outputOffset] = (intValue7 >>> 4) & mask;
+      output[20 + outputOffset] = (intValue7 >>> 16) & mask;
+      output[21 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask;
+      output[22 + outputOffset] = (intValue8 >>> 8) & mask;
+      output[23 + outputOffset] = intValue8 >>> 20;
+      output[24 + outputOffset] = intValue9 & mask;
+      output[25 + outputOffset] = (intValue9 >>> 12) & mask;
+      output[26 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask;
+      output[27 + outputOffset] = (intValue10 >>> 4) & mask;
+      output[28 + outputOffset] = (intValue10 >>> 16) & mask;
+      output[29 + outputOffset] = ((intValue10 >>> 28) | (intValue11 << 4)) & mask;
+      output[30 + outputOffset] = (intValue11 >>> 8) & mask;
+      output[31 + outputOffset] = intValue11 >>> 20;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For13Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For13Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For13Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For13Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,84 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For13Decompress extends ForDecompress {
+  static final int numFrameBits = 13;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      int intValue5 = compressedBuffer.get();
+      int intValue6 = compressedBuffer.get();
+      int intValue7 = compressedBuffer.get();
+      int intValue8 = compressedBuffer.get();
+      int intValue9 = compressedBuffer.get();
+      int intValue10 = compressedBuffer.get();
+      int intValue11 = compressedBuffer.get();
+      int intValue12 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = (intValue0 >>> 13) & mask;
+      output[2 + outputOffset] = ((intValue0 >>> 26) | (intValue1 << 6)) & mask;
+      output[3 + outputOffset] = (intValue1 >>> 7) & mask;
+      output[4 + outputOffset] = ((intValue1 >>> 20) | (intValue2 << 12)) & mask;
+      output[5 + outputOffset] = (intValue2 >>> 1) & mask;
+      output[6 + outputOffset] = (intValue2 >>> 14) & mask;
+      output[7 + outputOffset] = ((intValue2 >>> 27) | (intValue3 << 5)) & mask;
+      output[8 + outputOffset] = (intValue3 >>> 8) & mask;
+      output[9 + outputOffset] = ((intValue3 >>> 21) | (intValue4 << 11)) & mask;
+      output[10 + outputOffset] = (intValue4 >>> 2) & mask;
+      output[11 + outputOffset] = (intValue4 >>> 15) & mask;
+      output[12 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask;
+      output[13 + outputOffset] = (intValue5 >>> 9) & mask;
+      output[14 + outputOffset] = ((intValue5 >>> 22) | (intValue6 << 10)) & mask;
+      output[15 + outputOffset] = (intValue6 >>> 3) & mask;
+      output[16 + outputOffset] = (intValue6 >>> 16) & mask;
+      output[17 + outputOffset] = ((intValue6 >>> 29) | (intValue7 << 3)) & mask;
+      output[18 + outputOffset] = (intValue7 >>> 10) & mask;
+      output[19 + outputOffset] = ((intValue7 >>> 23) | (intValue8 << 9)) & mask;
+      output[20 + outputOffset] = (intValue8 >>> 4) & mask;
+      output[21 + outputOffset] = (intValue8 >>> 17) & mask;
+      output[22 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask;
+      output[23 + outputOffset] = (intValue9 >>> 11) & mask;
+      output[24 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask;
+      output[25 + outputOffset] = (intValue10 >>> 5) & mask;
+      output[26 + outputOffset] = (intValue10 >>> 18) & mask;
+      output[27 + outputOffset] = ((intValue10 >>> 31) | (intValue11 << 1)) & mask;
+      output[28 + outputOffset] = (intValue11 >>> 12) & mask;
+      output[29 + outputOffset] = ((intValue11 >>> 25) | (intValue12 << 7)) & mask;
+      output[30 + outputOffset] = (intValue12 >>> 6) & mask;
+      output[31 + outputOffset] = intValue12 >>> 19;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For14Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For14Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For14Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For14Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,85 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For14Decompress extends ForDecompress {
+  static final int numFrameBits = 14;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      int intValue5 = compressedBuffer.get();
+      int intValue6 = compressedBuffer.get();
+      int intValue7 = compressedBuffer.get();
+      int intValue8 = compressedBuffer.get();
+      int intValue9 = compressedBuffer.get();
+      int intValue10 = compressedBuffer.get();
+      int intValue11 = compressedBuffer.get();
+      int intValue12 = compressedBuffer.get();
+      int intValue13 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = (intValue0 >>> 14) & mask;
+      output[2 + outputOffset] = ((intValue0 >>> 28) | (intValue1 << 4)) & mask;
+      output[3 + outputOffset] = (intValue1 >>> 10) & mask;
+      output[4 + outputOffset] = ((intValue1 >>> 24) | (intValue2 << 8)) & mask;
+      output[5 + outputOffset] = (intValue2 >>> 6) & mask;
+      output[6 + outputOffset] = ((intValue2 >>> 20) | (intValue3 << 12)) & mask;
+      output[7 + outputOffset] = (intValue3 >>> 2) & mask;
+      output[8 + outputOffset] = (intValue3 >>> 16) & mask;
+      output[9 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask;
+      output[10 + outputOffset] = (intValue4 >>> 12) & mask;
+      output[11 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask;
+      output[12 + outputOffset] = (intValue5 >>> 8) & mask;
+      output[13 + outputOffset] = ((intValue5 >>> 22) | (intValue6 << 10)) & mask;
+      output[14 + outputOffset] = (intValue6 >>> 4) & mask;
+      output[15 + outputOffset] = intValue6 >>> 18;
+      output[16 + outputOffset] = intValue7 & mask;
+      output[17 + outputOffset] = (intValue7 >>> 14) & mask;
+      output[18 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask;
+      output[19 + outputOffset] = (intValue8 >>> 10) & mask;
+      output[20 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask;
+      output[21 + outputOffset] = (intValue9 >>> 6) & mask;
+      output[22 + outputOffset] = ((intValue9 >>> 20) | (intValue10 << 12)) & mask;
+      output[23 + outputOffset] = (intValue10 >>> 2) & mask;
+      output[24 + outputOffset] = (intValue10 >>> 16) & mask;
+      output[25 + outputOffset] = ((intValue10 >>> 30) | (intValue11 << 2)) & mask;
+      output[26 + outputOffset] = (intValue11 >>> 12) & mask;
+      output[27 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask;
+      output[28 + outputOffset] = (intValue12 >>> 8) & mask;
+      output[29 + outputOffset] = ((intValue12 >>> 22) | (intValue13 << 10)) & mask;
+      output[30 + outputOffset] = (intValue13 >>> 4) & mask;
+      output[31 + outputOffset] = intValue13 >>> 18;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For15Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For15Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For15Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For15Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,86 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For15Decompress extends ForDecompress {
+  static final int numFrameBits = 15;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      int intValue5 = compressedBuffer.get();
+      int intValue6 = compressedBuffer.get();
+      int intValue7 = compressedBuffer.get();
+      int intValue8 = compressedBuffer.get();
+      int intValue9 = compressedBuffer.get();
+      int intValue10 = compressedBuffer.get();
+      int intValue11 = compressedBuffer.get();
+      int intValue12 = compressedBuffer.get();
+      int intValue13 = compressedBuffer.get();
+      int intValue14 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = (intValue0 >>> 15) & mask;
+      output[2 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask;
+      output[3 + outputOffset] = (intValue1 >>> 13) & mask;
+      output[4 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask;
+      output[5 + outputOffset] = (intValue2 >>> 11) & mask;
+      output[6 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask;
+      output[7 + outputOffset] = (intValue3 >>> 9) & mask;
+      output[8 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask;
+      output[9 + outputOffset] = (intValue4 >>> 7) & mask;
+      output[10 + outputOffset] = ((intValue4 >>> 22) | (intValue5 << 10)) & mask;
+      output[11 + outputOffset] = (intValue5 >>> 5) & mask;
+      output[12 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask;
+      output[13 + outputOffset] = (intValue6 >>> 3) & mask;
+      output[14 + outputOffset] = ((intValue6 >>> 18) | (intValue7 << 14)) & mask;
+      output[15 + outputOffset] = (intValue7 >>> 1) & mask;
+      output[16 + outputOffset] = (intValue7 >>> 16) & mask;
+      output[17 + outputOffset] = ((intValue7 >>> 31) | (intValue8 << 1)) & mask;
+      output[18 + outputOffset] = (intValue8 >>> 14) & mask;
+      output[19 + outputOffset] = ((intValue8 >>> 29) | (intValue9 << 3)) & mask;
+      output[20 + outputOffset] = (intValue9 >>> 12) & mask;
+      output[21 + outputOffset] = ((intValue9 >>> 27) | (intValue10 << 5)) & mask;
+      output[22 + outputOffset] = (intValue10 >>> 10) & mask;
+      output[23 + outputOffset] = ((intValue10 >>> 25) | (intValue11 << 7)) & mask;
+      output[24 + outputOffset] = (intValue11 >>> 8) & mask;
+      output[25 + outputOffset] = ((intValue11 >>> 23) | (intValue12 << 9)) & mask;
+      output[26 + outputOffset] = (intValue12 >>> 6) & mask;
+      output[27 + outputOffset] = ((intValue12 >>> 21) | (intValue13 << 11)) & mask;
+      output[28 + outputOffset] = (intValue13 >>> 4) & mask;
+      output[29 + outputOffset] = ((intValue13 >>> 19) | (intValue14 << 13)) & mask;
+      output[30 + outputOffset] = (intValue14 >>> 2) & mask;
+      output[31 + outputOffset] = intValue14 >>> 17;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For16Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For16Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For16Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For16Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,87 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For16Decompress extends ForDecompress {
+  static final int numFrameBits = 16;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      int intValue5 = compressedBuffer.get();
+      int intValue6 = compressedBuffer.get();
+      int intValue7 = compressedBuffer.get();
+      int intValue8 = compressedBuffer.get();
+      int intValue9 = compressedBuffer.get();
+      int intValue10 = compressedBuffer.get();
+      int intValue11 = compressedBuffer.get();
+      int intValue12 = compressedBuffer.get();
+      int intValue13 = compressedBuffer.get();
+      int intValue14 = compressedBuffer.get();
+      int intValue15 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = intValue0 >>> 16;
+      output[2 + outputOffset] = intValue1 & mask;
+      output[3 + outputOffset] = intValue1 >>> 16;
+      output[4 + outputOffset] = intValue2 & mask;
+      output[5 + outputOffset] = intValue2 >>> 16;
+      output[6 + outputOffset] = intValue3 & mask;
+      output[7 + outputOffset] = intValue3 >>> 16;
+      output[8 + outputOffset] = intValue4 & mask;
+      output[9 + outputOffset] = intValue4 >>> 16;
+      output[10 + outputOffset] = intValue5 & mask;
+      output[11 + outputOffset] = intValue5 >>> 16;
+      output[12 + outputOffset] = intValue6 & mask;
+      output[13 + outputOffset] = intValue6 >>> 16;
+      output[14 + outputOffset] = intValue7 & mask;
+      output[15 + outputOffset] = intValue7 >>> 16;
+      output[16 + outputOffset] = intValue8 & mask;
+      output[17 + outputOffset] = intValue8 >>> 16;
+      output[18 + outputOffset] = intValue9 & mask;
+      output[19 + outputOffset] = intValue9 >>> 16;
+      output[20 + outputOffset] = intValue10 & mask;
+      output[21 + outputOffset] = intValue10 >>> 16;
+      output[22 + outputOffset] = intValue11 & mask;
+      output[23 + outputOffset] = intValue11 >>> 16;
+      output[24 + outputOffset] = intValue12 & mask;
+      output[25 + outputOffset] = intValue12 >>> 16;
+      output[26 + outputOffset] = intValue13 & mask;
+      output[27 + outputOffset] = intValue13 >>> 16;
+      output[28 + outputOffset] = intValue14 & mask;
+      output[29 + outputOffset] = intValue14 >>> 16;
+      output[30 + outputOffset] = intValue15 & mask;
+      output[31 + outputOffset] = intValue15 >>> 16;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For17Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For17Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For17Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For17Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,88 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For17Decompress extends ForDecompress {
+  static final int numFrameBits = 17;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      int intValue5 = compressedBuffer.get();
+      int intValue6 = compressedBuffer.get();
+      int intValue7 = compressedBuffer.get();
+      int intValue8 = compressedBuffer.get();
+      int intValue9 = compressedBuffer.get();
+      int intValue10 = compressedBuffer.get();
+      int intValue11 = compressedBuffer.get();
+      int intValue12 = compressedBuffer.get();
+      int intValue13 = compressedBuffer.get();
+      int intValue14 = compressedBuffer.get();
+      int intValue15 = compressedBuffer.get();
+      int intValue16 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = ((intValue0 >>> 17) | (intValue1 << 15)) & mask;
+      output[2 + outputOffset] = (intValue1 >>> 2) & mask;
+      output[3 + outputOffset] = ((intValue1 >>> 19) | (intValue2 << 13)) & mask;
+      output[4 + outputOffset] = (intValue2 >>> 4) & mask;
+      output[5 + outputOffset] = ((intValue2 >>> 21) | (intValue3 << 11)) & mask;
+      output[6 + outputOffset] = (intValue3 >>> 6) & mask;
+      output[7 + outputOffset] = ((intValue3 >>> 23) | (intValue4 << 9)) & mask;
+      output[8 + outputOffset] = (intValue4 >>> 8) & mask;
+      output[9 + outputOffset] = ((intValue4 >>> 25) | (intValue5 << 7)) & mask;
+      output[10 + outputOffset] = (intValue5 >>> 10) & mask;
+      output[11 + outputOffset] = ((intValue5 >>> 27) | (intValue6 << 5)) & mask;
+      output[12 + outputOffset] = (intValue6 >>> 12) & mask;
+      output[13 + outputOffset] = ((intValue6 >>> 29) | (intValue7 << 3)) & mask;
+      output[14 + outputOffset] = (intValue7 >>> 14) & mask;
+      output[15 + outputOffset] = ((intValue7 >>> 31) | (intValue8 << 1)) & mask;
+      output[16 + outputOffset] = ((intValue8 >>> 16) | (intValue9 << 16)) & mask;
+      output[17 + outputOffset] = (intValue9 >>> 1) & mask;
+      output[18 + outputOffset] = ((intValue9 >>> 18) | (intValue10 << 14)) & mask;
+      output[19 + outputOffset] = (intValue10 >>> 3) & mask;
+      output[20 + outputOffset] = ((intValue10 >>> 20) | (intValue11 << 12)) & mask;
+      output[21 + outputOffset] = (intValue11 >>> 5) & mask;
+      output[22 + outputOffset] = ((intValue11 >>> 22) | (intValue12 << 10)) & mask;
+      output[23 + outputOffset] = (intValue12 >>> 7) & mask;
+      output[24 + outputOffset] = ((intValue12 >>> 24) | (intValue13 << 8)) & mask;
+      output[25 + outputOffset] = (intValue13 >>> 9) & mask;
+      output[26 + outputOffset] = ((intValue13 >>> 26) | (intValue14 << 6)) & mask;
+      output[27 + outputOffset] = (intValue14 >>> 11) & mask;
+      output[28 + outputOffset] = ((intValue14 >>> 28) | (intValue15 << 4)) & mask;
+      output[29 + outputOffset] = (intValue15 >>> 13) & mask;
+      output[30 + outputOffset] = ((intValue15 >>> 30) | (intValue16 << 2)) & mask;
+      output[31 + outputOffset] = intValue16 >>> 15;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For18Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For18Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For18Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For18Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,89 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For18Decompress extends ForDecompress {
+  static final int numFrameBits = 18;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      int intValue5 = compressedBuffer.get();
+      int intValue6 = compressedBuffer.get();
+      int intValue7 = compressedBuffer.get();
+      int intValue8 = compressedBuffer.get();
+      int intValue9 = compressedBuffer.get();
+      int intValue10 = compressedBuffer.get();
+      int intValue11 = compressedBuffer.get();
+      int intValue12 = compressedBuffer.get();
+      int intValue13 = compressedBuffer.get();
+      int intValue14 = compressedBuffer.get();
+      int intValue15 = compressedBuffer.get();
+      int intValue16 = compressedBuffer.get();
+      int intValue17 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = ((intValue0 >>> 18) | (intValue1 << 14)) & mask;
+      output[2 + outputOffset] = (intValue1 >>> 4) & mask;
+      output[3 + outputOffset] = ((intValue1 >>> 22) | (intValue2 << 10)) & mask;
+      output[4 + outputOffset] = (intValue2 >>> 8) & mask;
+      output[5 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask;
+      output[6 + outputOffset] = (intValue3 >>> 12) & mask;
+      output[7 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask;
+      output[8 + outputOffset] = ((intValue4 >>> 16) | (intValue5 << 16)) & mask;
+      output[9 + outputOffset] = (intValue5 >>> 2) & mask;
+      output[10 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask;
+      output[11 + outputOffset] = (intValue6 >>> 6) & mask;
+      output[12 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask;
+      output[13 + outputOffset] = (intValue7 >>> 10) & mask;
+      output[14 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask;
+      output[15 + outputOffset] = intValue8 >>> 14;
+      output[16 + outputOffset] = intValue9 & mask;
+      output[17 + outputOffset] = ((intValue9 >>> 18) | (intValue10 << 14)) & mask;
+      output[18 + outputOffset] = (intValue10 >>> 4) & mask;
+      output[19 + outputOffset] = ((intValue10 >>> 22) | (intValue11 << 10)) & mask;
+      output[20 + outputOffset] = (intValue11 >>> 8) & mask;
+      output[21 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask;
+      output[22 + outputOffset] = (intValue12 >>> 12) & mask;
+      output[23 + outputOffset] = ((intValue12 >>> 30) | (intValue13 << 2)) & mask;
+      output[24 + outputOffset] = ((intValue13 >>> 16) | (intValue14 << 16)) & mask;
+      output[25 + outputOffset] = (intValue14 >>> 2) & mask;
+      output[26 + outputOffset] = ((intValue14 >>> 20) | (intValue15 << 12)) & mask;
+      output[27 + outputOffset] = (intValue15 >>> 6) & mask;
+      output[28 + outputOffset] = ((intValue15 >>> 24) | (intValue16 << 8)) & mask;
+      output[29 + outputOffset] = (intValue16 >>> 10) & mask;
+      output[30 + outputOffset] = ((intValue16 >>> 28) | (intValue17 << 4)) & mask;
+      output[31 + outputOffset] = intValue17 >>> 14;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}



Mime
View raw message