lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1049480 [3/3] - in /lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene: index/codecs/ index/codecs/pfordelta/ store/ util/pfor/
Date Wed, 15 Dec 2010 10:44:07 GMT
Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For3Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For3Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For3Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For3Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,74 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For3Decompress extends ForDecompress {
+  static final int numFrameBits = 3;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = (intValue0 >>> 3) & mask;
+      output[2 + outputOffset] = (intValue0 >>> 6) & mask;
+      output[3 + outputOffset] = (intValue0 >>> 9) & mask;
+      output[4 + outputOffset] = (intValue0 >>> 12) & mask;
+      output[5 + outputOffset] = (intValue0 >>> 15) & mask;
+      output[6 + outputOffset] = (intValue0 >>> 18) & mask;
+      output[7 + outputOffset] = (intValue0 >>> 21) & mask;
+      output[8 + outputOffset] = (intValue0 >>> 24) & mask;
+      output[9 + outputOffset] = (intValue0 >>> 27) & mask;
+      output[10 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask;
+      output[11 + outputOffset] = (intValue1 >>> 1) & mask;
+      output[12 + outputOffset] = (intValue1 >>> 4) & mask;
+      output[13 + outputOffset] = (intValue1 >>> 7) & mask;
+      output[14 + outputOffset] = (intValue1 >>> 10) & mask;
+      output[15 + outputOffset] = (intValue1 >>> 13) & mask;
+      output[16 + outputOffset] = (intValue1 >>> 16) & mask;
+      output[17 + outputOffset] = (intValue1 >>> 19) & mask;
+      output[18 + outputOffset] = (intValue1 >>> 22) & mask;
+      output[19 + outputOffset] = (intValue1 >>> 25) & mask;
+      output[20 + outputOffset] = (intValue1 >>> 28) & mask;
+      output[21 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask;
+      output[22 + outputOffset] = (intValue2 >>> 2) & mask;
+      output[23 + outputOffset] = (intValue2 >>> 5) & mask;
+      output[24 + outputOffset] = (intValue2 >>> 8) & mask;
+      output[25 + outputOffset] = (intValue2 >>> 11) & mask;
+      output[26 + outputOffset] = (intValue2 >>> 14) & mask;
+      output[27 + outputOffset] = (intValue2 >>> 17) & mask;
+      output[28 + outputOffset] = (intValue2 >>> 20) & mask;
+      output[29 + outputOffset] = (intValue2 >>> 23) & mask;
+      output[30 + outputOffset] = (intValue2 >>> 26) & mask;
+      output[31 + outputOffset] = intValue2 >>> 29;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For4Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For4Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For4Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For4Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,75 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For4Decompress extends ForDecompress {
+  static final int numFrameBits = 4;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = (intValue0 >>> 4) & mask;
+      output[2 + outputOffset] = (intValue0 >>> 8) & mask;
+      output[3 + outputOffset] = (intValue0 >>> 12) & mask;
+      output[4 + outputOffset] = (intValue0 >>> 16) & mask;
+      output[5 + outputOffset] = (intValue0 >>> 20) & mask;
+      output[6 + outputOffset] = (intValue0 >>> 24) & mask;
+      output[7 + outputOffset] = intValue0 >>> 28;
+      output[8 + outputOffset] = intValue1 & mask;
+      output[9 + outputOffset] = (intValue1 >>> 4) & mask;
+      output[10 + outputOffset] = (intValue1 >>> 8) & mask;
+      output[11 + outputOffset] = (intValue1 >>> 12) & mask;
+      output[12 + outputOffset] = (intValue1 >>> 16) & mask;
+      output[13 + outputOffset] = (intValue1 >>> 20) & mask;
+      output[14 + outputOffset] = (intValue1 >>> 24) & mask;
+      output[15 + outputOffset] = intValue1 >>> 28;
+      output[16 + outputOffset] = intValue2 & mask;
+      output[17 + outputOffset] = (intValue2 >>> 4) & mask;
+      output[18 + outputOffset] = (intValue2 >>> 8) & mask;
+      output[19 + outputOffset] = (intValue2 >>> 12) & mask;
+      output[20 + outputOffset] = (intValue2 >>> 16) & mask;
+      output[21 + outputOffset] = (intValue2 >>> 20) & mask;
+      output[22 + outputOffset] = (intValue2 >>> 24) & mask;
+      output[23 + outputOffset] = intValue2 >>> 28;
+      output[24 + outputOffset] = intValue3 & mask;
+      output[25 + outputOffset] = (intValue3 >>> 4) & mask;
+      output[26 + outputOffset] = (intValue3 >>> 8) & mask;
+      output[27 + outputOffset] = (intValue3 >>> 12) & mask;
+      output[28 + outputOffset] = (intValue3 >>> 16) & mask;
+      output[29 + outputOffset] = (intValue3 >>> 20) & mask;
+      output[30 + outputOffset] = (intValue3 >>> 24) & mask;
+      output[31 + outputOffset] = intValue3 >>> 28;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For5Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For5Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For5Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For5Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,76 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For5Decompress extends ForDecompress {
+  static final int numFrameBits = 5;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = (intValue0 >>> 5) & mask;
+      output[2 + outputOffset] = (intValue0 >>> 10) & mask;
+      output[3 + outputOffset] = (intValue0 >>> 15) & mask;
+      output[4 + outputOffset] = (intValue0 >>> 20) & mask;
+      output[5 + outputOffset] = (intValue0 >>> 25) & mask;
+      output[6 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask;
+      output[7 + outputOffset] = (intValue1 >>> 3) & mask;
+      output[8 + outputOffset] = (intValue1 >>> 8) & mask;
+      output[9 + outputOffset] = (intValue1 >>> 13) & mask;
+      output[10 + outputOffset] = (intValue1 >>> 18) & mask;
+      output[11 + outputOffset] = (intValue1 >>> 23) & mask;
+      output[12 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask;
+      output[13 + outputOffset] = (intValue2 >>> 1) & mask;
+      output[14 + outputOffset] = (intValue2 >>> 6) & mask;
+      output[15 + outputOffset] = (intValue2 >>> 11) & mask;
+      output[16 + outputOffset] = (intValue2 >>> 16) & mask;
+      output[17 + outputOffset] = (intValue2 >>> 21) & mask;
+      output[18 + outputOffset] = (intValue2 >>> 26) & mask;
+      output[19 + outputOffset] = ((intValue2 >>> 31) | (intValue3 << 1)) & mask;
+      output[20 + outputOffset] = (intValue3 >>> 4) & mask;
+      output[21 + outputOffset] = (intValue3 >>> 9) & mask;
+      output[22 + outputOffset] = (intValue3 >>> 14) & mask;
+      output[23 + outputOffset] = (intValue3 >>> 19) & mask;
+      output[24 + outputOffset] = (intValue3 >>> 24) & mask;
+      output[25 + outputOffset] = ((intValue3 >>> 29) | (intValue4 << 3)) & mask;
+      output[26 + outputOffset] = (intValue4 >>> 2) & mask;
+      output[27 + outputOffset] = (intValue4 >>> 7) & mask;
+      output[28 + outputOffset] = (intValue4 >>> 12) & mask;
+      output[29 + outputOffset] = (intValue4 >>> 17) & mask;
+      output[30 + outputOffset] = (intValue4 >>> 22) & mask;
+      output[31 + outputOffset] = intValue4 >>> 27;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For6Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For6Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For6Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For6Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,77 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For6Decompress extends ForDecompress {
+  static final int numFrameBits = 6;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      int intValue5 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = (intValue0 >>> 6) & mask;
+      output[2 + outputOffset] = (intValue0 >>> 12) & mask;
+      output[3 + outputOffset] = (intValue0 >>> 18) & mask;
+      output[4 + outputOffset] = (intValue0 >>> 24) & mask;
+      output[5 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask;
+      output[6 + outputOffset] = (intValue1 >>> 4) & mask;
+      output[7 + outputOffset] = (intValue1 >>> 10) & mask;
+      output[8 + outputOffset] = (intValue1 >>> 16) & mask;
+      output[9 + outputOffset] = (intValue1 >>> 22) & mask;
+      output[10 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask;
+      output[11 + outputOffset] = (intValue2 >>> 2) & mask;
+      output[12 + outputOffset] = (intValue2 >>> 8) & mask;
+      output[13 + outputOffset] = (intValue2 >>> 14) & mask;
+      output[14 + outputOffset] = (intValue2 >>> 20) & mask;
+      output[15 + outputOffset] = intValue2 >>> 26;
+      output[16 + outputOffset] = intValue3 & mask;
+      output[17 + outputOffset] = (intValue3 >>> 6) & mask;
+      output[18 + outputOffset] = (intValue3 >>> 12) & mask;
+      output[19 + outputOffset] = (intValue3 >>> 18) & mask;
+      output[20 + outputOffset] = (intValue3 >>> 24) & mask;
+      output[21 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask;
+      output[22 + outputOffset] = (intValue4 >>> 4) & mask;
+      output[23 + outputOffset] = (intValue4 >>> 10) & mask;
+      output[24 + outputOffset] = (intValue4 >>> 16) & mask;
+      output[25 + outputOffset] = (intValue4 >>> 22) & mask;
+      output[26 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask;
+      output[27 + outputOffset] = (intValue5 >>> 2) & mask;
+      output[28 + outputOffset] = (intValue5 >>> 8) & mask;
+      output[29 + outputOffset] = (intValue5 >>> 14) & mask;
+      output[30 + outputOffset] = (intValue5 >>> 20) & mask;
+      output[31 + outputOffset] = intValue5 >>> 26;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For7Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For7Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For7Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For7Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,78 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For7Decompress extends ForDecompress {
+  static final int numFrameBits = 7;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      int intValue5 = compressedBuffer.get();
+      int intValue6 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = (intValue0 >>> 7) & mask;
+      output[2 + outputOffset] = (intValue0 >>> 14) & mask;
+      output[3 + outputOffset] = (intValue0 >>> 21) & mask;
+      output[4 + outputOffset] = ((intValue0 >>> 28) | (intValue1 << 4)) & mask;
+      output[5 + outputOffset] = (intValue1 >>> 3) & mask;
+      output[6 + outputOffset] = (intValue1 >>> 10) & mask;
+      output[7 + outputOffset] = (intValue1 >>> 17) & mask;
+      output[8 + outputOffset] = (intValue1 >>> 24) & mask;
+      output[9 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask;
+      output[10 + outputOffset] = (intValue2 >>> 6) & mask;
+      output[11 + outputOffset] = (intValue2 >>> 13) & mask;
+      output[12 + outputOffset] = (intValue2 >>> 20) & mask;
+      output[13 + outputOffset] = ((intValue2 >>> 27) | (intValue3 << 5)) & mask;
+      output[14 + outputOffset] = (intValue3 >>> 2) & mask;
+      output[15 + outputOffset] = (intValue3 >>> 9) & mask;
+      output[16 + outputOffset] = (intValue3 >>> 16) & mask;
+      output[17 + outputOffset] = (intValue3 >>> 23) & mask;
+      output[18 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask;
+      output[19 + outputOffset] = (intValue4 >>> 5) & mask;
+      output[20 + outputOffset] = (intValue4 >>> 12) & mask;
+      output[21 + outputOffset] = (intValue4 >>> 19) & mask;
+      output[22 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask;
+      output[23 + outputOffset] = (intValue5 >>> 1) & mask;
+      output[24 + outputOffset] = (intValue5 >>> 8) & mask;
+      output[25 + outputOffset] = (intValue5 >>> 15) & mask;
+      output[26 + outputOffset] = (intValue5 >>> 22) & mask;
+      output[27 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask;
+      output[28 + outputOffset] = (intValue6 >>> 4) & mask;
+      output[29 + outputOffset] = (intValue6 >>> 11) & mask;
+      output[30 + outputOffset] = (intValue6 >>> 18) & mask;
+      output[31 + outputOffset] = intValue6 >>> 25;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For8Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For8Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For8Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For8Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,79 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For8Decompress extends ForDecompress {
+  static final int numFrameBits = 8;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      int intValue5 = compressedBuffer.get();
+      int intValue6 = compressedBuffer.get();
+      int intValue7 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = (intValue0 >>> 8) & mask;
+      output[2 + outputOffset] = (intValue0 >>> 16) & mask;
+      output[3 + outputOffset] = intValue0 >>> 24;
+      output[4 + outputOffset] = intValue1 & mask;
+      output[5 + outputOffset] = (intValue1 >>> 8) & mask;
+      output[6 + outputOffset] = (intValue1 >>> 16) & mask;
+      output[7 + outputOffset] = intValue1 >>> 24;
+      output[8 + outputOffset] = intValue2 & mask;
+      output[9 + outputOffset] = (intValue2 >>> 8) & mask;
+      output[10 + outputOffset] = (intValue2 >>> 16) & mask;
+      output[11 + outputOffset] = intValue2 >>> 24;
+      output[12 + outputOffset] = intValue3 & mask;
+      output[13 + outputOffset] = (intValue3 >>> 8) & mask;
+      output[14 + outputOffset] = (intValue3 >>> 16) & mask;
+      output[15 + outputOffset] = intValue3 >>> 24;
+      output[16 + outputOffset] = intValue4 & mask;
+      output[17 + outputOffset] = (intValue4 >>> 8) & mask;
+      output[18 + outputOffset] = (intValue4 >>> 16) & mask;
+      output[19 + outputOffset] = intValue4 >>> 24;
+      output[20 + outputOffset] = intValue5 & mask;
+      output[21 + outputOffset] = (intValue5 >>> 8) & mask;
+      output[22 + outputOffset] = (intValue5 >>> 16) & mask;
+      output[23 + outputOffset] = intValue5 >>> 24;
+      output[24 + outputOffset] = intValue6 & mask;
+      output[25 + outputOffset] = (intValue6 >>> 8) & mask;
+      output[26 + outputOffset] = (intValue6 >>> 16) & mask;
+      output[27 + outputOffset] = intValue6 >>> 24;
+      output[28 + outputOffset] = intValue7 & mask;
+      output[29 + outputOffset] = (intValue7 >>> 8) & mask;
+      output[30 + outputOffset] = (intValue7 >>> 16) & mask;
+      output[31 + outputOffset] = intValue7 >>> 24;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For9Decompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For9Decompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For9Decompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/For9Decompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,80 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* This program is generated, do not modify. See gendecompress.py */
+
+import java.nio.IntBuffer;
+class For9Decompress extends ForDecompress {
+  static final int numFrameBits = 9;
+  static final int mask = (int) ((1L<<numFrameBits) - 1);
+
+  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;
+    for(int step=0;step<4;step++) {
+      int intValue0 = compressedBuffer.get();
+      int intValue1 = compressedBuffer.get();
+      int intValue2 = compressedBuffer.get();
+      int intValue3 = compressedBuffer.get();
+      int intValue4 = compressedBuffer.get();
+      int intValue5 = compressedBuffer.get();
+      int intValue6 = compressedBuffer.get();
+      int intValue7 = compressedBuffer.get();
+      int intValue8 = compressedBuffer.get();
+      output[0 + outputOffset] = intValue0 & mask;
+      output[1 + outputOffset] = (intValue0 >>> 9) & mask;
+      output[2 + outputOffset] = (intValue0 >>> 18) & mask;
+      output[3 + outputOffset] = ((intValue0 >>> 27) | (intValue1 << 5)) & mask;
+      output[4 + outputOffset] = (intValue1 >>> 4) & mask;
+      output[5 + outputOffset] = (intValue1 >>> 13) & mask;
+      output[6 + outputOffset] = (intValue1 >>> 22) & mask;
+      output[7 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask;
+      output[8 + outputOffset] = (intValue2 >>> 8) & mask;
+      output[9 + outputOffset] = (intValue2 >>> 17) & mask;
+      output[10 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask;
+      output[11 + outputOffset] = (intValue3 >>> 3) & mask;
+      output[12 + outputOffset] = (intValue3 >>> 12) & mask;
+      output[13 + outputOffset] = (intValue3 >>> 21) & mask;
+      output[14 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask;
+      output[15 + outputOffset] = (intValue4 >>> 7) & mask;
+      output[16 + outputOffset] = (intValue4 >>> 16) & mask;
+      output[17 + outputOffset] = ((intValue4 >>> 25) | (intValue5 << 7)) & mask;
+      output[18 + outputOffset] = (intValue5 >>> 2) & mask;
+      output[19 + outputOffset] = (intValue5 >>> 11) & mask;
+      output[20 + outputOffset] = (intValue5 >>> 20) & mask;
+      output[21 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask;
+      output[22 + outputOffset] = (intValue6 >>> 6) & mask;
+      output[23 + outputOffset] = (intValue6 >>> 15) & mask;
+      output[24 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask;
+      output[25 + outputOffset] = (intValue7 >>> 1) & mask;
+      output[26 + outputOffset] = (intValue7 >>> 10) & mask;
+      output[27 + outputOffset] = (intValue7 >>> 19) & mask;
+      output[28 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask;
+      output[29 + outputOffset] = (intValue8 >>> 5) & mask;
+      output[30 + outputOffset] = (intValue8 >>> 14) & mask;
+      output[31 + outputOffset] = intValue8 >>> 23;
+      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/ForDecompress.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/ForDecompress.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/ForDecompress.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/ForDecompress.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,56 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.nio.IntBuffer;
+
+/** PFor frame decompression for any number of frame bits. */
+class ForDecompress {
+
+  static void decodeAnyFrame(
+        IntBuffer intBuffer, int bufIndex, int inputSize, int numFrameBits,
+        int[] output, int outputOffset) {
+
+    assert numFrameBits > 0 : numFrameBits;
+    assert numFrameBits <= 32 : numFrameBits;
+    final int mask = (int) ((1L<<numFrameBits) - 1);
+    int intValue1 = intBuffer.get(bufIndex);
+    output[outputOffset] = intValue1 & mask;
+    if (--inputSize == 0) return;
+    int bitPos = numFrameBits;
+
+    do {
+      while (bitPos <= (32 - numFrameBits)) {
+        // No mask needed when bitPos == (32 - numFrameBits), but prefer to avoid testing for this:
+        output[++outputOffset] = (intValue1 >>> bitPos) & mask;
+        if (--inputSize == 0) return;
+        bitPos += numFrameBits;
+      }
+      
+      int intValue2 = intBuffer.get(++bufIndex);
+      output[++outputOffset] = ( (bitPos == 32)
+                                  ? intValue2
+                                  : ((intValue1 >>> bitPos) | (intValue2 << (32 - bitPos)))
+                               ) & mask;
+        
+      if (--inputSize == 0) return;
+      
+      intValue1 = intValue2;
+      bitPos += numFrameBits - 32;
+    } while (true);
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/FrameOfRef.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/FrameOfRef.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/FrameOfRef.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/FrameOfRef.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,330 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.nio.IntBuffer;
+
+import org.apache.lucene.util.BitUtil;
+
+/** Frame of Reference lossless integer compression/decompression.
+ * For positive integers, the compression is done by leaving out
+ * the most significant bits, and storing all numbers with a fixed number of bits
+ * contiguously in a buffer of bits. This buffer is called the frame, and it
+ * can store positive numbers in a range from 0 to a constant maximum fitting in
+ * the number of bits available for a single compressed number.
+ * <p>
+ * This implementation uses 0 as the lower bound reference for the frame,
+ * so small positive integers can be most effectively compressed.
+ * <p>
+ * Optimized code is used for decompression, see class ForDecompress and its subclasses.
+ * <br>Use of the -server option helps performance for the Sun 1.6 jvm under Linux.
+ * <p>
+ * This class does not provide delta coding because the Lucene index
+ * structures already have that.
+ * <p>
+ * To be done:
+ * <ul>
+ * <li>
+ * Optimize compression code by specializing for number of frame bits.
+ * <li>
+ * IntBuffer.get() is somewhat faster that IntBuffer.get(index), adapt (de)compression to
+ * use the relative get() method.
+ * <li>
+ * Check javadoc generation and generated javadocs. Add javadoc code references.
+ * </ul>
+ */
+
+// nocommit -- make a dedicated decompressor -- much of this
+// is not needed for decompress:
+
+public class FrameOfRef {
+  /** Number of frame bits. 2**numFrameBits - 1 is the maximum compressed value. */
+  protected int numFrameBits;
+
+  /** Constant header tag to allow other compression methods, use value 0001 for
+   * Frame of reference.
+   * CHECKME: Move this to another class defining various decompression methods?
+   */
+  //protected int compressionMethod;
+  private final int FOR_COMPRESSION = 1; /** encode compression method in header */
+
+  /** IntBuffer for compressed data */
+  protected IntBuffer compressedBuffer;
+  
+  /** Index of header in int buffer */
+  protected final int HEADER_INDEX = 0;
+  
+  /** Start index in int buffer of array integers each compressed to numFrameBits. */
+  protected final int COMPRESSED_INDEX = HEADER_INDEX + 1;
+  protected final int HEADER_SIZE = 1; // one integer in IntBuffer
+
+  // used by decompress to read the "input" (compressed) ints
+  protected int[] scratch = new int[32];
+
+  /** Uncompressed data */
+  protected int[] unCompressedData;
+  /** Offset into unCompressedData */
+  protected int offset;
+  /** Size of unCompressedData, -1 when not available. */
+  protected int unComprSize = -1;
+
+  /** Create a Frame of Reference integer compressor/decompressor. */
+  public FrameOfRef() {
+  }
+
+  /** Integer buffer to hold the compressed data.<br>
+   *  Compression and decompression do not affect the current buffer position,
+   *  and the beginning of the compressed data should be or will be at the current
+   *  buffer position.<br>
+   *  When the buffer is not large enough, ArrayIndexOutOfBoundExceptions will occur
+   *  during compression/decompression.<br>
+   *  Without a buffer for compressed data, compress() will only determine the number
+   *  of integers needed in the buffer, see compress().<br>
+   *  Without a valid buffer, decompress() will throw a NullPointerException.<br>
+   *  For optimal speed when the IntBuffer is a view on a ByteBuffer,
+   *  the IntBuffer should have a byte offset of a  multiple of 4 bytes, possibly 0. <br>
+   *  An IntBuffer is used here because 32 bits can efficiently accessed in the buffer
+   *  on all current processors, and a positive int is normally large enough
+   *  for various purposes in a Lucene index.
+   *
+   * @param compressedBuffer    The buffer to hold the compressed integers.
+   *                            
+   */
+  public void setCompressedBuffer(IntBuffer compressedBuffer) {
+    this.compressedBuffer = compressedBuffer;
+  }
+
+
+  /** Array with offset holding uncompressed data.
+   * @param unCompressedData The array holding uncompressed integers.
+   * @param offset offset in unCompressedData.
+   * @param unComprSize The number of uncompressed integers, should be at least 1.
+   */
+  public void setUnCompressedData(int[] unCompressedData, int offset, int unComprSize) {
+    assert unCompressedData != null;
+    assert offset >= 0;
+    assert unComprSize >= 1;
+    assert (offset + unComprSize) <= unCompressedData.length;
+    this.unCompressedData = unCompressedData;
+    this.offset = offset;
+    this.unComprSize = unComprSize;
+  }
+
+  /** Compress the uncompressed data into the buffer using the given number of
+   * frame bits, storing only this number of least significant bits of the
+   * uncompressed integers in the compressed buffer.
+   * Should only be used after setUnCompressedData().
+   * <br>
+   * When setCompressBuffer() was not done, no actual compression is done.
+   * Regardless of the use of setCompressBuffer(), bufferByteSize() will return
+   * a valid value after calling compress().
+   * <p>
+   * When a buffer is available, the following is done.
+   * A header is stored as a first integer into the buffer, encoding
+   * the compression method, the number of frame bits and the number of compressed integers.
+   * All uncompressed integers are stored sequentially in compressed form
+   * in the buffer after the header.
+   *
+   * @param numFrameBits        The number of frame bits. Should be between 1 and 32.
+   *                            Note that when this value is 32, no compression occurs.
+   */
+  public void compress(int numFrameBits) {
+    assert numFrameBits >= 1;
+    assert numFrameBits <= 32;
+    this.numFrameBits = numFrameBits;
+    encodeHeader(unComprSize);
+    for (int i = 0; i < unComprSize; i++) {
+      int v = unCompressedData[i + offset];
+      encodeCompressedValue(i, v);
+    }
+  }
+
+  /** As compress(), using the result of frameBitsForCompression() as the number of frame bits. */
+  public void compress() {
+    compress( frameBitsForCompression());
+  }
+
+  /** Return the number of integers used in IntBuffer.
+   *  Only valid after compress() or decompress().
+   */
+  public int compressedSize() {
+    return HEADER_SIZE + (unComprSize * numFrameBits + 31) / 32;
+  }
+
+  /** Encode an integer value by compressing it into the buffer.
+   * @param compressedPos The index of the compressed integer in the compressed buffer.
+   * @param value The non negative value to be stored in compressed form.
+   *              This should fit into the number of frame bits.
+   */
+  protected void encodeCompressedValue(int compressedPos, int value) {
+    encodeCompressedValueBase(compressedPos, value, numFrameBits); // FIXME: inline private method.
+  }
+
+  /** Encode a value into the compressed buffer.
+   * Since numFrameBits is always smaller than the number of bits in an int,
+   * at most two ints in the buffer will be affected.
+   * <br>Has no effect when compressedBuffer == null.
+   * <br>This could be specialized for numBits just like decompressFrame().
+   */
+  private void encodeCompressedValueBase(int compressedPos, int value, int numBits) {
+    assert numBits >= 1;
+    assert numBits <= 32;
+    if (compressedBuffer == null) {
+      return;
+    }
+    final int mask = (int) ((1L << numBits) - 1);
+    assert ((value & mask) == value) : ("value " + value + ", mask " + mask + ", numBits " + numBits); // lossless compression
+    final int compressedBitPos = numBits * compressedPos;
+    final int firstBitPosition = compressedBitPos & 31;
+    int intIndex = COMPRESSED_INDEX + (compressedBitPos >> 5);
+    setBufferIntBits(intIndex, firstBitPosition, numBits, value);
+    if ((firstBitPosition + numBits) > 32) { // value does not fit in first int
+      setBufferIntBits(intIndex+1, 0, (firstBitPosition + numBits - 32), (value >>> (32 - firstBitPosition)));
+    }
+  }
+  
+  /** Change bits of an integer in the compressed buffer.
+   * <br> A more efficient implementation is possible when the compressed
+   * buffer is known to contain only zero bits, in that case one mask operation can be removed.
+   * @param intIndex The index of the affected integer in the compressed buffer.
+   * @param firstBitPosition The position of the least significant bit to be changed.
+   * @param numBits The number of more significant bits to be changed.
+   * @param value The new value of the bits to be changed, with the least significant bit at position zero.
+   */
+  protected void setBufferIntBits(int intIndex, int firstBitPosition, int numBits, int value) {
+    final int mask = (int) ((1L << numBits) - 1);
+    compressedBuffer.put(intIndex,
+          (compressedBuffer.get(intIndex)
+            & ~ (mask << firstBitPosition)) // masking superfluous on clear buffer
+          | (value << firstBitPosition));
+  }
+
+  /** The 4 byte header (32 bits) contains:
+   * <ul>
+   * <li>
+   *  <ul>
+   *  <li>4 bits for the compression method: 0b0001 for FrameOfRef,
+   *  <li>4 bits unused,
+   *  </ul>
+   * <li>
+   *  <ul>
+   *  <li>5 bits for (numFrameBits-1),
+   *  <li>3 bit unused,
+   *  </ul>
+   * <li>8 bits for number of compressed integers - 1,
+   * <li>8 bits unused.
+   * </ul>
+   */
+  private void encodeHeader(int unComprSize) {
+    assert numFrameBits >= 1;
+    assert numFrameBits <= (1 << 5); // 32
+    assert unComprSize >= 1;
+    assert unComprSize <= (1 << 8); // 256
+    if (compressedBuffer != null) {
+      compressedBuffer.put(HEADER_INDEX,
+                    ((unComprSize-1) << 16)
+                    | ((numFrameBits-1) << 8)
+                    | (FOR_COMPRESSION << 4));
+    }
+  }
+
+  protected void decodeHeader() {
+    int header = compressedBuffer.get();
+    // nocommit -- we know this will always be fixed (eg
+    // 128)?  silly to encode in every frame?
+    //unComprSize = ((header >>> 16) & 255) + 1;
+    numFrameBits = ((header >>> 8) & 31) + 1;
+    // verify compression method:
+    assert FOR_COMPRESSION == ((header >>> 4) & 15);
+  }
+
+  /** Decompress from the buffer into output from a given offset. */
+  public void decompress() {
+    decodeHeader();
+    decompressFrame();
+  }
+
+  /** Return the number of integers available for decompression.
+   * Do not use before an IntBuffer was passed to setCompressBuffer.
+   */
+  public int decompressedSize() {
+    decodeHeader();
+    return unComprSize;
+  }
+  
+  /** For performance, this delegates to classes with fixed numFrameBits. */
+  private void decompressFrame() {
+    switch (numFrameBits) {
+      // CHECKME: two other implementations might be faster:
+      // - array of static methods: Method[numFrameBits].invoke(null, [this]), 
+      // - array of non static decompressors: ForDecompressor[numFrameBits].decompressFrame(this) .
+      case 1: For1Decompress.decompressFrame(this); break;
+      case 2: For2Decompress.decompressFrame(this); break;
+      case 3: For3Decompress.decompressFrame(this); break;
+      case 4: For4Decompress.decompressFrame(this); break;
+      case 5: For5Decompress.decompressFrame(this); break;
+      case 6: For6Decompress.decompressFrame(this); break;
+      case 7: For7Decompress.decompressFrame(this); break;
+      case 8: For8Decompress.decompressFrame(this); break;
+      case 9: For9Decompress.decompressFrame(this); break;
+      case 10: For10Decompress.decompressFrame(this); break;
+      case 11: For11Decompress.decompressFrame(this); break;
+      case 12: For12Decompress.decompressFrame(this); break;
+      case 13: For13Decompress.decompressFrame(this); break;
+      case 14: For14Decompress.decompressFrame(this); break;
+      case 15: For15Decompress.decompressFrame(this); break;
+      case 16: For16Decompress.decompressFrame(this); break;
+      case 17: For17Decompress.decompressFrame(this); break;
+      case 18: For18Decompress.decompressFrame(this); break;
+      case 19: For19Decompress.decompressFrame(this); break;
+      case 20: For20Decompress.decompressFrame(this); break;
+      case 21: For21Decompress.decompressFrame(this); break;
+      case 22: For22Decompress.decompressFrame(this); break;
+      case 23: For23Decompress.decompressFrame(this); break;
+      case 24: For24Decompress.decompressFrame(this); break;
+      case 25: For25Decompress.decompressFrame(this); break;
+      case 26: For26Decompress.decompressFrame(this); break;
+      case 27: For27Decompress.decompressFrame(this); break;
+      case 28: For28Decompress.decompressFrame(this); break;
+      case 29: For29Decompress.decompressFrame(this); break;
+      case 30: For30Decompress.decompressFrame(this); break;
+      case 31: For31Decompress.decompressFrame(this); break;
+      case 32: For32Decompress.decompressFrame(this); break;
+      default:
+        throw new IllegalStateException("Unknown number of frame bits " + numFrameBits);
+    }
+  }
+
+  public int getNumFrameBits() {
+    return numFrameBits;
+  }
+
+  /** Determine the number of frame bits to be used for compression.
+   * Use only after setUnCompressedData().
+   * @return The number of bits needed to encode the maximum positive uncompressed value.
+   * Negative uncompressed values have no influence on the result.
+   */
+  public int frameBitsForCompression() {
+    int maxNonNegVal = 0;
+    for (int i = offset; i < (offset + unComprSize); i++) {
+      if (unCompressedData[i] > maxNonNegVal) {
+        maxNonNegVal = unCompressedData[i];
+      }
+    }
+    return BitUtil.logNextHigherPowerOfTwo(maxNonNegVal) + 1;
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/PFor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/PFor.java?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/PFor.java (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/PFor.java Wed Dec 15 10:44:06 2010
@@ -0,0 +1,413 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Arrays;
+
+/** Patched Frame of Reference PFOR compression/decompression.
+ * <p>
+ * As defined in:<br>
+ * Super-Scalar RAM-CPU Cache Compression<br>
+ * Marcin Zukowski, Sándor Héman, Niels Nes, Peter Boncz, 2006.<br>
+ * with extensions from:<br>
+ * Performance of Compressed Inverted List Caching in Search Engines<br>
+ * Jiangong Zhang, Xiaohui Long, Torsten Suel, 2008.<br>
+ * <p>
+ * This class does not provide delta coding because the lucene index
+ * structures already have that.
+ * <p>
+ * The implementation uses 0 as lower bound for the frame,
+ * so small positive integers will be most effectively compressed.
+ * <p>
+ * Some optimized code is used for decompression,
+ * see class ForDecompress and its subclasses.
+ * <br>Good decompression performance will depend on the performance
+ * of java.nio.IntBuffer indexed get() methods.
+ * <br>Use of the -server option helps performance for the Sun 1.6 jvm under Linux.
+ * <p>
+ * The start point of first exception is at its natural boundary:
+ * 2 byte exceptions at even byte position, 4 byte at quadruple.
+ * <p>
+ * To be done:
+ * <ul>
+ * <li>
+ * Optimize compression code.
+ * <li>
+ * IntBuffer.get() is somewhat faster that IntBuffer.get(index), adapt (de)compression for to
+ * use the relative get() method.
+ * <li>
+ * Check javadoc generation and generated javadocs. Add javadoc code references.
+ * </ul>
+ */
+public class PFor extends FrameOfRef {
+  /** Index on input and in compressed frame of first exception, -1 when no exceptions */
+  private int firstExceptionIndex;
+  
+  /** CHECKME: Move this to another class defining various decompression methods? */
+  private final int PFOR_COMPRESSION = 2; /** to encode compression method in header */
+  
+  /** How to encode PFor exceptions: 0: byte, 1: short, 2:int, unused: 3: long */
+  private int exceptionCode = -1;
+  
+  /** Total number of exception values */
+  private int numExceptions;
+
+  /** Create a PFor compressor/decompressor. */
+  public PFor() {
+  }
+  
+  /** Compress the decompressed data into the buffer.
+   * Should only be used after setUnCompressedData().
+   * <br>
+   * When setCompressBuffer() was not done, no actual compression is done.
+   * Regardless of the use of setCompressBuffer(), bufferByteSize() will return
+   * a valid value after calling compress().
+   * <p>
+   * When a buffer is available, the following is done.
+   * A header is stored into the buffer, encoding a.o. numFrameBits and unComprSize.
+   * All ints < 2**numFrameBits are stored sequentially in compressed form
+   * in the buffer.
+   * All other ints are stored in the buffer as exceptions after the compressed sequential ints,
+   * using 1, 2 or 4 bytes per exception, starting at the first byte after the compressed
+   * sequential ints.
+   * <br>
+   * The index of the first exception is encoded in the header in the buffer,
+   * all later exceptions have the offset to the next exception as their value,
+   * the last one offset to just after the available input size.
+   * After the first exception, when the next exception index does not fit in
+   * numFrameBits bits, an exception after 2**numFrameBits inputs is forced and inserted.
+   * <br>
+   * Exception values are stored in the order of the exceptions.
+   * The number of bytes used for an exception is also encoded in the header.
+   * This depends on the maximum exception value and does not vary between the exceptions.
+   */
+  public void compress(int numFrameBits) {
+    assert numFrameBits >= 1;
+    assert numFrameBits <= 32;
+    this.numFrameBits = numFrameBits;
+    numExceptions = 0;
+    int maxException = -1;
+    firstExceptionIndex = -1;
+    int lastExceptionIndex = -1;
+    int i;
+    int[] exceptionValues = new int[unComprSize];
+    int maxNonExceptionMask = (int) ((1L << numFrameBits) - 1);
+    int maxChain = 254; // maximum value of firstExceptionIndex in header
+    // CHECKME: maxChain 1 off because of initial value of lastExceptionIndex and force exception test below?
+    for (i = 0; i < unComprSize; i++) {
+      int v = unCompressedData[i + offset];
+      // FIXME: split this loop to avoid if statement in loop.
+      // use predication for this: (someBool ? 1 : 0), and hope that the jit optimizes this.
+      if ( (((v & maxNonExceptionMask) == v) // no value exception
+           && (i < (lastExceptionIndex + maxChain)))) { // no forced exception
+        encodeCompressedValue(i, v); // normal encoding
+      } else { // exception
+        exceptionValues[numExceptions] = v;
+        numExceptions++;
+        if (firstExceptionIndex == -1) {
+          firstExceptionIndex = i;
+          assert firstExceptionIndex <= 254; // maximum value of firstExceptionIndex in header
+          maxException = v;
+          maxChain = 1 << ((30 < numFrameBits) ? 30 : numFrameBits); // fewer bits available for exception chain value. 
+        } else if (v > maxException) {
+          maxException = v;
+        }
+        // encode the previous exception pointer
+        if (lastExceptionIndex >= 0) {
+          encodeCompressedValue(lastExceptionIndex, i - lastExceptionIndex - 1);
+        }
+        lastExceptionIndex = i;
+      }
+    }
+    if (lastExceptionIndex >= 0) {
+      encodeCompressedValue(lastExceptionIndex, i - lastExceptionIndex - 1); // end the exception chain.
+    }
+    int bitsInArray = numFrameBits * unCompressedData.length;
+    //int bytesInArray = (bitsInArray + 7) / 8;
+    if (maxException < (1 << 8)) { // exceptions as byte
+      exceptionCode = 0;
+    } else if (maxException < (1 << 16)) { // exceptions as 2 bytes
+      exceptionCode = 1;
+    } else /* if (maxException < (1L << 32)) */ { // exceptions as 4 bytes
+      exceptionCode = 2;
+    }
+    encodeHeader(unComprSize, firstExceptionIndex);
+    encodeExceptionValues(exceptionValues);
+  }
+
+  /** Return the number bytes used for a single exception */
+  private int exceptionByteSize() {
+    assert exceptionCode >= 0;
+    assert exceptionCode <= 2;
+    return exceptionCode == 0 ? 1
+          : exceptionCode == 1 ? 2
+          : 4;
+  }
+  
+  /** Return the number of exceptions.
+   *  Only valid after compress() or decompress().
+   */
+  public int getNumExceptions() {
+    return numExceptions;
+  }
+
+  private int compressedArrayByteSize() {
+    int compressedArrayBits = unComprSize * numFrameBits;
+    return (compressedArrayBits + 7) / 8;
+  }
+
+  /** Return the number of integers used in IntBuffer.
+   *  Only valid after compress() or decompress().
+   */
+  public int compressedSize() {
+    // numExceptions only valid after compress() or decompress()
+    return HEADER_SIZE
+           + ((compressedArrayByteSize()
+               + exceptionByteSize() * numExceptions
+               + 3) >> 2); // round up to next multiple of 4 and divide by 4
+  }
+  
+  private void encodeExceptionValues(int[] exceptionValues) {
+    if ((compressedBuffer == null) || (numExceptions == 0)) {
+      return;
+    }
+    int excByteOffset = compressedArrayByteSize();
+
+    switch (exceptionCode) {
+      case 0: { // 1 byte exceptions
+        int i = 0;
+        do { 
+          int intIndex = COMPRESSED_INDEX + (excByteOffset >> 2); // round down here.
+          setBufferIntBits(intIndex, ((excByteOffset & 3) * 8), 8, exceptionValues[i]);
+          excByteOffset++;
+        } while (++i < numExceptions);
+      }
+      break;
+
+      case 1: { // 2 byte exceptions
+        int excShortOffset = (excByteOffset + 1) >> 1; // to next multiple of two bytes.
+        int intIndex = COMPRESSED_INDEX + (excShortOffset >> 1); // round down here.
+        int i = 0;
+        if ((excShortOffset & 1) != 0) { // encode first 2 byte exception in high 2 bytes of same int as last frame bits.
+          setBufferIntBits(intIndex, 16, 16, exceptionValues[i]);
+          intIndex++;
+          i++;
+        }
+        for (; i < (numExceptions-1); i += 2) {
+          compressedBuffer.put(intIndex++, (exceptionValues[i+1] << 16) | exceptionValues[i]);
+        }
+        if (i < numExceptions) {
+          compressedBuffer.put(intIndex, exceptionValues[i]); // also clear the high 16 bits
+        }
+      }
+      break;
+
+      case 2: { // 4 byte exceptions
+        int excIntOffSet = COMPRESSED_INDEX + ((excByteOffset + 3) >> 2); // to next multiple of four bytes, in ints.
+        int i = 0;
+        do {
+          compressedBuffer.put(excIntOffSet + i, exceptionValues[i]);
+        } while(++i < numExceptions);
+      }
+      break;
+    }
+  }
+
+  /** Decode the exception values while going through the exception chain.
+   * <br>For performance, delegate/subclass this to classes with fixed exceptionCode.
+   * <br> Also, decoding exceptions is preferably done from an int border instead of
+   * from a random byte directly after the compressed array. This will allow faster
+   * decoding of exceptions, at the cost of at most 3 bytes.
+   * <br>When ((numFrameBits * unComprSize) % 32) == 0, this cost will always be
+   * zero bytes so specialize for these cases.
+   */
+  private void patchExceptions() {
+    numExceptions = 0;
+    if (firstExceptionIndex == -1) {
+      return;
+    }
+    int excIndex = firstExceptionIndex;
+    int excByteOffset = compressedArrayByteSize();
+    int excValue;
+    int intIndex;
+
+    switch (exceptionCode) {
+      case 0: { // 1 byte exceptions
+        do {
+          intIndex = COMPRESSED_INDEX + (excByteOffset >> 2);
+          int firstBitPosition = (excByteOffset & 3) << 3;
+          excValue = (compressedBuffer.get(intIndex) >>> firstBitPosition) & ((1 << 8) - 1);
+          excIndex = patch(excIndex, excValue);
+          excByteOffset++;
+        } while (excIndex < unComprSize);
+      }
+      break;
+
+      case 1: { // 2 byte exceptions
+        int excShortOffset = (excByteOffset + 1) >> 1; // to next multiple of two bytes.
+        intIndex = COMPRESSED_INDEX + (excShortOffset >> 1); // round down here.
+        if ((excShortOffset & 1) != 0) {
+          // decode first 2 byte exception from high 2 bytes of same int as last frame bits.
+          excValue = compressedBuffer.get(intIndex++) >>> 16;
+          excIndex = patch(excIndex, excValue);
+        }
+        while (excIndex < unComprSize) {
+          excValue = compressedBuffer.get(intIndex) & ((1<<16)-1);
+          excIndex = patch(excIndex, excValue);
+          if (excIndex >= unComprSize) {
+            break;
+          }
+          excValue = compressedBuffer.get(intIndex++) >>> 16;
+          excIndex = patch(excIndex, excValue);
+        }
+      }
+      break;
+
+      case 2: // 4 byte exceptions
+        intIndex = COMPRESSED_INDEX + ((excByteOffset + 3) >> 2); // to next multiple of four bytes, in ints.
+        do {
+          excValue = compressedBuffer.get(intIndex++);
+          excIndex = patch(excIndex, excValue);
+        } while (excIndex < unComprSize);
+      break;
+    }
+  }
+
+  /** The 4 byte header (32 bits) contains:
+   *
+   * - 4 bits for the compression method: 0b0001 for PFor
+   * - 4 bits unused
+   *
+   * - 5 bits for (numFrameBits-1)
+   * - 2 bits for the exception code: 0b00: byte, 0b01: short, 0b10: int, 0b11: long (unused).
+   * - 1 bit unused
+   *
+   * - 8 bits for uncompressed input size - 1,
+   *
+   * - 8 bits for the index of the first exception + 1, (0 when no exceptions)
+   */
+  private void encodeHeader(int unComprSize, int firstExceptionIndex) {
+    assert exceptionCode >= 0;
+    assert exceptionCode <= 2; // 3 for long, but unused for now.
+    assert numFrameBits >= 1;
+    assert numFrameBits <= 32;
+    assert unComprSize >= 1;
+    assert unComprSize <= 128;
+    assert firstExceptionIndex >= -1;
+    assert firstExceptionIndex < unComprSize;
+    if (compressedBuffer != null) {
+      compressedBuffer.put(HEADER_INDEX,
+              ((firstExceptionIndex+1) << 24)
+            | ((unComprSize-1) << 16)
+            | ((exceptionCode & 3) << 13) | ((numFrameBits-1) << 8) 
+            | (PFOR_COMPRESSION << 4));
+    }
+  }
+
+  protected void decodeHeader() {
+    int header = compressedBuffer.get();
+    firstExceptionIndex = ((header >>> 24) & 255) - 1; 
+    //unComprSize = ((header >>> 16) & 255) + 1;
+    numFrameBits = ((header >>> 8) & 31) + 1;
+    assert numFrameBits > 0: numFrameBits;
+    assert numFrameBits <= 32: numFrameBits;
+    // verify compression method:
+    assert PFOR_COMPRESSION == ((header >>> 4) & 15);
+    exceptionCode = (header >>> 13) & 3;
+    assert exceptionCode <= 2;
+  }
+
+  /** Decompress from the buffer into output from a given offset. */
+  public void decompress() {
+    super.decompress();
+    patchExceptions();
+  }
+  
+  /** Patch and return index of next exception */
+  private int patch(int excIndex, int excValue) {
+    int nextExceptionIndex = unCompressedData[excIndex] + excIndex + 1; // chain offset
+    unCompressedData[excIndex + offset] = excValue; // patch
+    assert nextExceptionIndex > excIndex;
+    numExceptions++;
+    return nextExceptionIndex;
+  }
+
+  /** Determine the number of frame bits to be used for compression.
+   * Use only after setUnCompressedData().
+   * This is done by taking a copy of the input, sorting it and using this
+   * to determine the compressed size for each possible numbits in a single pass,
+   * ignoring forced exceptions.
+   * Finally an estimation of the number of forced exceptions is reduced to
+   * less than 1 in 32 input numbers by increasing the number of frame bits.
+   * This implementation works by determining the total number of bytes needed for
+   * the compressed data, but does take into account alignment of exceptions
+   * at 2 or 4 byte boundaries.
+   */
+  public int frameBitsForCompression() {
+    if ((offset + unComprSize) > unCompressedData.length) {
+      throw new IllegalArgumentException( "(offset " + offset
+                                          + " + unComprSize " + unComprSize
+                                          + ") > unCompressedData.length " + unCompressedData.length);
+    }
+    int copy[] = Arrays.copyOfRange(unCompressedData, offset, offset + unComprSize);
+    assert copy.length == unComprSize;
+    Arrays.sort(copy);
+    int maxValue = copy[copy.length-1];
+    if (maxValue <= 1) {
+      return 1;
+    }
+    int bytesPerException = (maxValue < (1 << 8)) ? 1 : (maxValue < (1 << 16)) ? 2 : 4;
+    int frameBits = 1;
+    int bytesForFrame = (copy.length * frameBits + 7) / 8;
+    // initially assume all input is an exception.
+    int totalBytes = bytesForFrame + copy.length * bytesPerException; // excluding the header.
+    int bestBytes = totalBytes;
+    int bestFrameBits = frameBits;
+    int bestExceptions = copy.length;
+    for (int i = 0; i < copy.length; i++) {
+      // determine frameBits so that copy[i] is no more exception
+      while (copy[i] >= (1 << frameBits)) {
+        if (frameBits == 30) { // no point to increase further.
+          return bestFrameBits;
+        }
+        ++frameBits;
+        // increase bytesForFrame and totalBytes to correspond to frameBits
+        int newBytesForFrame = (copy.length * frameBits + 7) / 8;
+        totalBytes += newBytesForFrame - bytesForFrame;
+        bytesForFrame = newBytesForFrame;
+      }
+      totalBytes -= bytesPerException; // no more need to store copy[i] as exception
+      if (totalBytes <= bestBytes) { // <= : prefer fewer exceptions at higher number of frame bits.
+        bestBytes = totalBytes;
+        bestFrameBits = frameBits;
+        bestExceptions = (copy.length - i - 1);
+      }
+    }
+    if (bestExceptions > 0) { // check for forced exceptions.
+      // This ignores the position of the first exception, for which enough bits are available in the header.
+      int allowedNumExceptions = bestExceptions + (copy.length >> 5); // 1 in 32 is allowed to be forced.
+      // (copy.length >> bestFrameBits): Minimum exception chain size including forced ones,
+      // ignoring the position of the first exception.
+      while (allowedNumExceptions < (copy.length >> bestFrameBits)) { // Too many forced?
+        bestFrameBits++; // Reduce forced exceptions and perhaps reduce actual exceptions
+        // Dilemma: decompression speed reduces with increasing number of frame bits,
+        // so it may be better to increase no more than once or twice here.
+      }
+    }
+    return bestFrameBits;
+  }
+}

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/gendecompress.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/gendecompress.py?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/gendecompress.py (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/gendecompress.py Wed Dec 15 10:44:06 2010
@@ -0,0 +1,121 @@
+"""
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+  
+     http://www.apache.org/licenses/LICENSE-2.0
+  
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+"""
+
+"""
+Generate source code for java classes for FOR decompression.
+"""
+
+USE_SCRATCH = False
+
+def bitsExpr(i, numFrameBits):
+  framePos = i * numFrameBits
+  intValNum = (framePos / 32)
+  bitPos = framePos % 32
+  if USE_SCRATCH:
+    bitsInInt = "inputInts[" + str(intValNum) + "]"
+  else:
+    bitsInInt = "intValue" + str(intValNum)
+  needBrackets = 0
+  if bitPos > 0:
+    bitsInInt +=  " >>> " + str(bitPos)
+    needBrackets = 1
+  if bitPos + numFrameBits > 32:
+    if needBrackets:
+      bitsInInt = "(" + bitsInInt + ")"
+    if USE_SCRATCH:
+      bitsInInt += " | (inputInts[" + str(intValNum+1) + "] << "+ str(32 - bitPos) + ")"
+    else:
+      bitsInInt += " | (intValue" + str(intValNum+1) + " << "+ str(32 - bitPos) + ")"
+    needBrackets = 1
+  if bitPos + numFrameBits != 32:
+    if needBrackets:
+      bitsInInt = "(" + bitsInInt + ")"
+    bitsInInt += " & mask"
+  return bitsInInt
+
+
+def genDecompressClass(numFrameBits):
+  className = "For" + str(numFrameBits) + "Decompress"
+  fileName = className + ".java"
+  imports = "import java.nio.IntBuffer;\n"
+  f = open(fileName, 'w')
+  w = f.write
+  try:
+    w("package org.apache.lucene.util.pfor;\n")
+    w("""/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */""")
+    w("\n/* This program is generated, do not modify. See gendecompress.py */\n\n")
+    w("import java.nio.IntBuffer;\n")
+    w("class " + className + " extends ForDecompress {\n")
+    w("  static final int numFrameBits = " + str(numFrameBits) + ";\n")
+    w("  static final int mask = (int) ((1L<<numFrameBits) - 1);\n")
+    w("\n")
+    w("""  static void decompressFrame(FrameOfRef frameOfRef) {
+    int[] output = frameOfRef.unCompressedData;
+    IntBuffer compressedBuffer = frameOfRef.compressedBuffer;
+    int outputOffset = frameOfRef.offset;
+    //int inputSize = frameOfRef.unComprSize;\n""")
+    if USE_SCRATCH:
+      w('    final int[] inputInts = frameOfRef.scratch;\n')
+    #w("    while (inputSize >= 32) {\n")
+    w('    for(int step=0;step<4;step++) {\n')
+    if USE_SCRATCH:
+      w('      compressedBuffer.get(inputInts, 0, %d);\n' % numFrameBits)
+    else:
+      for i in range(numFrameBits): # declare int vars and init from buffer
+        w("      int intValue" + str(i) + " = compressedBuffer.get();\n")
+
+    for i in range(32): # set output from int vars
+      w("      output[" + str(i) + " + outputOffset] = " + bitsExpr(i, numFrameBits) + ";\n")
+    w("""      // inputSize -= 32;
+      outputOffset += 32;
+    }
+    
+    //if (inputSize > 0) {
+    //  decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+    //}
+  }
+}
+""")
+  finally: f.close()
+  
+  
+
+def genDecompressClasses():
+  numFrameBits = 1
+  while numFrameBits <= 31: # 32 special case, not generated.
+    genDecompressClass(numFrameBits)
+    numFrameBits += 1
+
+
+
+if __name__ == "__main__":
+  genDecompressClasses()

Added: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/package.html?rev=1049480&view=auto
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/package.html (added)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/pfor/package.html Wed Dec 15 10:44:06 2010
@@ -0,0 +1,25 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+Classes dealing with (patched) frame of reference compression and decompression.
+</body>
+</html>



Mime
View raw message