lucene-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] [lucene-solr] jpountz commented on a change in pull request #973: LUCENE-9027: Use SIMD instructions to decode postings.
Date Wed, 13 Nov 2019 15:50:28 GMT
jpountz commented on a change in pull request #973: LUCENE-9027: Use SIMD instructions to decode
postings.
URL: https://github.com/apache/lucene-solr/pull/973#discussion_r345838605
 
 

 ##########
 File path: lucene/core/src/java/org/apache/lucene/codecs/lucene84/gen_ForUtil.py
 ##########
 @@ -0,0 +1,341 @@
+#! /usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from fractions import gcd
+
+"""Code generation for ForUtil.java"""
+
+MAX_SPECIALIZED_BITS_PER_VALUE = 24
+OUTPUT_FILE = "ForUtil.java"
+HEADER = """// This file has been automatically generated, DO NOT EDIT
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene84;
+
+import java.io.IOException;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+import java.util.function.IntToLongFunction;
+
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+
+// Inspired from https://fulmicoton.com/posts/bitpacking/
+// Encodes multiple integers in a long to get SIMD-like speedups.
+// If bitsPerValue <= 8 then we pack 8 ints per long
+// else if bitsPerValue <= 16 we pack 4 ints per long
+// else we pack 2 ints per long
+final class ForUtil {
+
+  static final int BLOCK_SIZE = 128;
+  private static final int BLOCK_SIZE_LOG2 = 7;
+
+  private static long expandMask32(long mask32) {
+    return mask32 | (mask32 << 32);
+  }
+
+  private static long expandMask16(long mask16) {
+    return expandMask32(mask16 | (mask16 << 16));
+  }
+
+  private static long expandMask8(long mask8) {
+    return expandMask16(mask8 | (mask8 << 8));
+  }
+
+  private static long mask32(int bitsPerValue) {
+    return expandMask32((1L << bitsPerValue) - 1);
+  }
+
+  private static long mask16(int bitsPerValue) {
+    return expandMask16((1L << bitsPerValue) - 1);
+  }
+
+  private static long mask8(int bitsPerValue) {
+    return expandMask8((1L << bitsPerValue) - 1);
+  }
+
+  private static void expand8(long[] arr) {
+    for (int i = 0; i < 16; ++i) {
+      long l = arr[i];
+      arr[i] = (l >>> 56) & 0xFFL;
+      arr[16+i] = (l >>> 48) & 0xFFL;
+      arr[32+i] = (l >>> 40) & 0xFFL;
+      arr[48+i] = (l >>> 32) & 0xFFL;
+      arr[64+i] = (l >>> 24) & 0xFFL;
+      arr[80+i] = (l >>> 16) & 0xFFL;
+      arr[96+i] = (l >>> 8) & 0xFFL;
+      arr[112+i] = l & 0xFFL;
+    }
+  }
+
+  private static void collapse8(long[] arr) {
+    for (int i = 0; i < 16; ++i) {
+      arr[i] = (arr[i] << 56) | (arr[16+i] << 48) | (arr[32+i] << 40) |
(arr[48+i] << 32) | (arr[64+i] << 24) | (arr[80+i] << 16) | (arr[96+i] <<
8) | arr[112+i];
+    }
+  }
+
+  private static void expand16(long[] arr) {
+    for (int i = 0; i < 32; ++i) {
+      long l = arr[i];
+      arr[i] = (l >>> 48) & 0xFFFFL;
+      arr[32+i] = (l >>> 32) & 0xFFFFL;
+      arr[64+i] = (l >>> 16) & 0xFFFFL;
+      arr[96+i] = l & 0xFFFFL;
+    }
+  }
+
+  private static void collapse16(long[] arr) {
+    for (int i = 0; i < 32; ++i) {
+      arr[i] = (arr[i] << 48) | (arr[32+i] << 32) | (arr[64+i] << 16) |
arr[96+i];
+    }
+  }
+
+  private static void expand32(long[] arr) {
+    for (int i = 0; i < 64; ++i) {
+      long l = arr[i];
+      arr[i] = l >>> 32;
+      arr[64 + i] = l & 0xFFFFFFFFL;
+    }
+  }
+
+  private static void collapse32(long[] arr) {
+    for (int i = 0; i < 64; ++i) {
+      arr[i] = (arr[i] << 32) | arr[64+i];
+    }
+  }
+
+  private final ByteOrder byteOrder;
+  private final long[] tmp = new long[BLOCK_SIZE/2];
+
+  ForUtil(ByteOrder byteOrder) {
+    this.byteOrder = byteOrder;
+  }
+
+  /**
+   * Encode 128 8-bits integers from {@code data} into {@code out}.
+   */
+  void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException {
+    final int nextPrimitive;
+    final int numLongs;
+    final IntToLongFunction maskFunction;
+    if (bitsPerValue <= 8) {
+      nextPrimitive = 8;
+      numLongs = BLOCK_SIZE / 8;
+      maskFunction = ForUtil::mask8;
+      collapse8(longs);
+    } else if (bitsPerValue <= 16) {
+      nextPrimitive = 16;
+      numLongs = BLOCK_SIZE / 4;
+      maskFunction = ForUtil::mask16;
+      collapse16(longs);
+    } else {
+      nextPrimitive = 32;
+      numLongs = BLOCK_SIZE / 2;
+      maskFunction = ForUtil::mask32;
+      collapse32(longs);
+    }
+
+    final int numLongsPerShift = bitsPerValue * 2;
+    Arrays.fill(tmp, 0L);
+    int idx = 0;
+    for (int shift = nextPrimitive - bitsPerValue; shift >= 0; shift -= bitsPerValue)
{
+      for (int i = 0; i < numLongsPerShift; ++i) {
+        tmp[i] |= longs[idx++] << shift;
+      }
+    }
+
+    final int remainingBitsPerLong = nextPrimitive % bitsPerValue;
+    int tmpIdx = 0;
+    int remainingBitsPerValue = bitsPerValue;
+    while (idx < numLongs) {
+      if (remainingBitsPerValue > remainingBitsPerLong) {
+        remainingBitsPerValue -= remainingBitsPerLong;
+        tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & maskFunction.applyAsLong(remainingBitsPerLong);
+        if (remainingBitsPerValue == 0) {
+          idx++;
+          remainingBitsPerValue = bitsPerValue;
+        }
+      } else {
+        tmp[tmpIdx] |= (longs[idx++] & maskFunction.applyAsLong(remainingBitsPerValue))
<< (remainingBitsPerLong - remainingBitsPerValue);
+        remainingBitsPerValue = bitsPerValue - remainingBitsPerLong + remainingBitsPerValue;
+        tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & maskFunction.applyAsLong(bitsPerValue
- remainingBitsPerValue);
+      }
+    }
+
+    for (int i = 0; i < numLongsPerShift; ++i) {
+      long l = tmp[i];
+      if (byteOrder != ByteOrder.BIG_ENDIAN) {
 
 Review comment:
   So this one comes from the fact that I wanted to avoid adding new methods to DataOutput.
So even though we read with `DataInput#readLongs(long[])`, we still write with `DataOutput#writeLong(long)`.
Then if I want to write longs with the native byte order we need to check whether it differs
from BIG_ENDIAN which is the endianness of Java longs. I'll add a comment to clarify.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@lucene.apache.org
For additional commands, e-mail: issues-help@lucene.apache.org


Mime
View raw message