lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dwe...@apache.org
Subject [1/2] lucene-solr:master: LUCENE-8625: int overflow in ByteBuffersDataInput.sliceBufferList
Date Fri, 28 Dec 2018 11:11:35 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/branch_7x 4bcf34a9b -> a56297d71
  refs/heads/master 45c50a027 -> c9c3ef0ef


LUCENE-8625: int overflow in ByteBuffersDataInput.sliceBufferList


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c9c3ef0e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c9c3ef0e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c9c3ef0e

Branch: refs/heads/master
Commit: c9c3ef0ef9251ee07fc65c1e06b5e887485f1f31
Parents: 45c50a0
Author: Dawid Weiss <dweiss@apache.org>
Authored: Fri Dec 28 10:54:16 2018 +0100
Committer: Dawid Weiss <dweiss@apache.org>
Committed: Fri Dec 28 12:10:48 2018 +0100

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |  3 ++
 .../lucene/store/ByteBuffersDataInput.java      | 10 ++--
 .../lucene/store/TestByteBuffersDataInput.java  | 49 ++++++++++++++++++++
 3 files changed, 57 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c9c3ef0e/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 242a45f..7ce6f46 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -243,6 +243,9 @@ Bug fixes:
 * LUCENE-8624: int overflow in ByteBuffersDataOutput.size(). (Mulugeta Mammo, 
   Dawid Weiss)
 
+* LUCENE-8625: int overflow in ByteBuffersDataInput.sliceBufferList. (Mulugeta Mammo,
+  Dawid Weiss)
+
 New Features
 
 * LUCENE-8026: ExitableDirectoryReader may now time out queries that run on

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c9c3ef0e/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java
index 9b4f4af..bc6cec5 100644
--- a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java
@@ -290,17 +290,17 @@ public final class ByteBuffersDataInput extends DataInput implements
Accountable
     if (buffers.size() == 1) {
       ByteBuffer cloned = buffers.get(0).asReadOnlyBuffer();
       cloned.position(Math.toIntExact(cloned.position() + offset));
-      cloned.limit(Math.toIntExact(length + cloned.position()));
+      cloned.limit(Math.toIntExact(cloned.position() + length));
       return Arrays.asList(cloned);
     } else {
       long absStart = buffers.get(0).position() + offset;
-      long absEnd = Math.toIntExact(absStart + length);
+      long absEnd = absStart + length;
 
       int blockBytes = ByteBuffersDataInput.determineBlockPage(buffers);
       int blockBits = Integer.numberOfTrailingZeros(blockBytes);
-      int blockMask = (1 << blockBits) - 1;
+      long blockMask = (1L << blockBits) - 1;
 
-      int endOffset = (int) absEnd & blockMask;
+      int endOffset = Math.toIntExact(absEnd & blockMask);
 
       ArrayList<ByteBuffer> cloned = 
         buffers.subList(Math.toIntExact(absStart / blockBytes), 
@@ -313,7 +313,7 @@ public final class ByteBuffersDataInput extends DataInput implements Accountable
         cloned.add(ByteBuffer.allocate(0));
       }
 
-      cloned.get(0).position((int) absStart & blockMask);
+      cloned.get(0).position(Math.toIntExact(absStart & blockMask));
       cloned.get(cloned.size() - 1).limit(endOffset);
       return cloned;
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c9c3ef0e/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java
index 5d3d7f6..20e3bd2 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.*;
 import java.io.EOFException;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.lucene.util.ArrayUtil;
@@ -203,4 +204,52 @@ public final class TestByteBuffersDataInput extends RandomizedTest {
       in.readBytes(ByteBuffer.allocate(100), 100);
     });
   }
+
+  // https://issues.apache.org/jira/browse/LUCENE-8625
+  @Test
+  public void testSlicingLargeBuffers() throws IOException {
+    // Simulate a "large" (> 4GB) input by duplicating
+    // buffers with the same content.
+    int MB = 1024 * 1024;
+    byte [] pageBytes = randomBytesOfLength(4 * MB);
+    ByteBuffer page = ByteBuffer.wrap(pageBytes);
+
+    // Add some head shift on the first buffer.
+    final int shift = randomIntBetween(0, pageBytes.length / 2);
+
+    final long simulatedLength =
+        randomLongBetween(0, 2018) + 4L * Integer.MAX_VALUE;
+
+    List<ByteBuffer> buffers = new ArrayList<>();
+    long remaining = simulatedLength + shift;
+    while (remaining > 0) {
+      ByteBuffer bb = page.duplicate();
+      if (bb.remaining() > remaining) {
+        bb.limit(Math.toIntExact(bb.position() + remaining));
+      }
+      buffers.add(bb);
+      remaining -= bb.remaining();
+    }
+    buffers.get(0).position(shift);
+
+    ByteBuffersDataInput dst = new ByteBuffersDataInput(buffers);
+    assertEquals(simulatedLength, dst.size());
+
+    final long max = dst.size();
+    long offset = 0;
+    for (; offset < max; offset += randomIntBetween(MB, 4 * MB)) {
+      assertEquals(0, dst.slice(offset, 0).size());
+      assertEquals(1, dst.slice(offset, 1).size());
+
+      long window = Math.min(max - offset, 1024);
+      ByteBuffersDataInput slice = dst.slice(offset, window);
+      assertEquals(window, slice.size());
+
+      // Sanity check of the content against original pages.
+      for (int i = 0; i < window; i++) {
+        byte expected = pageBytes[(int) ((shift + offset + i) % pageBytes.length)];
+        assertEquals(expected, slice.readByte(i));
+      }
+    }
+  }
 }


Mime
View raw message