arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject [arrow] branch master updated: ARROW-1864: [Java] Upgrade Netty to 4.1.17
Date Fri, 08 Dec 2017 01:20:12 GMT
This is an automated email from the ASF dual-hosted git repository.

wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
     new 543808d  ARROW-1864: [Java] Upgrade Netty to 4.1.17
543808d is described below

commit 543808d5b2b64df008b57efec2ab9057a9bdc723
Author: Shixiong Zhu <zsxwing@gmail.com>
AuthorDate: Thu Dec 7 20:20:07 2017 -0500

    ARROW-1864: [Java] Upgrade Netty to 4.1.17
    
    Upgrade Netty to 4.1.17 since the Netty community will deprecate 4.0.x soon. This PR includes
the following changes:
    - Bump Netty version.
    - Implement new ByteBuf APIs added in Netty 4.1.x: a bunch of get/setXXXLE methods. They
are the opposite of get/setXXX method regarding byte order. E.g., as ArrowBuf is little endian,
`setInt` will put an `int` to the buffer in little endian byte order, while `setIntLE` will
put `int` in big byte endian order. The method naming seems confusing anyway, and I opened
a Netty issue: https://github.com/netty/netty/issues/7465. The user can call these new methods
to get or set multi-byte i [...]
    - Make ArrowByteBufAllocator overwrite AbstractByteBufAllocator.
    
    Author: Shixiong Zhu <zsxwing@gmail.com>
    
    Closes #1376 from zsxwing/ARROW-1864 and squashes the following commits:
    
    96a93e18 [Shixiong Zhu] extend AbstractByteBufAllocator; add javadoc for new methods
    bb973335 [Shixiong Zhu] Add comment for calculateNewCapacity
    555f88ae [Shixiong Zhu] Add methods back
    5e09cca6 [Shixiong Zhu] Upgrade Netty to 4.1.x
---
 .../src/main/java/io/netty/buffer/ArrowBuf.java    | 199 ++++++++++++++++++++-
 .../io/netty/buffer/MutableWrappedByteBuf.java     | 116 +++++++++++-
 .../apache/arrow/memory/ArrowByteBufAllocator.java |  15 +-
 java/pom.xml                                       |   2 +-
 .../apache/arrow/vector/util/MapWithOrdinal.java   |   4 +-
 5 files changed, 320 insertions(+), 16 deletions(-)

diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index e2bbe35..23f5d65 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -23,6 +23,7 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
 import java.nio.channels.GatheringByteChannel;
 import java.nio.channels.ScatteringByteChannel;
 import java.nio.charset.Charset;
@@ -494,6 +495,16 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable
{
   }
 
   @Override
+  public ByteBuf touch() {
+    return this;
+  }
+
+  @Override
+  public ByteBuf touch(Object hint) {
+    return this;
+  }
+
+  @Override
   public long getLong(int index) {
     chk(index, 8);
     final long v = PlatformDependent.getLong(addr(index));
@@ -505,6 +516,17 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable
{
     return Float.intBitsToFloat(getInt(index));
   }
 
+  /**
+   * Gets a 64-bit long integer at the specified absolute {@code index} in
+   * this buffer in Big Endian Byte Order.
+   */
+  @Override
+  public long getLongLE(int index) {
+    chk(index, 8);
+    final long v = PlatformDependent.getLong(addr(index));
+    return Long.reverseBytes(v);
+  }
+
   @Override
   public double getDouble(int index) {
     return Double.longBitsToDouble(getLong(index));
@@ -527,6 +549,17 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable
{
     return v;
   }
 
+  /**
+   * Gets a 32-bit integer at the specified absolute {@code index} in
+   * this buffer in Big Endian Byte Order.
+   */
+  @Override
+  public int getIntLE(int index) {
+    chk(index, 4);
+    final int v = PlatformDependent.getInt(addr(index));
+    return Integer.reverseBytes(v);
+  }
+
   @Override
   public int getUnsignedShort(int index) {
     return getShort(index) & 0xFFFF;
@@ -535,10 +568,44 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable
{
   @Override
   public short getShort(int index) {
     chk(index, 2);
-    short v = PlatformDependent.getShort(addr(index));
+    final short v = PlatformDependent.getShort(addr(index));
     return v;
   }
 
+  /**
+   * Gets a 16-bit short integer at the specified absolute {@code index} in
+   * this buffer in Big Endian Byte Order.
+   */
+  @Override
+  public short getShortLE(int index) {
+    final short v = PlatformDependent.getShort(addr(index));
+    return Short.reverseBytes(v);
+  }
+
+  /**
+   * Gets an unsigned 24-bit medium integer at the specified absolute
+   * {@code index} in this buffer.
+   */
+  @Override
+  public int getUnsignedMedium(int index) {
+    chk(index, 3);
+    final long addr = addr(index);
+    return (PlatformDependent.getByte(addr) & 0xff) << 16 |
+        (PlatformDependent.getShort(addr + 1) & 0xffff);
+  }
+
+  /**
+   * Gets an unsigned 24-bit medium integer at the specified absolute {@code index} in
+   * this buffer in Big Endian Byte Order.
+   */
+  @Override
+  public int getUnsignedMediumLE(int index) {
+    chk(index, 3);
+    final long addr = addr(index);
+    return (PlatformDependent.getByte(addr) & 0xff) |
+        (Short.reverseBytes(PlatformDependent.getShort(addr + 1)) & 0xffff) <<
8;
+  }
+
   @Override
   public ArrowBuf setShort(int index, int value) {
     chk(index, 2);
@@ -546,6 +613,44 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable
{
     return this;
   }
 
+  /**
+   * Sets the specified 16-bit short integer at the specified absolute {@code index}
+   * in this buffer with Big Endian byte order.
+   */
+  @Override
+  public ByteBuf setShortLE(int index, int value) {
+    chk(index, 2);
+    PlatformDependent.putShort(addr(index), Short.reverseBytes((short) value));
+    return this;
+  }
+
+  /**
+   * Sets the specified 24-bit medium integer at the specified absolute
+   * {@code index} in this buffer.
+   */
+  @Override
+  public ByteBuf setMedium(int index, int value) {
+    chk(index, 3);
+    final long addr = addr(index);
+    PlatformDependent.putByte(addr, (byte) (value >>> 16));
+    PlatformDependent.putShort(addr + 1, (short) value);
+    return this;
+  }
+
+
+  /**
+   * Sets the specified 24-bit medium integer at the specified absolute {@code index}
+   * in this buffer with Big Endian byte order.
+   */
+  @Override
+  public ByteBuf setMediumLE(int index, int value) {
+    chk(index, 3);
+    final long addr = addr(index);
+    PlatformDependent.putByte(addr, (byte) value);
+    PlatformDependent.putShort(addr + 1, Short.reverseBytes((short) (value >>> 8)));
+    return this;
+  }
+
   @Override
   public ArrowBuf setInt(int index, int value) {
     chk(index, 4);
@@ -553,6 +658,17 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable
{
     return this;
   }
 
+  /**
+   * Sets the specified 32-bit integer at the specified absolute {@code index}
+   * in this buffer with Big Endian byte order.
+   */
+  @Override
+  public ByteBuf setIntLE(int index, int value) {
+    chk(index, 4);
+    PlatformDependent.putInt(addr(index), Integer.reverseBytes(value));
+    return this;
+  }
+
   @Override
   public ArrowBuf setLong(int index, long value) {
     chk(index, 8);
@@ -560,6 +676,17 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable
{
     return this;
   }
 
+  /**
+   * Sets the specified 64-bit long integer at the specified absolute {@code index}
+   * in this buffer with Big Endian byte order.
+   */
+  @Override
+  public ByteBuf setLongLE(int index, long value) {
+    chk(index, 8);
+    PlatformDependent.putLong(addr(index), Long.reverseBytes(value));
+    return this;
+  }
+
   @Override
   public ArrowBuf setChar(int index, int value) {
     chk(index, 2);
@@ -668,16 +795,46 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable
{
     return getShort(index);
   }
 
+  /** @see  {@link #getShortLE(int)} */
+  @Override
+  protected short _getShortLE(int index) {
+    return getShortLE(index);
+  }
+
   @Override
   protected int _getInt(int index) {
     return getInt(index);
   }
 
+  /** @see  {@link #getIntLE(int)} */
+  @Override
+  protected int _getIntLE(int index) {
+    return getIntLE(index);
+  }
+
+  /** @see  {@link #getUnsignedMedium(int)} */
+  @Override
+  protected int _getUnsignedMedium(int index) {
+    return getUnsignedMedium(index);
+  }
+
+  /** @see  {@link #getUnsignedMediumLE(int)} */
+  @Override
+  protected int _getUnsignedMediumLE(int index) {
+    return getUnsignedMediumLE(index);
+  }
+
   @Override
   protected long _getLong(int index) {
     return getLong(index);
   }
 
+  /** @see  {@link #getLongLE(int)} */
+  @Override
+  protected long _getLongLE(int index) {
+    return getLongLE(index);
+  }
+
   @Override
   protected void _setByte(int index, int value) {
     setByte(index, value);
@@ -688,21 +845,45 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable
{
     setShort(index, value);
   }
 
+  /** @see  {@link #setShortLE(int, int)} */
+  @Override
+  protected void _setShortLE(int index, int value) {
+    setShortLE(index, value);
+  }
+
   @Override
   protected void _setMedium(int index, int value) {
     setMedium(index, value);
   }
 
+  /** @see  {@link #setMediumLE(int, int)} */
+  @Override
+  protected void _setMediumLE(int index, int value) {
+    setMediumLE(index, value);
+  }
+
   @Override
   protected void _setInt(int index, int value) {
     setInt(index, value);
   }
 
+  /** @see  {@link #setIntLE(int, int)} */
+  @Override
+  protected void _setIntLE(int index, int value) {
+    setIntLE(index, value);
+  }
+
   @Override
   protected void _setLong(int index, long value) {
     setLong(index, value);
   }
 
+  /** @see  {@link #setLongLE(int, long)} */
+  @Override
+  public void _setLongLE(int index, long value) {
+    setLongLE(index, value);
+  }
+
   @Override
   public ArrowBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
     udle.getBytes(index + offset, dst, dstIndex, length);
@@ -716,16 +897,13 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable
{
   }
 
   @Override
-  protected int _getUnsignedMedium(int index) {
-    final long addr = addr(index);
-    return (PlatformDependent.getByte(addr) & 0xff) << 16 |
-        (PlatformDependent.getByte(addr + 1) & 0xff) << 8 |
-        PlatformDependent.getByte(addr + 2) & 0xff;
+  public int getBytes(int index, GatheringByteChannel out, int length) throws IOException
{
+    return udle.getBytes(index + offset, out, length);
   }
 
   @Override
-  public int getBytes(int index, GatheringByteChannel out, int length) throws IOException
{
-    return udle.getBytes(index + offset, out, length);
+  public int getBytes(int index, FileChannel out, long position, int length) throws IOException
{
+    return udle.getBytes(index + offset, out, position, length);
   }
 
   @Override
@@ -777,6 +955,11 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable
{
   }
 
   @Override
+  public int setBytes(int index, FileChannel in, long position, int length) throws IOException
{
+    return udle.setBytes(index + offset, in, position, length);
+  }
+
+  @Override
   public byte getByte(int index) {
     chk(index, 1);
     return PlatformDependent.getByte(addr(index));
diff --git a/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java b/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java
index a5683ad..f0bc84c 100644
--- a/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java
@@ -23,9 +23,12 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
 import java.nio.channels.GatheringByteChannel;
 import java.nio.channels.ScatteringByteChannel;
 
+import io.netty.util.ByteProcessor;
+
 /**
  * This is basically a complete copy of DuplicatedByteBuf. We copy because we want to override
  * some behaviors and make
@@ -129,6 +132,16 @@ abstract class MutableWrappedByteBuf extends AbstractByteBuf {
   }
 
   @Override
+  public short getShortLE(int index) {
+    return buffer.getShortLE(index);
+  }
+
+  @Override
+  protected short _getShortLE(int index) {
+    return buffer.getShortLE(index);
+  }
+
+  @Override
   public int getUnsignedMedium(int index) {
     return _getUnsignedMedium(index);
   }
@@ -139,6 +152,16 @@ abstract class MutableWrappedByteBuf extends AbstractByteBuf {
   }
 
   @Override
+  public int getUnsignedMediumLE(int index) {
+    return buffer.getUnsignedMediumLE(index);
+  }
+
+  @Override
+  protected int _getUnsignedMediumLE(int index) {
+    return buffer.getUnsignedMediumLE(index);
+  }
+
+  @Override
   public int getInt(int index) {
     return _getInt(index);
   }
@@ -149,6 +172,16 @@ abstract class MutableWrappedByteBuf extends AbstractByteBuf {
   }
 
   @Override
+  public int getIntLE(int index) {
+    return buffer.getIntLE(index);
+  }
+
+  @Override
+  protected int _getIntLE(int index) {
+    return buffer.getIntLE(index);
+  }
+
+  @Override
   public long getLong(int index) {
     return _getLong(index);
   }
@@ -159,6 +192,16 @@ abstract class MutableWrappedByteBuf extends AbstractByteBuf {
   }
 
   @Override
+  public long getLongLE(int index) {
+    return buffer.getLongLE(index);
+  }
+
+  @Override
+  protected long _getLongLE(int index) {
+    return buffer.getLongLE(index);
+  }
+
+  @Override
   public abstract ByteBuf copy(int index, int length);
 
   @Override
@@ -207,6 +250,17 @@ abstract class MutableWrappedByteBuf extends AbstractByteBuf {
   }
 
   @Override
+  public ByteBuf setShortLE(int index, int value) {
+    buffer.setShortLE(index, value);
+    return this;
+  }
+
+  @Override
+  protected void _setShortLE(int index, int value) {
+    buffer.setShortLE(index, value);
+  }
+
+  @Override
   public ByteBuf setMedium(int index, int value) {
     _setMedium(index, value);
     return this;
@@ -218,6 +272,17 @@ abstract class MutableWrappedByteBuf extends AbstractByteBuf {
   }
 
   @Override
+  public ByteBuf setMediumLE(int index, int value) {
+    buffer.setMediumLE(index, value);
+    return this;
+  }
+
+  @Override
+  protected void _setMediumLE(int index, int value) {
+    buffer.setMediumLE(index, value);
+  }
+
+  @Override
   public ByteBuf setInt(int index, int value) {
     _setInt(index, value);
     return this;
@@ -229,6 +294,17 @@ abstract class MutableWrappedByteBuf extends AbstractByteBuf {
   }
 
   @Override
+  public ByteBuf setIntLE(int index, int value) {
+    buffer.setIntLE(index, value);
+    return this;
+  }
+
+  @Override
+  protected void _setIntLE(int index, int value) {
+    buffer.setIntLE(index, value);
+  }
+
+  @Override
   public ByteBuf setLong(int index, long value) {
     _setLong(index, value);
     return this;
@@ -240,6 +316,17 @@ abstract class MutableWrappedByteBuf extends AbstractByteBuf {
   }
 
   @Override
+  public ByteBuf setLongLE(int index, long value) {
+    buffer.setLongLE(index, value);
+    return this;
+  }
+
+  @Override
+  protected void _setLongLE(int index, long value) {
+    buffer.setLongLE(index, value);
+  }
+
+  @Override
   public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
     buffer.setBytes(index, src, srcIndex, length);
     return this;
@@ -258,6 +345,12 @@ abstract class MutableWrappedByteBuf extends AbstractByteBuf {
   }
 
   @Override
+  public int setBytes(int index, FileChannel in, long position, int length)
+      throws IOException {
+    return buffer.setBytes(index, in, position, length);
+  }
+
+  @Override
   public ByteBuf getBytes(int index, OutputStream out, int length)
       throws IOException {
     buffer.getBytes(index, out, length);
@@ -282,6 +375,13 @@ abstract class MutableWrappedByteBuf extends AbstractByteBuf {
     return buffer.setBytes(index, in, length);
   }
 
+
+  @Override
+  public int getBytes(int index, FileChannel out, long position, int length)
+      throws IOException {
+    return buffer.getBytes(index, out, position, length);
+  }
+
   @Override
   public int nioBufferCount() {
     return buffer.nioBufferCount();
@@ -298,12 +398,12 @@ abstract class MutableWrappedByteBuf extends AbstractByteBuf {
   }
 
   @Override
-  public int forEachByte(int index, int length, ByteBufProcessor processor) {
+  public int forEachByte(int index, int length, ByteProcessor processor) {
     return buffer.forEachByte(index, length, processor);
   }
 
   @Override
-  public int forEachByteDesc(int index, int length, ByteBufProcessor processor) {
+  public int forEachByteDesc(int index, int length, ByteProcessor processor) {
     return buffer.forEachByteDesc(index, length, processor);
   }
 
@@ -313,6 +413,18 @@ abstract class MutableWrappedByteBuf extends AbstractByteBuf {
   }
 
   @Override
+  public final ByteBuf touch() {
+    unwrap().touch();
+    return this;
+  }
+
+  @Override
+  public final ByteBuf touch(Object hint) {
+    unwrap().touch(hint);
+    return this;
+  }
+
+  @Override
   public final ByteBuf retain() {
     unwrap().retain();
     return this;
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
index b8b5283..9410299 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
@@ -18,8 +18,8 @@
 
 package org.apache.arrow.memory;
 
+import io.netty.buffer.AbstractByteBufAllocator;
 import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufAllocator;
 import io.netty.buffer.CompositeByteBuf;
 import io.netty.buffer.ExpandableByteBuf;
 
@@ -32,7 +32,7 @@ import io.netty.buffer.ExpandableByteBuf;
  * otherwise non-expandable
  * ArrowBufs to be expandable.
  */
-public class ArrowByteBufAllocator implements ByteBufAllocator {
+public class ArrowByteBufAllocator extends AbstractByteBufAllocator {
 
   private static final int DEFAULT_BUFFER_SIZE = 4096;
   private static final int DEFAULT_MAX_COMPOSITE_COMPONENTS = 16;
@@ -142,8 +142,17 @@ public class ArrowByteBufAllocator implements ByteBufAllocator {
     throw fail();
   }
 
+  @Override
+  protected ByteBuf newHeapBuffer(int initialCapacity, int maxCapacity) {
+    throw fail();
+  }
+
+  @Override
+  protected ByteBuf newDirectBuffer(int initialCapacity, int maxCapacity) {
+    return buffer(initialCapacity, maxCapacity);
+  }
+
   private RuntimeException fail() {
     throw new UnsupportedOperationException("Allocator doesn't support heap-based memory.");
   }
-
 }
diff --git a/java/pom.xml b/java/pom.xml
index c479d65..162c534 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -32,7 +32,7 @@
     <dep.junit.version>4.11</dep.junit.version>
     <dep.slf4j.version>1.7.25</dep.slf4j.version>
     <dep.guava.version>18.0</dep.guava.version>
-    <dep.netty.version>4.0.49.Final</dep.netty.version>
+    <dep.netty.version>4.1.17.Final</dep.netty.version>
     <dep.jackson.version>2.7.9</dep.jackson.version>
     <dep.hadoop.version>2.7.1</dep.hadoop.version>
     <fbs.version>1.2.0-3f79e055</fbs.version>
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
index 6d3b390..b863fa8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
@@ -134,9 +134,9 @@ public class MapWithOrdinal<K, V> implements Map<K, V> {
 
     @Override
     public Collection<V> values() {
-      return Lists.newArrayList(Iterables.transform(secondary.entries(), new Function<IntObjectMap.Entry<V>,
V>() {
+      return Lists.newArrayList(Iterables.transform(secondary.entries(), new Function<IntObjectMap.PrimitiveEntry<V>,
V>() {
         @Override
-        public V apply(IntObjectMap.Entry<V> entry) {
+        public V apply(IntObjectMap.PrimitiveEntry<V> entry) {
           return Preconditions.checkNotNull(entry).value();
         }
       }));

-- 
To stop receiving notification emails like this one, please contact
['"commits@arrow.apache.org" <commits@arrow.apache.org>'].

Mime
View raw message