Return-Path: X-Original-To: apmail-arrow-commits-archive@minotaur.apache.org Delivered-To: apmail-arrow-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id AB35D18A07 for ; Sat, 20 Feb 2016 18:35:18 +0000 (UTC) Received: (qmail 42876 invoked by uid 500); 20 Feb 2016 18:35:18 -0000 Delivered-To: apmail-arrow-commits-archive@arrow.apache.org Received: (qmail 42865 invoked by uid 500); 20 Feb 2016 18:35:18 -0000 Mailing-List: contact commits-help@arrow.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@arrow.apache.org Delivered-To: mailing list commits@arrow.apache.org Received: (qmail 42856 invoked by uid 99); 20 Feb 2016 18:35:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 20 Feb 2016 18:35:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 2F619E0104; Sat, 20 Feb 2016 18:35:18 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jacques@apache.org To: commits@arrow.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: arrow git commit: ARROW-9: Replace straggler references to Drill Date: Sat, 20 Feb 2016 18:35:18 +0000 (UTC) Repository: arrow Updated Branches: refs/heads/master e9cc8ce39 -> e6905effb ARROW-9: Replace straggler references to Drill - Renaming drill to arrow for TestBaseAllocator - Fix ArrowBuffer as ArrowBuf - Replace Drill with Arrow for ValueHolder This closes #2. Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/e6905eff Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/e6905eff Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/e6905eff Branch: refs/heads/master Commit: e6905effbb9383afd2423a4f86cf9a33ca680b9d Parents: e9cc8ce Author: proflin Authored: Sat Feb 20 15:50:45 2016 +0800 Committer: Jacques Nadeau Committed: Sat Feb 20 10:31:58 2016 -0800 ---------------------------------------------------------------------- .../src/main/java/io/netty/buffer/ArrowBuf.java | 36 +-- .../java/io/netty/buffer/ExpandableByteBuf.java | 2 +- .../netty/buffer/PooledByteBufAllocatorL.java | 6 +- .../netty/buffer/UnsafeDirectLittleEndian.java | 4 +- .../apache/arrow/memory/AllocationManager.java | 34 +-- .../arrow/memory/ArrowByteBufAllocator.java | 141 +++++++++++ .../org/apache/arrow/memory/BaseAllocator.java | 22 +- .../apache/arrow/memory/BufferAllocator.java | 4 +- .../org/apache/arrow/memory/BufferManager.java | 2 +- .../arrow/memory/DrillByteBufAllocator.java | 141 ----------- .../org/apache/arrow/memory/package-info.java | 2 +- .../apache/arrow/memory/TestBaseAllocator.java | 232 +++++++++---------- .../src/main/codegen/templates/ListWriters.java | 2 +- .../vector/complex/AbstractContainerVector.java | 2 +- .../arrow/vector/complex/AbstractMapVector.java | 2 +- .../arrow/vector/holders/ValueHolder.java | 4 +- .../arrow/vector/util/ByteFunctionHelpers.java | 16 +- .../arrow/vector/util/DecimalUtility.java | 16 +- 18 files changed, 334 insertions(+), 334 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java ---------------------------------------------------------------------- diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java index f033ba6..bbec26a 100644 --- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java +++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java @@ -56,7 +56,7 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable { private final boolean isEmpty; private volatile int length; private final HistoricalLog historicalLog = BaseAllocator.DEBUG ? - new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH, "DrillBuf[%d]", id) : null; + new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH, "ArrowBuf[%d]", id) : null; public ArrowBuf( final AtomicInteger refCnt, @@ -155,18 +155,18 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable { } /** - * Create a new DrillBuf that is associated with an alternative allocator for the purposes of memory ownership and - * accounting. This has no impact on the reference counting for the current DrillBuf except in the situation where the + * Create a new ArrowBuf that is associated with an alternative allocator for the purposes of memory ownership and + * accounting. This has no impact on the reference counting for the current ArrowBuf except in the situation where the * passed in Allocator is the same as the current buffer. * - * This operation has no impact on the reference count of this DrillBuf. The newly created DrillBuf with either have a + * This operation has no impact on the reference count of this ArrowBuf. The newly created ArrowBuf with either have a * reference count of 1 (in the case that this is the first time this memory is being associated with the new * allocator) or the current value of the reference count + 1 for the other AllocationManager/BufferLedger combination * in the case that the provided allocator already had an association to this underlying memory. * * @param target * The target allocator to create an association with. - * @return A new DrillBuf which shares the same underlying memory as this DrillBuf. + * @return A new ArrowBuf which shares the same underlying memory as this ArrowBuf. */ public ArrowBuf retain(BufferAllocator target) { @@ -178,17 +178,17 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable { historicalLog.recordEvent("retain(%s)", target.getName()); } final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target); - return otherLedger.newDrillBuf(offset, length, null); + return otherLedger.newArrowBuf(offset, length, null); } /** - * Transfer the memory accounting ownership of this DrillBuf to another allocator. This will generate a new DrillBuf - * that carries an association with the underlying memory of this DrillBuf. If this DrillBuf is connected to the + * Transfer the memory accounting ownership of this ArrowBuf to another allocator. This will generate a new ArrowBuf + * that carries an association with the underlying memory of this ArrowBuf. If this ArrowBuf is connected to the * owning BufferLedger of this memory, that memory ownership/accounting will be transferred to the taret allocator. If - * this DrillBuf does not currently own the memory underlying it (and is only associated with it), this does not - * transfer any ownership to the newly created DrillBuf. + * this ArrowBuf does not currently own the memory underlying it (and is only associated with it), this does not + * transfer any ownership to the newly created ArrowBuf. * - * This operation has no impact on the reference count of this DrillBuf. The newly created DrillBuf with either have a + * This operation has no impact on the reference count of this ArrowBuf. The newly created ArrowBuf with either have a * reference count of 1 (in the case that this is the first time this memory is being associated with the new * allocator) or the current value of the reference count for the other AllocationManager/BufferLedger combination in * the case that the provided allocator already had an association to this underlying memory. @@ -203,7 +203,7 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable { * @param target * The allocator to transfer ownership to. * @return A new transfer result with the impact of the transfer (whether it was overlimit) as well as the newly - * created DrillBuf. + * created ArrowBuf. */ public TransferResult transferOwnership(BufferAllocator target) { @@ -212,7 +212,7 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable { } final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target); - final ArrowBuf newBuf = otherLedger.newDrillBuf(offset, length, null); + final ArrowBuf newBuf = otherLedger.newArrowBuf(offset, length, null); final boolean allocationFit = this.ledger.transferBalance(otherLedger); return new TransferResult(allocationFit, newBuf); } @@ -267,7 +267,7 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable { if (refCnt < 0) { throw new IllegalStateException( - String.format("DrillBuf[%d] refCnt has gone negative. Buffer Info: %s", id, toVerboseString())); + String.format("ArrowBuf[%d] refCnt has gone negative. Buffer Info: %s", id, toVerboseString())); } return refCnt == 0; @@ -370,7 +370,7 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable { * Re the behavior of reference counting, see http://netty.io/wiki/reference-counted-objects.html#wiki-h3-5, which * explains that derived buffers share their reference count with their parent */ - final ArrowBuf newBuf = ledger.newDrillBuf(offset + index, length); + final ArrowBuf newBuf = ledger.newArrowBuf(offset + index, length); newBuf.writerIndex(length); return newBuf; } @@ -437,7 +437,7 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable { @Override public String toString() { - return String.format("DrillBuf[%d], udle: [%d %d..%d]", id, udle.id, offset, offset + capacity()); + return String.format("ArrowBuf[%d], udle: [%d %d..%d]", id, udle.id, offset, offset + capacity()); } @Override @@ -782,7 +782,7 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable { } /** - * Returns the possible memory consumed by this DrillBuf in the worse case scenario. (not shared, connected to larger + * Returns the possible memory consumed by this ArrowBuf in the worse case scenario. (not shared, connected to larger * underlying buffer of allocated memory) * * @return Size in bytes. @@ -833,7 +833,7 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable { } /** - * Get the integer id assigned to this DrillBuf for debugging purposes. + * Get the integer id assigned to this ArrowBuf for debugging purposes. * * @return integer id */ http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java ---------------------------------------------------------------------- diff --git a/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java b/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java index 5988647..7fb884d 100644 --- a/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java +++ b/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java @@ -20,7 +20,7 @@ package io.netty.buffer; import org.apache.arrow.memory.BufferAllocator; /** - * Allows us to decorate DrillBuf to make it expandable so that we can use them in the context of the Netty framework + * Allows us to decorate ArrowBuf to make it expandable so that we can use them in the context of the Netty framework * (thus supporting RPC level memory accounting). */ public class ExpandableByteBuf extends MutableWrappedByteBuf { http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java ---------------------------------------------------------------------- diff --git a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java index 1610028..0b6e3f7 100644 --- a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java +++ b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java @@ -32,7 +32,7 @@ import com.codahale.metrics.MetricFilter; import com.codahale.metrics.MetricRegistry; /** - * The base allocator that we use for all of Drill's memory management. Returns UnsafeDirectLittleEndian buffers. + * The base allocator that we use for all of Arrow's memory management. Returns UnsafeDirectLittleEndian buffers. */ public class PooledByteBufAllocatorL { private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("drill.allocator"); @@ -184,7 +184,7 @@ public class PooledByteBufAllocatorL { private UnsupportedOperationException fail() { return new UnsupportedOperationException( - "Drill requries that the JVM used supports access sun.misc.Unsafe. This platform didn't provide that functionality."); + "Arrow requries that the JVM used supports access sun.misc.Unsafe. This platform didn't provide that functionality."); } public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacity) { @@ -197,7 +197,7 @@ public class PooledByteBufAllocatorL { @Override public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) { - throw new UnsupportedOperationException("Drill doesn't support using heap buffers."); + throw new UnsupportedOperationException("Arrow doesn't support using heap buffers."); } http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java ---------------------------------------------------------------------- diff --git a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java index 6495d5d..a94c6d1 100644 --- a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java +++ b/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java @@ -24,7 +24,7 @@ import java.nio.ByteOrder; import java.util.concurrent.atomic.AtomicLong; /** - * The underlying class we use for little-endian access to memory. Is used underneath DrillBufs to abstract away the + * The underlying class we use for little-endian access to memory. Is used underneath ArrowBufs to abstract away the * Netty classes and underlying Netty memory management. */ public final class UnsafeDirectLittleEndian extends WrappedByteBuf { @@ -55,7 +55,7 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf { private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake, AtomicLong bufferCount, AtomicLong bufferSize) { super(buf); if (!NATIVE_ORDER || buf.order() != ByteOrder.BIG_ENDIAN) { - throw new IllegalStateException("Drill only runs on LittleEndian systems."); + throw new IllegalStateException("Arrow only runs on LittleEndian systems."); } this.bufferCount = bufferCount; http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java ---------------------------------------------------------------------- diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java index 0db6144..37d1d34 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java @@ -41,7 +41,7 @@ import com.google.common.base.Preconditions; * This class is also responsible for managing when memory is allocated and returned to the Netty-based * PooledByteBufAllocatorL. * - * The only reason that this isn't package private is we're forced to put DrillBuf in Netty's package which need access + * The only reason that this isn't package private is we're forced to put ArrowBuf in Netty's package which need access * to these objects or methods. * * Threading: AllocationManager manages thread-safety internally. Operations within the context of a single BufferLedger @@ -185,8 +185,8 @@ public class AllocationManager { /** * The reference manager that binds an allocator manager to a particular BaseAllocator. Also responsible for creating - * a set of DrillBufs that share a common fate and set of reference counts. - * As with AllocationManager, the only reason this is public is due to DrillBuf being in io.netty.buffer package. + * a set of ArrowBufs that share a common fate and set of reference counts. + * As with AllocationManager, the only reason this is public is due to ArrowBuf being in io.netty.buffer package. */ public class BufferLedger { @@ -322,7 +322,7 @@ public class AllocationManager { /** * Returns the ledger associated with a particular BufferAllocator. If the BufferAllocator doesn't currently have a * ledger associated with this AllocationManager, a new one is created. This is placed on BufferLedger rather than - * AllocationManager directly because DrillBufs don't have access to AllocationManager and they are the ones + * AllocationManager directly because ArrowBufs don't have access to AllocationManager and they are the ones * responsible for exposing the ability to associate multiple allocators with a particular piece of underlying * memory. Note that this will increment the reference count of this ledger by one to ensure the ledger isn't * destroyed before use. @@ -335,32 +335,32 @@ public class AllocationManager { } /** - * Create a new DrillBuf associated with this AllocationManager and memory. Does not impact reference count. + * Create a new ArrowBuf associated with this AllocationManager and memory. Does not impact reference count. * Typically used for slicing. * @param offset - * The offset in bytes to start this new DrillBuf. + * The offset in bytes to start this new ArrowBuf. * @param length - * The length in bytes that this DrillBuf will provide access to. - * @return A new DrillBuf that shares references with all DrillBufs associated with this BufferLedger + * The length in bytes that this ArrowBuf will provide access to. + * @return A new ArrowBuf that shares references with all ArrowBufs associated with this BufferLedger */ - public ArrowBuf newDrillBuf(int offset, int length) { + public ArrowBuf newArrowBuf(int offset, int length) { allocator.assertOpen(); - return newDrillBuf(offset, length, null); + return newArrowBuf(offset, length, null); } /** - * Create a new DrillBuf associated with this AllocationManager and memory. + * Create a new ArrowBuf associated with this AllocationManager and memory. * @param offset - * The offset in bytes to start this new DrillBuf. + * The offset in bytes to start this new ArrowBuf. * @param length - * The length in bytes that this DrillBuf will provide access to. + * The length in bytes that this ArrowBuf will provide access to. * @param manager - * An optional BufferManager argument that can be used to manage expansion of this DrillBuf + * An optional BufferManager argument that can be used to manage expansion of this ArrowBuf * @param retain * Whether or not the newly created buffer should get an additional reference count added to it. - * @return A new DrillBuf that shares references with all DrillBufs associated with this BufferLedger + * @return A new ArrowBuf that shares references with all ArrowBufs associated with this BufferLedger */ - public ArrowBuf newDrillBuf(int offset, int length, BufferManager manager) { + public ArrowBuf newArrowBuf(int offset, int length, BufferManager manager) { allocator.assertOpen(); final ArrowBuf buf = new ArrowBuf( @@ -375,7 +375,7 @@ public class AllocationManager { if (BaseAllocator.DEBUG) { historicalLog.recordEvent( - "DrillBuf(BufferLedger, BufferAllocator[%s], UnsafeDirectLittleEndian[identityHashCode == " + "ArrowBuf(BufferLedger, BufferAllocator[%s], UnsafeDirectLittleEndian[identityHashCode == " + "%d](%s)) => ledger hc == %d", allocator.name, System.identityHashCode(buf), buf.toString(), System.identityHashCode(this)); http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java ---------------------------------------------------------------------- diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java new file mode 100644 index 0000000..f3f72fa --- /dev/null +++ b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.memory; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; +import io.netty.buffer.CompositeByteBuf; +import io.netty.buffer.ExpandableByteBuf; + +/** + * An implementation of ByteBufAllocator that wraps a Arrow BufferAllocator. This allows the RPC layer to be accounted + * and managed using Arrow's BufferAllocator infrastructure. The only thin different from a typical BufferAllocator is + * the signature and the fact that this Allocator returns ExpandableByteBufs which enable otherwise non-expandable + * ArrowBufs to be expandable. + */ +public class ArrowByteBufAllocator implements ByteBufAllocator { + + private static final int DEFAULT_BUFFER_SIZE = 4096; + private static final int DEFAULT_MAX_COMPOSITE_COMPONENTS = 16; + + private final BufferAllocator allocator; + + public ArrowByteBufAllocator(BufferAllocator allocator) { + this.allocator = allocator; + } + + @Override + public ByteBuf buffer() { + return buffer(DEFAULT_BUFFER_SIZE); + } + + @Override + public ByteBuf buffer(int initialCapacity) { + return new ExpandableByteBuf(allocator.buffer(initialCapacity), allocator); + } + + @Override + public ByteBuf buffer(int initialCapacity, int maxCapacity) { + return buffer(initialCapacity); + } + + @Override + public ByteBuf ioBuffer() { + return buffer(); + } + + @Override + public ByteBuf ioBuffer(int initialCapacity) { + return buffer(initialCapacity); + } + + @Override + public ByteBuf ioBuffer(int initialCapacity, int maxCapacity) { + return buffer(initialCapacity); + } + + @Override + public ByteBuf directBuffer() { + return buffer(); + } + + @Override + public ByteBuf directBuffer(int initialCapacity) { + return allocator.buffer(initialCapacity); + } + + @Override + public ByteBuf directBuffer(int initialCapacity, int maxCapacity) { + return buffer(initialCapacity, maxCapacity); + } + + @Override + public CompositeByteBuf compositeBuffer() { + return compositeBuffer(DEFAULT_MAX_COMPOSITE_COMPONENTS); + } + + @Override + public CompositeByteBuf compositeBuffer(int maxNumComponents) { + return new CompositeByteBuf(this, true, maxNumComponents); + } + + @Override + public CompositeByteBuf compositeDirectBuffer() { + return compositeBuffer(); + } + + @Override + public CompositeByteBuf compositeDirectBuffer(int maxNumComponents) { + return compositeBuffer(maxNumComponents); + } + + @Override + public boolean isDirectBufferPooled() { + return false; + } + + @Override + public ByteBuf heapBuffer() { + throw fail(); + } + + @Override + public ByteBuf heapBuffer(int initialCapacity) { + throw fail(); + } + + @Override + public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) { + throw fail(); + } + + @Override + public CompositeByteBuf compositeHeapBuffer() { + throw fail(); + } + + @Override + public CompositeByteBuf compositeHeapBuffer(int maxNumComponents) { + throw fail(); + } + + private RuntimeException fail() { + throw new UnsupportedOperationException("Allocator doesn't support heap-based memory."); + } + +} http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java ---------------------------------------------------------------------- diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java index 72f77ab..90257bb 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java @@ -82,7 +82,7 @@ public abstract class BaseAllocator extends Accountant implements BufferAllocato this.parentAllocator = parentAllocator; this.name = name; - this.thisAsByteBufAllocator = new DrillByteBufAllocator(this); + this.thisAsByteBufAllocator = new ArrowByteBufAllocator(this); if (DEBUG) { childAllocators = new IdentityHashMap<>(); @@ -236,7 +236,7 @@ public abstract class BaseAllocator extends Accountant implements BufferAllocato final AllocationManager manager = new AllocationManager(this, size); final BufferLedger ledger = manager.associate(this); // +1 ref cnt (required) - final ArrowBuf buffer = ledger.newDrillBuf(0, size, bufferManager); + final ArrowBuf buffer = ledger.newArrowBuf(0, size, bufferManager); // make sure that our allocation is equal to what we expected. Preconditions.checkArgument(buffer.capacity() == size, @@ -314,9 +314,9 @@ public abstract class BaseAllocator extends Accountant implements BufferAllocato Preconditions.checkState(!closed, "Attempt to allocate after closed"); Preconditions.checkState(!used, "Attempt to allocate more than once"); - final ArrowBuf drillBuf = allocate(nBytes); + final ArrowBuf arrowBuf = allocate(nBytes); used = true; - return drillBuf; + return arrowBuf; } public int getSize() { @@ -397,13 +397,13 @@ public abstract class BaseAllocator extends Accountant implements BufferAllocato * as well, so we need to return the same number back to avoid double-counting them. */ try { - final ArrowBuf drillBuf = BaseAllocator.this.bufferWithoutReservation(nBytes, null); + final ArrowBuf arrowBuf = BaseAllocator.this.bufferWithoutReservation(nBytes, null); if (DEBUG) { - historicalLog.recordEvent("allocate() => %s", String.format("DrillBuf[%d]", drillBuf.getId())); + historicalLog.recordEvent("allocate() => %s", String.format("ArrowBuf[%d]", arrowBuf.getId())); } success = true; - return drillBuf; + return arrowBuf; } finally { if (!success) { releaseBytes(nBytes); @@ -565,7 +565,7 @@ public abstract class BaseAllocator extends Accountant implements BufferAllocato * Verifies the accounting state of the allocator. Only works for DEBUG. * *

- * This overload is used for recursive calls, allowing for checking that DrillBufs are unique across all allocators + * This overload is used for recursive calls, allowing for checking that ArrowBufs are unique across all allocators * that are checked. *

* @@ -594,7 +594,7 @@ public abstract class BaseAllocator extends Accountant implements BufferAllocato * Verify my relationships with my descendants. * * The sum of direct child allocators' owned memory must be <= my allocated memory; my allocated memory also - * includes DrillBuf's directly allocated by me. + * includes ArrowBuf's directly allocated by me. */ long childTotal = 0; for (final BaseAllocator childAllocator : childSet) { @@ -623,11 +623,11 @@ public abstract class BaseAllocator extends Accountant implements BufferAllocato final UnsafeDirectLittleEndian udle = ledger.getUnderlying(); /* - * Even when shared, DrillBufs are rewrapped, so we should never see the same instance twice. + * Even when shared, ArrowBufs are rewrapped, so we should never see the same instance twice. */ final BaseAllocator otherOwner = buffersSeen.get(udle); if (otherOwner != null) { - throw new IllegalStateException("This allocator's drillBuf already owned by another allocator"); + throw new IllegalStateException("This allocator's ArrowBuf already owned by another allocator"); } buffersSeen.put(udle, this); http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java ---------------------------------------------------------------------- diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java index 16a6812..356a341 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java @@ -30,7 +30,7 @@ public interface BufferAllocator extends AutoCloseable { * * @param size * The size in bytes. - * @return a new DrillBuf, or null if the request can't be satisfied + * @return a new ArrowBuf, or null if the request can't be satisfied * @throws OutOfMemoryException * if buffer cannot be allocated */ @@ -44,7 +44,7 @@ public interface BufferAllocator extends AutoCloseable { * The size in bytes. * @param manager * A buffer manager to manage reallocation. - * @return a new DrillBuf, or null if the request can't be satisfied + * @return a new ArrowBuf, or null if the request can't be satisfied * @throws OutOfMemoryException * if buffer cannot be allocated */ http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java ---------------------------------------------------------------------- diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java index 0610ff0..d6470fa 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java @@ -27,7 +27,7 @@ import io.netty.buffer.ArrowBuf; * * The current uses of these types of buffers are within the pluggable components of Drill. * In UDFs, memory management should not be a concern. We provide access to re-allocatable - * DrillBufs to give UDF writers general purpose buffers we can account for. To prevent the need + * ArrowBufs to give UDF writers general purpose buffers we can account for. To prevent the need * for UDFs to contain boilerplate to close all of the buffers they request, this list * is tracked at a higher level and all of the buffers are freed once we are sure that * the code depending on them is done executing (currently {@link FragmentContext} http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/memory/src/main/java/org/apache/arrow/memory/DrillByteBufAllocator.java ---------------------------------------------------------------------- diff --git a/java/memory/src/main/java/org/apache/arrow/memory/DrillByteBufAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/DrillByteBufAllocator.java deleted file mode 100644 index 23d6448..0000000 --- a/java/memory/src/main/java/org/apache/arrow/memory/DrillByteBufAllocator.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.memory; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; -import io.netty.buffer.CompositeByteBuf; -import io.netty.buffer.ExpandableByteBuf; - -/** - * An implementation of ByteBufAllocator that wraps a Drill BufferAllocator. This allows the RPC layer to be accounted - * and managed using Drill's BufferAllocator infrastructure. The only thin different from a typical BufferAllocator is - * the signature and the fact that this Allocator returns ExpandableByteBufs which enable otherwise non-expandable - * DrillBufs to be expandable. - */ -public class DrillByteBufAllocator implements ByteBufAllocator { - - private static final int DEFAULT_BUFFER_SIZE = 4096; - private static final int DEFAULT_MAX_COMPOSITE_COMPONENTS = 16; - - private final BufferAllocator allocator; - - public DrillByteBufAllocator(BufferAllocator allocator) { - this.allocator = allocator; - } - - @Override - public ByteBuf buffer() { - return buffer(DEFAULT_BUFFER_SIZE); - } - - @Override - public ByteBuf buffer(int initialCapacity) { - return new ExpandableByteBuf(allocator.buffer(initialCapacity), allocator); - } - - @Override - public ByteBuf buffer(int initialCapacity, int maxCapacity) { - return buffer(initialCapacity); - } - - @Override - public ByteBuf ioBuffer() { - return buffer(); - } - - @Override - public ByteBuf ioBuffer(int initialCapacity) { - return buffer(initialCapacity); - } - - @Override - public ByteBuf ioBuffer(int initialCapacity, int maxCapacity) { - return buffer(initialCapacity); - } - - @Override - public ByteBuf directBuffer() { - return buffer(); - } - - @Override - public ByteBuf directBuffer(int initialCapacity) { - return allocator.buffer(initialCapacity); - } - - @Override - public ByteBuf directBuffer(int initialCapacity, int maxCapacity) { - return buffer(initialCapacity, maxCapacity); - } - - @Override - public CompositeByteBuf compositeBuffer() { - return compositeBuffer(DEFAULT_MAX_COMPOSITE_COMPONENTS); - } - - @Override - public CompositeByteBuf compositeBuffer(int maxNumComponents) { - return new CompositeByteBuf(this, true, maxNumComponents); - } - - @Override - public CompositeByteBuf compositeDirectBuffer() { - return compositeBuffer(); - } - - @Override - public CompositeByteBuf compositeDirectBuffer(int maxNumComponents) { - return compositeBuffer(maxNumComponents); - } - - @Override - public boolean isDirectBufferPooled() { - return false; - } - - @Override - public ByteBuf heapBuffer() { - throw fail(); - } - - @Override - public ByteBuf heapBuffer(int initialCapacity) { - throw fail(); - } - - @Override - public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) { - throw fail(); - } - - @Override - public CompositeByteBuf compositeHeapBuffer() { - throw fail(); - } - - @Override - public CompositeByteBuf compositeHeapBuffer(int maxNumComponents) { - throw fail(); - } - - private RuntimeException fail() { - throw new UnsupportedOperationException("Allocator doesn't support heap-based memory."); - } - -} http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/memory/src/main/java/org/apache/arrow/memory/package-info.java ---------------------------------------------------------------------- diff --git a/java/memory/src/main/java/org/apache/arrow/memory/package-info.java b/java/memory/src/main/java/org/apache/arrow/memory/package-info.java index 712af30..40d25ca 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/package-info.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/package-info.java @@ -18,7 +18,7 @@ /** * Memory Allocation, Account and Management * - * See the README.md file in this directory for detailed information about Drill's memory allocation subsystem. + * See the README.md file in this directory for detailed information about Arrow's memory allocation subsystem. * */ package org.apache.arrow.memory; http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java ---------------------------------------------------------------------- diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java index e13dabb..aa6b70c 100644 --- a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java +++ b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java @@ -64,17 +64,17 @@ public class TestBaseAllocator { public void test_privateMax() throws Exception { try(final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) { - final ArrowBuf drillBuf1 = rootAllocator.buffer(MAX_ALLOCATION / 2); - assertNotNull("allocation failed", drillBuf1); + final ArrowBuf arrowBuf1 = rootAllocator.buffer(MAX_ALLOCATION / 2); + assertNotNull("allocation failed", arrowBuf1); try(final BufferAllocator childAllocator = rootAllocator.newChildAllocator("noLimits", 0, MAX_ALLOCATION)) { - final ArrowBuf drillBuf2 = childAllocator.buffer(MAX_ALLOCATION / 2); - assertNotNull("allocation failed", drillBuf2); - drillBuf2.release(); + final ArrowBuf arrowBuf2 = childAllocator.buffer(MAX_ALLOCATION / 2); + assertNotNull("allocation failed", arrowBuf2); + arrowBuf2.release(); } - drillBuf1.release(); + arrowBuf1.release(); } } @@ -83,8 +83,8 @@ public class TestBaseAllocator { try { try(final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) { - final ArrowBuf drillBuf = rootAllocator.buffer(512); - assertNotNull("allocation failed", drillBuf); + final ArrowBuf arrowBuf = rootAllocator.buffer(512); + assertNotNull("allocation failed", arrowBuf); } } finally { /* @@ -105,10 +105,10 @@ public class TestBaseAllocator { public void testRootAllocator_getEmpty() throws Exception { try(final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) { - final ArrowBuf drillBuf = rootAllocator.buffer(0); - assertNotNull("allocation failed", drillBuf); - assertEquals("capacity was non-zero", 0, drillBuf.capacity()); - drillBuf.release(); + final ArrowBuf arrowBuf = rootAllocator.buffer(0); + assertNotNull("allocation failed", arrowBuf); + assertEquals("capacity was non-zero", 0, arrowBuf.capacity()); + arrowBuf.release(); } } @@ -118,7 +118,7 @@ public class TestBaseAllocator { try(final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) { @SuppressWarnings("unused") - final ArrowBuf drillBuf = rootAllocator.buffer(0); + final ArrowBuf arrowBuf = rootAllocator.buffer(0); } } @@ -131,14 +131,14 @@ public class TestBaseAllocator { final BufferAllocator childAllocator2 = rootAllocator.newChildAllocator("changeOwnership2", 0, MAX_ALLOCATION); - final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 4); + final ArrowBuf arrowBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 4); rootAllocator.verify(); - TransferResult transferOwnership = drillBuf1.transferOwnership(childAllocator2); + TransferResult transferOwnership = arrowBuf1.transferOwnership(childAllocator2); final boolean allocationFit = transferOwnership.allocationFit; rootAllocator.verify(); assertTrue(allocationFit); - drillBuf1.release(); + arrowBuf1.release(); childAllocator1.close(); rootAllocator.verify(); @@ -152,34 +152,34 @@ public class TestBaseAllocator { try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) { final BufferAllocator childAllocator1 = rootAllocator.newChildAllocator("shareOwnership1", 0, MAX_ALLOCATION); final BufferAllocator childAllocator2 = rootAllocator.newChildAllocator("shareOwnership2", 0, MAX_ALLOCATION); - final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 4); + final ArrowBuf arrowBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 4); rootAllocator.verify(); // share ownership of buffer. - final ArrowBuf drillBuf2 = drillBuf1.retain(childAllocator2); + final ArrowBuf arrowBuf2 = arrowBuf1.retain(childAllocator2); rootAllocator.verify(); - assertNotNull(drillBuf2); - assertNotEquals(drillBuf2, drillBuf1); + assertNotNull(arrowBuf2); + assertNotEquals(arrowBuf2, arrowBuf1); // release original buffer (thus transferring ownership to allocator 2. (should leave allocator 1 in empty state) - drillBuf1.release(); + arrowBuf1.release(); rootAllocator.verify(); childAllocator1.close(); rootAllocator.verify(); final BufferAllocator childAllocator3 = rootAllocator.newChildAllocator("shareOwnership3", 0, MAX_ALLOCATION); - final ArrowBuf drillBuf3 = drillBuf1.retain(childAllocator3); - assertNotNull(drillBuf3); - assertNotEquals(drillBuf3, drillBuf1); - assertNotEquals(drillBuf3, drillBuf2); + final ArrowBuf arrowBuf3 = arrowBuf1.retain(childAllocator3); + assertNotNull(arrowBuf3); + assertNotEquals(arrowBuf3, arrowBuf1); + assertNotEquals(arrowBuf3, arrowBuf2); rootAllocator.verify(); - drillBuf2.release(); + arrowBuf2.release(); rootAllocator.verify(); childAllocator2.close(); rootAllocator.verify(); - drillBuf3.release(); + arrowBuf3.release(); rootAllocator.verify(); childAllocator3.close(); } @@ -190,9 +190,9 @@ public class TestBaseAllocator { try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) { try (final BufferAllocator childAllocator = rootAllocator.newChildAllocator("createChildAndUse", 0, MAX_ALLOCATION)) { - final ArrowBuf drillBuf = childAllocator.buffer(512); - assertNotNull("allocation failed", drillBuf); - drillBuf.release(); + final ArrowBuf arrowBuf = childAllocator.buffer(512); + assertNotNull("allocation failed", arrowBuf); + arrowBuf.release(); } } } @@ -203,8 +203,8 @@ public class TestBaseAllocator { try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) { final BufferAllocator childAllocator = rootAllocator.newChildAllocator("createChildDontClose", 0, MAX_ALLOCATION); - final ArrowBuf drillBuf = childAllocator.buffer(512); - assertNotNull("allocation failed", drillBuf); + final ArrowBuf arrowBuf = childAllocator.buffer(512); + assertNotNull("allocation failed", arrowBuf); } } finally { /* @@ -222,23 +222,23 @@ public class TestBaseAllocator { } private static void allocateAndFree(final BufferAllocator allocator) { - final ArrowBuf drillBuf = allocator.buffer(512); - assertNotNull("allocation failed", drillBuf); - drillBuf.release(); + final ArrowBuf arrowBuf = allocator.buffer(512); + assertNotNull("allocation failed", arrowBuf); + arrowBuf.release(); - final ArrowBuf drillBuf2 = allocator.buffer(MAX_ALLOCATION); - assertNotNull("allocation failed", drillBuf2); - drillBuf2.release(); + final ArrowBuf arrowBuf2 = allocator.buffer(MAX_ALLOCATION); + assertNotNull("allocation failed", arrowBuf2); + arrowBuf2.release(); final int nBufs = 8; - final ArrowBuf[] drillBufs = new ArrowBuf[nBufs]; - for(int i = 0; i < drillBufs.length; ++i) { - ArrowBuf drillBufi = allocator.buffer(MAX_ALLOCATION / nBufs); - assertNotNull("allocation failed", drillBufi); - drillBufs[i] = drillBufi; + final ArrowBuf[] arrowBufs = new ArrowBuf[nBufs]; + for(int i = 0; i < arrowBufs.length; ++i) { + ArrowBuf arrowBufi = allocator.buffer(MAX_ALLOCATION / nBufs); + assertNotNull("allocation failed", arrowBufi); + arrowBufs[i] = arrowBufi; } - for(ArrowBuf drillBufi : drillBufs) { - drillBufi.release(); + for(ArrowBuf arrowBufi : arrowBufs) { + arrowBufi.release(); } } @@ -277,10 +277,10 @@ public class TestBaseAllocator { new RootAllocator(MAX_ALLOCATION)) { try(final BufferAllocator childAllocator = rootAllocator.newChildAllocator("overAllocateParent", 0, MAX_ALLOCATION)) { - final ArrowBuf drillBuf1 = rootAllocator.buffer(MAX_ALLOCATION / 2); - assertNotNull("allocation failed", drillBuf1); - final ArrowBuf drillBuf2 = childAllocator.buffer(MAX_ALLOCATION / 2); - assertNotNull("allocation failed", drillBuf2); + final ArrowBuf arrowBuf1 = rootAllocator.buffer(MAX_ALLOCATION / 2); + assertNotNull("allocation failed", arrowBuf1); + final ArrowBuf arrowBuf2 = childAllocator.buffer(MAX_ALLOCATION / 2); + assertNotNull("allocation failed", arrowBuf2); try { childAllocator.buffer(MAX_ALLOCATION / 4); @@ -289,26 +289,26 @@ public class TestBaseAllocator { // expected } - drillBuf1.release(); - drillBuf2.release(); + arrowBuf1.release(); + arrowBuf2.release(); } } } private static void testAllocator_sliceUpBufferAndRelease( final RootAllocator rootAllocator, final BufferAllocator bufferAllocator) { - final ArrowBuf drillBuf1 = bufferAllocator.buffer(MAX_ALLOCATION / 2); + final ArrowBuf arrowBuf1 = bufferAllocator.buffer(MAX_ALLOCATION / 2); rootAllocator.verify(); - final ArrowBuf drillBuf2 = drillBuf1.slice(16, drillBuf1.capacity() - 32); + final ArrowBuf arrowBuf2 = arrowBuf1.slice(16, arrowBuf1.capacity() - 32); rootAllocator.verify(); - final ArrowBuf drillBuf3 = drillBuf2.slice(16, drillBuf2.capacity() - 32); + final ArrowBuf arrowBuf3 = arrowBuf2.slice(16, arrowBuf2.capacity() - 32); rootAllocator.verify(); @SuppressWarnings("unused") - final ArrowBuf drillBuf4 = drillBuf3.slice(16, drillBuf3.capacity() - 32); + final ArrowBuf arrowBuf4 = arrowBuf3.slice(16, arrowBuf3.capacity() - 32); rootAllocator.verify(); - drillBuf3.release(); // since they share refcounts, one is enough to release them all + arrowBuf3.release(); // since they share refcounts, one is enough to release them all rootAllocator.verify(); } @@ -327,11 +327,11 @@ public class TestBaseAllocator { try (final BufferAllocator childAllocator = rootAllocator.newChildAllocator("createSlices", 0, MAX_ALLOCATION)) { try (final BufferAllocator childAllocator2 = childAllocator.newChildAllocator("createSlices", 0, MAX_ALLOCATION)) { - final ArrowBuf drillBuf1 = childAllocator2.buffer(MAX_ALLOCATION / 8); + final ArrowBuf arrowBuf1 = childAllocator2.buffer(MAX_ALLOCATION / 8); @SuppressWarnings("unused") - final ArrowBuf drillBuf2 = drillBuf1.slice(MAX_ALLOCATION / 16, MAX_ALLOCATION / 16); + final ArrowBuf arrowBuf2 = arrowBuf1.slice(MAX_ALLOCATION / 16, MAX_ALLOCATION / 16); testAllocator_sliceUpBufferAndRelease(rootAllocator, childAllocator); - drillBuf1.release(); + arrowBuf1.release(); rootAllocator.verify(); } rootAllocator.verify(); @@ -348,14 +348,14 @@ public class TestBaseAllocator { try(final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) { // Populate a buffer with byte values corresponding to their indices. - final ArrowBuf drillBuf = rootAllocator.buffer(256); - assertEquals(256, drillBuf.capacity()); - assertEquals(0, drillBuf.readerIndex()); - assertEquals(0, drillBuf.readableBytes()); - assertEquals(0, drillBuf.writerIndex()); - assertEquals(256, drillBuf.writableBytes()); - - final ArrowBuf slice3 = (ArrowBuf) drillBuf.slice(); + final ArrowBuf arrowBuf = rootAllocator.buffer(256); + assertEquals(256, arrowBuf.capacity()); + assertEquals(0, arrowBuf.readerIndex()); + assertEquals(0, arrowBuf.readableBytes()); + assertEquals(0, arrowBuf.writerIndex()); + assertEquals(256, arrowBuf.writableBytes()); + + final ArrowBuf slice3 = (ArrowBuf) arrowBuf.slice(); assertEquals(0, slice3.readerIndex()); assertEquals(0, slice3.readableBytes()); assertEquals(0, slice3.writerIndex()); @@ -363,14 +363,14 @@ public class TestBaseAllocator { // assertEquals(256, slice3.writableBytes()); for(int i = 0; i < 256; ++i) { - drillBuf.writeByte(i); + arrowBuf.writeByte(i); } - assertEquals(0, drillBuf.readerIndex()); - assertEquals(256, drillBuf.readableBytes()); - assertEquals(256, drillBuf.writerIndex()); - assertEquals(0, drillBuf.writableBytes()); + assertEquals(0, arrowBuf.readerIndex()); + assertEquals(256, arrowBuf.readableBytes()); + assertEquals(256, arrowBuf.writerIndex()); + assertEquals(0, arrowBuf.writableBytes()); - final ArrowBuf slice1 = (ArrowBuf) drillBuf.slice(); + final ArrowBuf slice1 = (ArrowBuf) arrowBuf.slice(); assertEquals(0, slice1.readerIndex()); assertEquals(256, slice1.readableBytes()); for(int i = 0; i < 10; ++i) { @@ -381,7 +381,7 @@ public class TestBaseAllocator { assertEquals((byte) i, slice1.getByte(i)); } - final ArrowBuf slice2 = (ArrowBuf) drillBuf.slice(25, 25); + final ArrowBuf slice2 = (ArrowBuf) arrowBuf.slice(25, 25); assertEquals(0, slice2.readerIndex()); assertEquals(25, slice2.readableBytes()); for(int i = 25; i < 50; ++i) { @@ -397,7 +397,7 @@ public class TestBaseAllocator { } */ - drillBuf.release(); // all the derived buffers share this fate + arrowBuf.release(); // all the derived buffers share this fate } } @@ -407,33 +407,33 @@ public class TestBaseAllocator { try(final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) { // Populate a buffer with byte values corresponding to their indices. - final ArrowBuf drillBuf = rootAllocator.buffer(256); + final ArrowBuf arrowBuf = rootAllocator.buffer(256); for(int i = 0; i < 256; ++i) { - drillBuf.writeByte(i); + arrowBuf.writeByte(i); } // Slice it up. - final ArrowBuf slice0 = drillBuf.slice(0, drillBuf.capacity()); + final ArrowBuf slice0 = arrowBuf.slice(0, arrowBuf.capacity()); for(int i = 0; i < 256; ++i) { - assertEquals((byte) i, drillBuf.getByte(i)); + assertEquals((byte) i, arrowBuf.getByte(i)); } - final ArrowBuf slice10 = slice0.slice(10, drillBuf.capacity() - 10); + final ArrowBuf slice10 = slice0.slice(10, arrowBuf.capacity() - 10); for(int i = 10; i < 256; ++i) { assertEquals((byte) i, slice10.getByte(i - 10)); } - final ArrowBuf slice20 = slice10.slice(10, drillBuf.capacity() - 20); + final ArrowBuf slice20 = slice10.slice(10, arrowBuf.capacity() - 20); for(int i = 20; i < 256; ++i) { assertEquals((byte) i, slice20.getByte(i - 20)); } - final ArrowBuf slice30 = slice20.slice(10, drillBuf.capacity() - 30); + final ArrowBuf slice30 = slice20.slice(10, arrowBuf.capacity() - 30); for(int i = 30; i < 256; ++i) { assertEquals((byte) i, slice30.getByte(i - 30)); } - drillBuf.release(); + arrowBuf.release(); } } @@ -443,24 +443,24 @@ public class TestBaseAllocator { final BufferAllocator childAllocator1 = rootAllocator.newChildAllocator("transferSliced1", 0, MAX_ALLOCATION); final BufferAllocator childAllocator2 = rootAllocator.newChildAllocator("transferSliced2", 0, MAX_ALLOCATION); - final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8); - final ArrowBuf drillBuf2 = childAllocator2.buffer(MAX_ALLOCATION / 8); + final ArrowBuf arrowBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8); + final ArrowBuf arrowBuf2 = childAllocator2.buffer(MAX_ALLOCATION / 8); - final ArrowBuf drillBuf1s = drillBuf1.slice(0, drillBuf1.capacity() / 2); - final ArrowBuf drillBuf2s = drillBuf2.slice(0, drillBuf2.capacity() / 2); + final ArrowBuf arrowBuf1s = arrowBuf1.slice(0, arrowBuf1.capacity() / 2); + final ArrowBuf arrowBuf2s = arrowBuf2.slice(0, arrowBuf2.capacity() / 2); rootAllocator.verify(); - TransferResult result1 = drillBuf2s.transferOwnership(childAllocator1); + TransferResult result1 = arrowBuf2s.transferOwnership(childAllocator1); rootAllocator.verify(); - TransferResult result2 = drillBuf1s.transferOwnership(childAllocator2); + TransferResult result2 = arrowBuf1s.transferOwnership(childAllocator2); rootAllocator.verify(); result1.buffer.release(); result2.buffer.release(); - drillBuf1s.release(); // releases drillBuf1 - drillBuf2s.release(); // releases drillBuf2 + arrowBuf1s.release(); // releases arrowBuf1 + arrowBuf2s.release(); // releases arrowBuf2 childAllocator1.close(); childAllocator2.close(); @@ -473,24 +473,24 @@ public class TestBaseAllocator { final BufferAllocator childAllocator1 = rootAllocator.newChildAllocator("transferSliced", 0, MAX_ALLOCATION); final BufferAllocator childAllocator2 = rootAllocator.newChildAllocator("transferSliced", 0, MAX_ALLOCATION); - final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8); - final ArrowBuf drillBuf2 = childAllocator2.buffer(MAX_ALLOCATION / 8); + final ArrowBuf arrowBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8); + final ArrowBuf arrowBuf2 = childAllocator2.buffer(MAX_ALLOCATION / 8); - final ArrowBuf drillBuf1s = drillBuf1.slice(0, drillBuf1.capacity() / 2); - final ArrowBuf drillBuf2s = drillBuf2.slice(0, drillBuf2.capacity() / 2); + final ArrowBuf arrowBuf1s = arrowBuf1.slice(0, arrowBuf1.capacity() / 2); + final ArrowBuf arrowBuf2s = arrowBuf2.slice(0, arrowBuf2.capacity() / 2); rootAllocator.verify(); - final ArrowBuf drillBuf2s1 = drillBuf2s.retain(childAllocator1); - final ArrowBuf drillBuf1s2 = drillBuf1s.retain(childAllocator2); + final ArrowBuf arrowBuf2s1 = arrowBuf2s.retain(childAllocator1); + final ArrowBuf arrowBuf1s2 = arrowBuf1s.retain(childAllocator2); rootAllocator.verify(); - drillBuf1s.release(); // releases drillBuf1 - drillBuf2s.release(); // releases drillBuf2 + arrowBuf1s.release(); // releases arrowBuf1 + arrowBuf2s.release(); // releases arrowBuf2 rootAllocator.verify(); - drillBuf2s1.release(); // releases the shared drillBuf2 slice - drillBuf1s2.release(); // releases the shared drillBuf1 slice + arrowBuf2s1.release(); // releases the shared arrowBuf2 slice + arrowBuf1s2.release(); // releases the shared arrowBuf1 slice childAllocator1.close(); childAllocator2.close(); @@ -504,42 +504,42 @@ public class TestBaseAllocator { final BufferAllocator childAllocator2 = rootAllocator.newChildAllocator("transferShared2", 0, MAX_ALLOCATION); final BufferAllocator childAllocator3 = rootAllocator.newChildAllocator("transferShared3", 0, MAX_ALLOCATION); - final ArrowBuf drillBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8); + final ArrowBuf arrowBuf1 = childAllocator1.buffer(MAX_ALLOCATION / 8); boolean allocationFit; - ArrowBuf drillBuf2 = drillBuf1.retain(childAllocator2); + ArrowBuf arrowBuf2 = arrowBuf1.retain(childAllocator2); rootAllocator.verify(); - assertNotNull(drillBuf2); - assertNotEquals(drillBuf2, drillBuf1); + assertNotNull(arrowBuf2); + assertNotEquals(arrowBuf2, arrowBuf1); - TransferResult result = drillBuf1.transferOwnership(childAllocator3); + TransferResult result = arrowBuf1.transferOwnership(childAllocator3); allocationFit = result.allocationFit; - final ArrowBuf drillBuf3 = result.buffer; + final ArrowBuf arrowBuf3 = result.buffer; assertTrue(allocationFit); rootAllocator.verify(); // Since childAllocator3 now has childAllocator1's buffer, 1, can close - drillBuf1.release(); + arrowBuf1.release(); childAllocator1.close(); rootAllocator.verify(); - drillBuf2.release(); + arrowBuf2.release(); childAllocator2.close(); rootAllocator.verify(); final BufferAllocator childAllocator4 = rootAllocator.newChildAllocator("transferShared4", 0, MAX_ALLOCATION); - TransferResult result2 = drillBuf3.transferOwnership(childAllocator4); + TransferResult result2 = arrowBuf3.transferOwnership(childAllocator4); allocationFit = result.allocationFit; - final ArrowBuf drillBuf4 = result2.buffer; + final ArrowBuf arrowBuf4 = result2.buffer; assertTrue(allocationFit); rootAllocator.verify(); - drillBuf3.release(); + arrowBuf3.release(); childAllocator3.close(); rootAllocator.verify(); - drillBuf4.release(); + arrowBuf4.release(); childAllocator4.close(); rootAllocator.verify(); } @@ -569,11 +569,11 @@ public class TestBaseAllocator { assertTrue(reservation.add(32)); assertTrue(reservation.add(32)); - final ArrowBuf drillBuf = reservation.allocateBuffer(); - assertEquals(64, drillBuf.capacity()); + final ArrowBuf arrowBuf = reservation.allocateBuffer(); + assertEquals(64, arrowBuf.capacity()); rootAllocator.verify(); - drillBuf.release(); + arrowBuf.release(); rootAllocator.verify(); } rootAllocator.verify(); http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/vector/src/main/codegen/templates/ListWriters.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/ListWriters.java b/java/vector/src/main/codegen/templates/ListWriters.java index cf9fa30..94b812b 100644 --- a/java/vector/src/main/codegen/templates/ListWriters.java +++ b/java/vector/src/main/codegen/templates/ListWriters.java @@ -226,7 +226,7 @@ public class ${mode}ListWriter extends AbstractFieldWriter { private String getUnsupportedErrorMsg(String expected, String found) { final String f = found.substring(3); return String.format("In a list of type %s, encountered a value of type %s. "+ - "Drill does not support lists of different types.", + "Arrow does not support lists of different types.", f, expected ); } http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java index c671c9e..9fae238 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java @@ -106,7 +106,7 @@ public abstract class AbstractContainerVector implements ValueVector { if (clazz.isAssignableFrom(v.getClass())) { return (T) v; } - throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s]. Drill doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName())); + throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s]. Arrow doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName())); } MajorType getLastPathType() { http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java index d4189b2..de6ae82 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java @@ -137,7 +137,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector { } return vector; } - final String message = "Drill does not support schema change yet. Existing[%s] and desired[%s] vector types mismatch"; + final String message = "Arrow does not support schema change yet. Existing[%s] and desired[%s] vector types mismatch"; throw new IllegalStateException(String.format(message, existing.getClass().getSimpleName(), clazz.getSimpleName())); } http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java index 88cbcd4..16777c8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/ValueHolder.java @@ -18,10 +18,10 @@ package org.apache.arrow.vector.holders; /** - * Wrapper object for an individual value in Drill. + * Wrapper object for an individual value in Arrow. * * ValueHolders are designed to be mutable wrapper objects for defining clean - * APIs that access data in Drill. For performance, object creation is avoided + * APIs that access data in Arrow. For performance, object creation is avoided * at all costs throughout execution. For this reason, ValueHolders are * disallowed from implementing any methods, this allows for them to be * replaced by their java primitive inner members during optimization of http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java index 2bdfd70..b6dd13a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java @@ -29,12 +29,12 @@ public class ByteFunctionHelpers { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteFunctionHelpers.class); /** - * Helper function to check for equality of bytes in two DrillBuffers + * Helper function to check for equality of bytes in two ArrowBufs * - * @param left Left DrillBuf for comparison + * @param left Left ArrowBuf for comparison * @param lStart start offset in the buffer * @param lEnd end offset in the buffer - * @param right Right DrillBuf for comparison + * @param right Right ArrowBuf for comparison * @param rStart start offset in the buffer * @param rEnd end offset in the buffer * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise @@ -81,14 +81,14 @@ public class ByteFunctionHelpers { } /** - * Helper function to compare a set of bytes in two DrillBuffers. + * Helper function to compare a set of bytes in two ArrowBufs. * * Function will check data before completing in the case that * - * @param left Left DrillBuf to compare + * @param left Left ArrowBuf to compare * @param lStart start offset in the buffer * @param lEnd end offset in the buffer - * @param right Right DrillBuf to compare + * @param right Right ArrowBuf to compare * @param rStart start offset in the buffer * @param rEnd end offset in the buffer * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise @@ -138,9 +138,9 @@ public class ByteFunctionHelpers { } /** - * Helper function to compare a set of bytes in DrillBuf to a ByteArray. + * Helper function to compare a set of bytes in ArrowBuf to a ByteArray. * - * @param left Left DrillBuf for comparison purposes + * @param left Left ArrowBuf for comparison purposes * @param lStart start offset in the buffer * @param lEnd end offset in the buffer * @param right second input to be compared http://git-wip-us.apache.org/repos/asf/arrow/blob/e6905eff/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java index 576a5b6..a3763cd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java @@ -145,16 +145,16 @@ public class DecimalUtility extends CoreDecimalUtility{ public static BigDecimal getBigDecimalFromIntermediate(ByteBuf data, int startIndex, int nDecimalDigits, int scale) { // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false - return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, false); + return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, false); } public static BigDecimal getBigDecimalFromSparse(ArrowBuf data, int startIndex, int nDecimalDigits, int scale) { // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate - return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, true); + return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, true); } - public static BigDecimal getBigDecimalFromDrillBuf(ArrowBuf bytebuf, int start, int length, int scale) { + public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int start, int length, int scale) { byte[] value = new byte[length]; bytebuf.getBytes(start, value, 0, length); BigInteger unscaledValue = new BigInteger(value); @@ -168,17 +168,17 @@ public class DecimalUtility extends CoreDecimalUtility{ return new BigDecimal(unscaledValue, scale); } - /* Create a BigDecimal object using the data in the DrillBuf. + /* Create a BigDecimal object using the data in the ArrowBuf. * This function assumes that data is provided in a non-dense format * It works on both sparse and intermediate representations. */ - public static BigDecimal getBigDecimalFromDrillBuf(ByteBuf data, int startIndex, int nDecimalDigits, int scale, + public static BigDecimal getBigDecimalFromArrowBuf(ByteBuf data, int startIndex, int nDecimalDigits, int scale, boolean truncateScale) { // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal. int actualDigits; - // Initialize the BigDecimal, first digit in the DrillBuf has the sign so mask it out + // Initialize the BigDecimal, first digit in the ArrowBuf has the sign so mask it out BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF); BigInteger base = BigInteger.valueOf(DIGITS_BASE); @@ -208,7 +208,7 @@ public class DecimalUtility extends CoreDecimalUtility{ /* This function returns a BigDecimal object from the dense decimal representation. * First step is to convert the dense representation into an intermediate representation - * and then invoke getBigDecimalFromDrillBuf() to get the BigDecimal object + * and then invoke getBigDecimalFromArrowBuf() to get the BigDecimal object */ public static BigDecimal getBigDecimalFromDense(ArrowBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) { @@ -340,7 +340,7 @@ public class DecimalUtility extends CoreDecimalUtility{ destIndex = nDecimalDigits - 1; while (scale > 0) { - // Get next set of MAX_DIGITS (9) store it in the DrillBuf + // Get next set of MAX_DIGITS (9) store it in the ArrowBuf fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS); BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);