Return-Path: X-Original-To: apmail-geode-commits-archive@minotaur.apache.org Delivered-To: apmail-geode-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 128CF19DB1 for ; Wed, 16 Mar 2016 18:07:19 +0000 (UTC) Received: (qmail 60786 invoked by uid 500); 16 Mar 2016 18:07:19 -0000 Delivered-To: apmail-geode-commits-archive@geode.apache.org Received: (qmail 60744 invoked by uid 500); 16 Mar 2016 18:07:18 -0000 Mailing-List: contact commits-help@geode.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@geode.incubator.apache.org Delivered-To: mailing list commits@geode.incubator.apache.org Received: (qmail 60731 invoked by uid 99); 16 Mar 2016 18:07:18 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Mar 2016 18:07:18 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 58ACEC6A09 for ; Wed, 16 Mar 2016 18:07:18 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -3.221 X-Spam-Level: X-Spam-Status: No, score=-3.221 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-0.001] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id atm4PL0rqnNf for ; Wed, 16 Mar 2016 18:07:12 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id 151525F56B for ; Wed, 16 Mar 2016 18:07:09 +0000 (UTC) Received: (qmail 60632 invoked by uid 99); 16 Mar 2016 18:07:09 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Mar 2016 18:07:09 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 18F41DFC6D; Wed, 16 Mar 2016 18:07:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dschneider@apache.org To: commits@geode.incubator.apache.org Date: Wed, 16 Mar 2016 18:07:11 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [3/3] incubator-geode git commit: GEODE-1101: rename SimpleMemoryAllocator to MemoryAllocator GEODE-1101: rename SimpleMemoryAllocator to MemoryAllocator Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/82faa8af Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/82faa8af Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/82faa8af Branch: refs/heads/develop Commit: 82faa8affc7517776418d84c21df9304ebf988de Parents: 4e84f1a Author: Darrel Schneider Authored: Tue Mar 15 11:39:30 2016 -0700 Committer: Darrel Schneider Committed: Wed Mar 16 11:05:37 2016 -0700 ---------------------------------------------------------------------- .../internal/cache/AbstractRegionEntry.java | 22 +- .../gemfire/internal/cache/UpdateOperation.java | 2 +- .../tier/sockets/ClientUpdateMessageImpl.java | 2 +- .../cache/wan/GatewaySenderEventImpl.java | 10 - .../gemfire/internal/offheap/Fragment.java | 2 +- .../internal/offheap/FreeListManager.java | 8 +- .../internal/offheap/LifecycleListener.java | 20 +- .../internal/offheap/MemoryAllocatorImpl.java | 507 ++++++++++++++++ .../internal/offheap/MemoryBlockNode.java | 4 +- .../internal/offheap/OffHeapStorage.java | 2 +- .../internal/offheap/OffHeapStoredObject.java | 26 +- .../OffHeapStoredObjectAddressStack.java | 4 +- .../internal/offheap/RefCountChangeInfo.java | 2 +- .../internal/offheap/ReferenceCountHelper.java | 4 +- .../offheap/SimpleMemoryAllocatorImpl.java | 507 ---------------- .../gemfire/cache30/MultiVMRegionTestCase.java | 22 +- .../cache/ClientServerGetAllDUnitTest.java | 4 +- .../gemfire/internal/cache/OffHeapTestUtil.java | 6 +- .../cache/OffHeapValueWrapperJUnitTest.java | 8 +- .../cache/OldValueImporterTestBase.java | 26 +- .../internal/offheap/FreeListManagerTest.java | 6 +- .../offheap/LifecycleListenerJUnitTest.java | 36 +- ...moryAllocatorFillPatternIntegrationTest.java | 246 ++++++++ .../MemoryAllocatorFillPatternJUnitTest.java | 183 ++++++ .../offheap/MemoryAllocatorJUnitTest.java | 594 +++++++++++++++++++ .../offheap/MemoryBlockNodeJUnitTest.java | 6 +- .../offheap/OffHeapHelperJUnitTest.java | 4 +- .../internal/offheap/OffHeapIndexJUnitTest.java | 2 +- .../internal/offheap/OffHeapRegionBase.java | 4 +- .../OffHeapRegionEntryHelperJUnitTest.java | 4 +- .../offheap/OffHeapStorageJUnitTest.java | 6 +- ...ffHeapStoredObjectAddressStackJUnitTest.java | 40 +- .../offheap/OffHeapStoredObjectJUnitTest.java | 4 +- .../offheap/OffHeapValidationJUnitTest.java | 2 +- .../OffHeapWriteObjectAsByteArrayJUnitTest.java | 8 +- .../offheap/OutOfOffHeapMemoryDUnitTest.java | 2 +- ...moryAllocatorFillPatternIntegrationTest.java | 246 -------- ...mpleMemoryAllocatorFillPatternJUnitTest.java | 183 ------ .../offheap/SimpleMemoryAllocatorJUnitTest.java | 594 ------------------- .../offheap/TinyMemoryBlockJUnitTest.java | 8 +- .../TxReleasesOffHeapOnCloseJUnitTest.java | 2 +- .../OffHeapByteBufferByteSourceJUnitTest.java | 4 +- .../gemfire/pdx/OffHeapByteSourceJUnitTest.java | 8 +- 43 files changed, 1676 insertions(+), 1704 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java index a103e96..90d36d1 100644 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java @@ -64,7 +64,7 @@ import com.gemstone.gemfire.internal.logging.log4j.LogMarker; import com.gemstone.gemfire.internal.offheap.MemoryAllocator; import com.gemstone.gemfire.internal.offheap.OffHeapHelper; import com.gemstone.gemfire.internal.offheap.ReferenceCountHelper; -import com.gemstone.gemfire.internal.offheap.SimpleMemoryAllocatorImpl; +import com.gemstone.gemfire.internal.offheap.MemoryAllocatorImpl; import com.gemstone.gemfire.internal.offheap.StoredObject; import com.gemstone.gemfire.internal.offheap.annotations.Released; import com.gemstone.gemfire.internal.offheap.annotations.Retained; @@ -1084,24 +1084,6 @@ public abstract class AbstractRegionEntry implements RegionEntry, } return basicEquals(deserializedObj, cd.getDeserializedForReading()); } -// boolean result = Arrays.equals((byte[])cdVal, serializedObj); -// if (!result) { -// try { -// Object o1 = BlobHelper.deserializeBlob((byte[])cdVal); -// Object o2 = BlobHelper.deserializeBlob(serializedObj); -// SimpleMemoryAllocatorImpl.debugLog("checkCDEquals o1=<" + o1 + "> o2=<" + o2 + ">", false); -// if (o1.equals(o2)) { -// SimpleMemoryAllocatorImpl.debugLog("they are equal! a1=<" + Arrays.toString((byte[])cdVal) + "> a2=<" + Arrays.toString(serializedObj) + ">", false); -// } -// } catch (IOException e) { -// // TODO Auto-generated catch block -// e.printStackTrace(); -// } catch (ClassNotFoundException e) { -// // TODO Auto-generated catch block -// e.printStackTrace(); -// } -// } -// return result; } else { // prefer object form if (obj instanceof CachedDeserializable) { @@ -1337,7 +1319,7 @@ public abstract class AbstractRegionEntry implements RegionEntry, byte[] compressedData = compressBytes(r, data); boolean isCompressed = compressedData != data; ReferenceCountHelper.setReferenceCountOwner(this); - MemoryAllocator ma = SimpleMemoryAllocatorImpl.getAllocator(); // fix for bug 47875 + MemoryAllocator ma = MemoryAllocatorImpl.getAllocator(); // fix for bug 47875 val = ma.allocateAndInitialize(compressedData, isSerialized, isCompressed, data); ReferenceCountHelper.setReferenceCountOwner(null); } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/UpdateOperation.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/UpdateOperation.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/UpdateOperation.java index d26f50c..acdec28 100644 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/UpdateOperation.java +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/UpdateOperation.java @@ -43,7 +43,7 @@ import com.gemstone.gemfire.internal.cache.EntryEventImpl.SerializedCacheValueIm import com.gemstone.gemfire.internal.cache.tier.sockets.ClientProxyMembershipID; import com.gemstone.gemfire.internal.i18n.LocalizedStrings; import com.gemstone.gemfire.internal.logging.LogService; -import com.gemstone.gemfire.internal.offheap.SimpleMemoryAllocatorImpl; +import com.gemstone.gemfire.internal.offheap.MemoryAllocatorImpl; import com.gemstone.gemfire.internal.offheap.StoredObject; import com.gemstone.gemfire.internal.offheap.annotations.Unretained; import com.gemstone.gemfire.internal.util.BlobHelper; http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/tier/sockets/ClientUpdateMessageImpl.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/tier/sockets/ClientUpdateMessageImpl.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/tier/sockets/ClientUpdateMessageImpl.java index 97d9a3d..6d1508f 100755 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/tier/sockets/ClientUpdateMessageImpl.java +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/tier/sockets/ClientUpdateMessageImpl.java @@ -56,7 +56,7 @@ import com.gemstone.gemfire.internal.cache.lru.Sizeable; import com.gemstone.gemfire.internal.cache.tier.MessageType; import com.gemstone.gemfire.internal.cache.versions.VersionTag; import com.gemstone.gemfire.internal.logging.LogService; -import com.gemstone.gemfire.internal.offheap.SimpleMemoryAllocatorImpl; +import com.gemstone.gemfire.internal.offheap.MemoryAllocatorImpl; /** * Class ClientUpdateMessageImpl is a message representing a cache http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/GatewaySenderEventImpl.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/GatewaySenderEventImpl.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/GatewaySenderEventImpl.java index b64a654..d8922f8 100644 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/GatewaySenderEventImpl.java +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/GatewaySenderEventImpl.java @@ -571,16 +571,6 @@ public class GatewaySenderEventImpl implements } /** - * This method is meant for internal use by the SimpleMemoryAllocatorImpl. - * Others should use getRawValue instead. - * @return if the result is an off-heap reference then callers must use it before this event is released. - */ - @Unretained(OffHeapIdentifier.GATEWAY_SENDER_EVENT_IMPL_VALUE) - public Object getValueObject() { - return this.valueObj; - } - - /** * Return this event's deserialized value * * @return this event's deserialized value http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/Fragment.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/Fragment.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/Fragment.java index d36a71c..0ea6cf8 100644 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/Fragment.java +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/Fragment.java @@ -38,7 +38,7 @@ public class Fragment implements MemoryBlock { private static AtomicIntegerFieldUpdater freeIdxUpdater = AtomicIntegerFieldUpdater.newUpdater(Fragment.class, "freeIdx"); public Fragment(long addr, int size) { - SimpleMemoryAllocatorImpl.validateAddress(addr); + MemoryAllocatorImpl.validateAddress(addr); this.baseAddr = addr; this.size = size; freeIdxUpdater.set(this, 0); http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java index 05010ab..c943a7e 100644 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java @@ -35,7 +35,7 @@ import com.gemstone.gemfire.OutOfOffHeapMemoryException; import com.gemstone.gemfire.internal.logging.LogService; /** - * Manages the free lists for a SimpleMemoryAllocatorImpl + * Manages the free lists and slabs for a MemoryAllocator */ public class FreeListManager { static final Logger logger = LogService.getLogger(); @@ -127,9 +127,9 @@ public class FreeListManager { */ private final AtomicInteger lastFragmentAllocation = new AtomicInteger(0); private final CopyOnWriteArrayList fragmentList; - private final SimpleMemoryAllocatorImpl ma; + private final MemoryAllocatorImpl ma; - public FreeListManager(SimpleMemoryAllocatorImpl ma, final Slab[] slabs) { + public FreeListManager(MemoryAllocatorImpl ma, final Slab[] slabs) { this.ma = ma; this.slabs = slabs; long total = 0; @@ -762,7 +762,7 @@ public class FreeListManager { private List getTinyFreeBlocks() { final List value = new ArrayList(); - final SimpleMemoryAllocatorImpl sma = this.ma; + final MemoryAllocatorImpl sma = this.ma; for (int i = 0; i < this.tinyFreeLists.length(); i++) { if (this.tinyFreeLists.get(i) == null) continue; long addr = this.tinyFreeLists.get(i).getTopAddress(); http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/LifecycleListener.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/LifecycleListener.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/LifecycleListener.java index 613b12a..4a80057 100644 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/LifecycleListener.java +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/LifecycleListener.java @@ -22,23 +22,23 @@ import java.util.concurrent.CopyOnWriteArrayList; /** * Used by tests to get notifications about the lifecycle of a - * SimpleMemoryAllocatorImpl. + * MemoryAllocatorImpl. * * @author Kirk Lund */ public interface LifecycleListener { /** - * Callback is invoked after creating a new SimpleMemoryAllocatorImpl. + * Callback is invoked after creating a new MemoryAllocatorImpl. * * Create occurs during the first initialization of an * InternalDistributedSystem within the JVM. * * @param allocator the instance that has just been created */ - public void afterCreate(SimpleMemoryAllocatorImpl allocator); + public void afterCreate(MemoryAllocatorImpl allocator); /** - * Callback is invoked after reopening an existing SimpleMemoryAllocatorImpl + * Callback is invoked after reopening an existing MemoryAllocatorImpl * for reuse. * * Reuse occurs during any intialization of an @@ -47,30 +47,30 @@ public interface LifecycleListener { * * @param allocator the instance that has just been reopened for reuse */ - public void afterReuse(SimpleMemoryAllocatorImpl allocator); + public void afterReuse(MemoryAllocatorImpl allocator); /** - * Callback is invoked before closing the SimpleMemoryAllocatorImpl + * Callback is invoked before closing the MemoryAllocatorImpl * * Close occurs after the InternalDistributedSystem and DistributionManager * have completely disconnected. * * @param allocator the instance that is about to be closed */ - public void beforeClose(SimpleMemoryAllocatorImpl allocator); + public void beforeClose(MemoryAllocatorImpl allocator); - static void invokeBeforeClose(SimpleMemoryAllocatorImpl allocator) { + static void invokeBeforeClose(MemoryAllocatorImpl allocator) { for (Iterator iter = lifecycleListeners.iterator(); iter.hasNext();) { LifecycleListener listener = iter.next(); listener.beforeClose(allocator); } } - static void invokeAfterReuse(SimpleMemoryAllocatorImpl allocator) { + static void invokeAfterReuse(MemoryAllocatorImpl allocator) { for (Iterator iter = lifecycleListeners.iterator(); iter.hasNext();) { LifecycleListener listener = iter.next(); listener.afterReuse(allocator); } } - static void invokeAfterCreate(SimpleMemoryAllocatorImpl allocator) { + static void invokeAfterCreate(MemoryAllocatorImpl allocator) { for (Iterator iter = lifecycleListeners.iterator(); iter.hasNext();) { LifecycleListener listener = iter.next(); listener.afterCreate(allocator); http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/MemoryAllocatorImpl.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/MemoryAllocatorImpl.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/MemoryAllocatorImpl.java new file mode 100644 index 0000000..2050dd4 --- /dev/null +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/MemoryAllocatorImpl.java @@ -0,0 +1,507 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gemstone.gemfire.internal.offheap; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.logging.log4j.Logger; + +import com.gemstone.gemfire.cache.CacheClosedException; +import com.gemstone.gemfire.cache.Region; +import com.gemstone.gemfire.cache.RegionService; +import com.gemstone.gemfire.internal.cache.BucketRegion; +import com.gemstone.gemfire.internal.cache.GemFireCacheImpl; +import com.gemstone.gemfire.internal.cache.LocalRegion; +import com.gemstone.gemfire.internal.cache.PartitionedRegion; +import com.gemstone.gemfire.internal.cache.PartitionedRegionDataStore; +import com.gemstone.gemfire.internal.cache.RegionEntry; +import com.gemstone.gemfire.internal.logging.LogService; +import com.gemstone.gemfire.internal.offheap.annotations.OffHeapIdentifier; +import com.gemstone.gemfire.internal.offheap.annotations.Unretained; + +/** + * This allocator is somewhat like an Arena allocator. + * We start out with an array of multiple large chunks of memory. + * We also keep lists of any chunk that have been allocated and freed. + * An allocation will always try to find a chunk in a free list that is a close fit to the requested size. + * If no close fits exist then it allocates the next slice from the front of one the original large chunks. + * If we can not find enough free memory then all the existing free memory is compacted. + * If we still do not have enough to make the allocation an exception is thrown. + * + * @author darrel + * @author Kirk Lund + * @since 9.0 + */ +public class MemoryAllocatorImpl implements MemoryAllocator { + + static final Logger logger = LogService.getLogger(); + + public static final String FREE_OFF_HEAP_MEMORY_PROPERTY = "gemfire.free-off-heap-memory"; + + private volatile OffHeapMemoryStats stats; + + private volatile OutOfOffHeapMemoryListener ooohml; + + OutOfOffHeapMemoryListener getOutOfOffHeapMemoryListener() { + return this.ooohml; + } + + public final FreeListManager freeList; + + private MemoryInspector memoryInspector; + + private volatile MemoryUsageListener[] memoryUsageListeners = new MemoryUsageListener[0]; + + private static MemoryAllocatorImpl singleton = null; + + public static MemoryAllocatorImpl getAllocator() { + MemoryAllocatorImpl result = singleton; + if (result == null) { + throw new CacheClosedException("Off Heap memory allocator does not exist."); + } + return result; + } + + private static final boolean DO_EXPENSIVE_VALIDATION = Boolean.getBoolean("gemfire.OFF_HEAP_DO_EXPENSIVE_VALIDATION"); + + public static MemoryAllocator create(OutOfOffHeapMemoryListener ooohml, OffHeapMemoryStats stats, int slabCount, + long offHeapMemorySize, long maxSlabSize) { + return create(ooohml, stats, slabCount, offHeapMemorySize, maxSlabSize, null, + new SlabFactory() { + @Override + public Slab create(int size) { + return new SlabImpl(size); + } + }); + } + + private static MemoryAllocatorImpl create(OutOfOffHeapMemoryListener ooohml, OffHeapMemoryStats stats, int slabCount, + long offHeapMemorySize, long maxSlabSize, Slab[] slabs, + SlabFactory slabFactory) { + MemoryAllocatorImpl result = singleton; + boolean created = false; + try { + if (result != null) { + result.reuse(ooohml, stats, offHeapMemorySize, slabs); + logger.info("Reusing {} bytes of off-heap memory. The maximum size of a single off-heap object is {} bytes.", result.getTotalMemory(), result.freeList.getLargestSlabSize()); + created = true; + LifecycleListener.invokeAfterReuse(result); + } else { + if (slabs == null) { + // allocate memory chunks + logger.info("Allocating {} bytes of off-heap memory. The maximum size of a single off-heap object is {} bytes.", offHeapMemorySize, maxSlabSize); + slabs = new SlabImpl[slabCount]; + long uncreatedMemory = offHeapMemorySize; + for (int i=0; i < slabCount; i++) { + try { + if (uncreatedMemory >= maxSlabSize) { + slabs[i] = slabFactory.create((int) maxSlabSize); + uncreatedMemory -= maxSlabSize; + } else { + // the last slab can be smaller then maxSlabSize + slabs[i] = slabFactory.create((int) uncreatedMemory); + } + } catch (OutOfMemoryError err) { + if (i > 0) { + logger.error("Off-heap memory creation failed after successfully allocating {} bytes of off-heap memory.", (i*maxSlabSize)); + } + for (int j=0; j < i; j++) { + if (slabs[j] != null) { + slabs[j].free(); + } + } + throw err; + } + } + } + + result = new MemoryAllocatorImpl(ooohml, stats, slabs); + singleton = result; + LifecycleListener.invokeAfterCreate(result); + created = true; + } + } finally { + if (!created) { + if (stats != null) { + stats.close(); + } + if (ooohml != null) { + ooohml.close(); + } + } + } + return result; + } + static MemoryAllocatorImpl createForUnitTest(OutOfOffHeapMemoryListener ooohml, OffHeapMemoryStats stats, int slabCount, + long offHeapMemorySize, long maxSlabSize, SlabFactory memChunkFactory) { + return create(ooohml, stats, slabCount, offHeapMemorySize, maxSlabSize, null, + memChunkFactory); + } + public static MemoryAllocatorImpl createForUnitTest(OutOfOffHeapMemoryListener oooml, OffHeapMemoryStats stats, Slab[] slabs) { + int slabCount = 0; + long offHeapMemorySize = 0; + long maxSlabSize = 0; + if (slabs != null) { + slabCount = slabs.length; + for (int i=0; i < slabCount; i++) { + int slabSize = slabs[i].getSize(); + offHeapMemorySize += slabSize; + if (slabSize > maxSlabSize) { + maxSlabSize = slabSize; + } + } + } + return create(oooml, stats, slabCount, offHeapMemorySize, maxSlabSize, slabs, null); + } + + + private void reuse(OutOfOffHeapMemoryListener oooml, OffHeapMemoryStats newStats, long offHeapMemorySize, Slab[] slabs) { + if (isClosed()) { + throw new IllegalStateException("Can not reuse a closed off-heap memory manager."); + } + if (oooml == null) { + throw new IllegalArgumentException("OutOfOffHeapMemoryListener is null"); + } + if (getTotalMemory() != offHeapMemorySize) { + logger.warn("Using {} bytes of existing off-heap memory instead of the requested {}.", getTotalMemory(), offHeapMemorySize); + } + if (!this.freeList.okToReuse(slabs)) { + throw new IllegalStateException("attempted to reuse existing off-heap memory even though new off-heap memory was allocated"); + } + this.ooohml = oooml; + newStats.initialize(this.stats); + this.stats = newStats; + } + + private MemoryAllocatorImpl(final OutOfOffHeapMemoryListener oooml, final OffHeapMemoryStats stats, final Slab[] slabs) { + if (oooml == null) { + throw new IllegalArgumentException("OutOfOffHeapMemoryListener is null"); + } + + this.ooohml = oooml; + this.stats = stats; + + this.stats.setFragments(slabs.length); + this.stats.setLargestFragment(slabs[0].getSize()); + + this.freeList = new FreeListManager(this, slabs); + this.memoryInspector = new MemoryInspectorImpl(this.freeList); + + this.stats.incMaxMemory(this.freeList.getTotalMemory()); + this.stats.incFreeMemory(this.freeList.getTotalMemory()); + } + + public List getLostChunks() { + List liveChunks = this.freeList.getLiveChunks(); + List regionChunks = getRegionLiveChunks(); + Set liveChunksSet = new HashSet<>(liveChunks); + Set regionChunksSet = new HashSet<>(regionChunks); + liveChunksSet.removeAll(regionChunksSet); + return new ArrayList(liveChunksSet); + } + + /** + * Returns a possibly empty list that contains all the Chunks used by regions. + */ + private List getRegionLiveChunks() { + ArrayList result = new ArrayList(); + RegionService gfc = GemFireCacheImpl.getInstance(); + if (gfc != null) { + Iterator> rootIt = gfc.rootRegions().iterator(); + while (rootIt.hasNext()) { + Region rr = rootIt.next(); + getRegionLiveChunks(rr, result); + Iterator> srIt = rr.subregions(true).iterator(); + while (srIt.hasNext()) { + getRegionLiveChunks(srIt.next(), result); + } + } + } + return result; + } + + private void getRegionLiveChunks(Region r, List result) { + if (r.getAttributes().getOffHeap()) { + + if (r instanceof PartitionedRegion) { + PartitionedRegionDataStore prs = ((PartitionedRegion) r).getDataStore(); + if (prs != null) { + Set brs = prs.getAllLocalBucketRegions(); + if (brs != null) { + for (BucketRegion br : brs) { + if (br != null && !br.isDestroyed()) { + this.basicGetRegionLiveChunks(br, result); + } + + } + } + } + } else { + this.basicGetRegionLiveChunks((LocalRegion) r, result); + } + + } + + } + + private void basicGetRegionLiveChunks(LocalRegion r, List result) { + for (Object key : r.keySet()) { + RegionEntry re = ((LocalRegion) r).getRegionEntry(key); + if (re != null) { + /** + * value could be GATEWAY_SENDER_EVENT_IMPL_VALUE or region entry value. + */ + @Unretained(OffHeapIdentifier.GATEWAY_SENDER_EVENT_IMPL_VALUE) + Object value = re._getValue(); + if (value instanceof OffHeapStoredObject) { + result.add((OffHeapStoredObject) value); + } + } + } + } + + private OffHeapStoredObject allocateOffHeapStoredObject(int size) { + OffHeapStoredObject result = this.freeList.allocate(size); + int resultSize = result.getSize(); + stats.incObjects(1); + stats.incUsedMemory(resultSize); + stats.incFreeMemory(-resultSize); + notifyListeners(); + if (ReferenceCountHelper.trackReferenceCounts()) { + ReferenceCountHelper.refCountChanged(result.getAddress(), false, 1); + } + return result; + } + + @Override + public StoredObject allocate(int size) { + //System.out.println("allocating " + size); + OffHeapStoredObject result = allocateOffHeapStoredObject(size); + //("allocated off heap object of size " + size + " @" + Long.toHexString(result.getMemoryAddress()), true); + return result; + } + + public static void debugLog(String msg, boolean logStack) { + if (logStack) { + logger.info(msg, new RuntimeException(msg)); + } else { + logger.info(msg); + } + } + + @Override + public StoredObject allocateAndInitialize(byte[] v, boolean isSerialized, boolean isCompressed) { + return allocateAndInitialize(v, isSerialized, isCompressed, null); + } + @Override + public StoredObject allocateAndInitialize(byte[] v, boolean isSerialized, boolean isCompressed, byte[] originalHeapData) { + long addr = OffHeapRegionEntryHelper.encodeDataAsAddress(v, isSerialized, isCompressed); + if (addr != 0L) { + return new TinyStoredObject(addr); + } + OffHeapStoredObject result = allocateOffHeapStoredObject(v.length); + //debugLog("allocated off heap object of size " + v.length + " @" + Long.toHexString(result.getMemoryAddress()), true); + //debugLog("allocated off heap object of size " + v.length + " @" + Long.toHexString(result.getMemoryAddress()) + "chunkSize=" + result.getSize() + " isSerialized=" + isSerialized + " v=" + Arrays.toString(v), true); + result.setSerializedValue(v); + result.setSerialized(isSerialized); + result.setCompressed(isCompressed); + if (originalHeapData != null) { + result = new OffHeapStoredObjectWithHeapForm(result, originalHeapData); + } + return result; + } + + @Override + public long getFreeMemory() { + return this.freeList.getFreeMemory(); + } + + @Override + public long getUsedMemory() { + return this.freeList.getUsedMemory(); + } + + @Override + public long getTotalMemory() { + return this.freeList.getTotalMemory(); + } + + @Override + public void close() { + try { + LifecycleListener.invokeBeforeClose(this); + } finally { + this.ooohml.close(); + if (Boolean.getBoolean(FREE_OFF_HEAP_MEMORY_PROPERTY)) { + realClose(); + } + } + } + + public static void freeOffHeapMemory() { + MemoryAllocatorImpl ma = singleton; + if (ma != null) { + ma.realClose(); + } + } + + private void realClose() { + // Removing this memory immediately can lead to a SEGV. See 47885. + if (setClosed()) { + this.freeList.freeSlabs(); + this.stats.close(); + singleton = null; + } + } + + private final AtomicBoolean closed = new AtomicBoolean(); + private boolean isClosed() { + return this.closed.get(); + } + /** + * Returns true if caller is the one who should close; false if some other thread + * is already closing. + */ + private boolean setClosed() { + return this.closed.compareAndSet(false, true); + } + + + FreeListManager getFreeListManager() { + return this.freeList; + } + + /** + * Return the slabId of the slab that contains the given addr. + */ + int findSlab(long addr) { + return this.freeList.findSlab(addr); + } + + public OffHeapMemoryStats getStats() { + return this.stats; + } + + @Override + public void addMemoryUsageListener(final MemoryUsageListener listener) { + synchronized (this.memoryUsageListeners) { + final MemoryUsageListener[] newMemoryUsageListeners = Arrays.copyOf(this.memoryUsageListeners, this.memoryUsageListeners.length + 1); + newMemoryUsageListeners[this.memoryUsageListeners.length] = listener; + this.memoryUsageListeners = newMemoryUsageListeners; + } + } + + @Override + public void removeMemoryUsageListener(final MemoryUsageListener listener) { + synchronized (this.memoryUsageListeners) { + int listenerIndex = -1; + for (int i = 0; i < this.memoryUsageListeners.length; i++) { + if (this.memoryUsageListeners[i] == listener) { + listenerIndex = i; + break; + } + } + + if (listenerIndex != -1) { + final MemoryUsageListener[] newMemoryUsageListeners = new MemoryUsageListener[this.memoryUsageListeners.length - 1]; + System.arraycopy(this.memoryUsageListeners, 0, newMemoryUsageListeners, 0, listenerIndex); + System.arraycopy(this.memoryUsageListeners, listenerIndex + 1, newMemoryUsageListeners, listenerIndex, + this.memoryUsageListeners.length - listenerIndex - 1); + this.memoryUsageListeners = newMemoryUsageListeners; + } + } + } + + void notifyListeners() { + final MemoryUsageListener[] savedListeners = this.memoryUsageListeners; + + if (savedListeners.length == 0) { + return; + } + + final long bytesUsed = getUsedMemory(); + for (int i = 0; i < savedListeners.length; i++) { + savedListeners[i].updateMemoryUsed(bytesUsed); + } + } + + static void validateAddress(long addr) { + validateAddressAndSize(addr, -1); + } + + static void validateAddressAndSize(long addr, int size) { + // if the caller does not have a "size" to provide then use -1 + if ((addr & 7) != 0) { + StringBuilder sb = new StringBuilder(); + sb.append("address was not 8 byte aligned: 0x").append(Long.toString(addr, 16)); + MemoryAllocatorImpl ma = MemoryAllocatorImpl.singleton; + if (ma != null) { + sb.append(". Valid addresses must be in one of the following ranges: "); + ma.freeList.getSlabDescriptions(sb); + } + throw new IllegalStateException(sb.toString()); + } + if (addr >= 0 && addr < 1024) { + throw new IllegalStateException("addr was smaller than expected 0x" + addr); + } + validateAddressAndSizeWithinSlab(addr, size, DO_EXPENSIVE_VALIDATION); + } + + static void validateAddressAndSizeWithinSlab(long addr, int size, boolean doExpensiveValidation) { + if (doExpensiveValidation) { + MemoryAllocatorImpl ma = MemoryAllocatorImpl.singleton; + if (ma != null) { + if (!ma.freeList.validateAddressAndSizeWithinSlab(addr, size)) { + throw new IllegalStateException(" address 0x" + Long.toString(addr, 16) + " does not address the original slab memory"); + } + } + } + } + + public synchronized List getOrphans() { + List liveChunks = this.freeList.getLiveChunks(); + List regionChunks = getRegionLiveChunks(); + liveChunks.removeAll(regionChunks); + List orphans = new ArrayList(); + for (OffHeapStoredObject chunk: liveChunks) { + orphans.add(new MemoryBlockNode(this, chunk)); + } + Collections.sort(orphans, + new Comparator() { + @Override + public int compare(MemoryBlock o1, MemoryBlock o2) { + return Long.valueOf(o1.getAddress()).compareTo(o2.getAddress()); + } + }); + //this.memoryBlocks = new WeakReference>(orphans); + return orphans; + } + + @Override + public MemoryInspector getMemoryInspector() { + return this.memoryInspector; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/MemoryBlockNode.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/MemoryBlockNode.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/MemoryBlockNode.java index 6e2414f..a72d618 100644 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/MemoryBlockNode.java +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/MemoryBlockNode.java @@ -26,9 +26,9 @@ import com.gemstone.gemfire.cache.CacheClosedException; * Basic implementation of MemoryBlock for test validation only. */ public class MemoryBlockNode implements MemoryBlock { - private final SimpleMemoryAllocatorImpl ma; + private final MemoryAllocatorImpl ma; private final MemoryBlock block; - MemoryBlockNode(SimpleMemoryAllocatorImpl ma, MemoryBlock block) { + MemoryBlockNode(MemoryAllocatorImpl ma, MemoryBlock block) { this.ma = ma; this.block = block; } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStorage.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStorage.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStorage.java index 9c6c75a..2bdcfba 100755 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStorage.java +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStorage.java @@ -199,7 +199,7 @@ public class OffHeapStorage implements OffHeapMemoryStats { final int slabCount = calcSlabCount(maxSlabSize, offHeapMemorySize); - return SimpleMemoryAllocatorImpl.create(ooohml, stats, slabCount, offHeapMemorySize, maxSlabSize); + return MemoryAllocatorImpl.create(ooohml, stats, slabCount, offHeapMemorySize, maxSlabSize); } private static final long MAX_SLAB_SIZE = Integer.MAX_VALUE; http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStoredObject.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStoredObject.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStoredObject.java index 68c9bdd..9861a54 100644 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStoredObject.java +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStoredObject.java @@ -96,7 +96,7 @@ import com.gemstone.gemfire.internal.offheap.annotations.Unretained; final static byte FILL_BYTE = 0x3c; protected OffHeapStoredObject(long memoryAddress, int chunkSize) { - SimpleMemoryAllocatorImpl.validateAddressAndSize(memoryAddress, chunkSize); + MemoryAllocatorImpl.validateAddressAndSize(memoryAddress, chunkSize); this.memoryAddress = memoryAddress; setSize(chunkSize); AddressableMemoryManager.writeIntVolatile(getAddress()+REF_COUNT_OFFSET, MAGIC_NUMBER); @@ -121,7 +121,7 @@ import com.gemstone.gemfire.internal.offheap.annotations.Unretained; * memoryAddress. The off heap header has already been initialized. */ protected OffHeapStoredObject(long memoryAddress) { - SimpleMemoryAllocatorImpl.validateAddress(memoryAddress); + MemoryAllocatorImpl.validateAddress(memoryAddress); this.memoryAddress = memoryAddress; } @@ -168,8 +168,8 @@ import com.gemstone.gemfire.internal.offheap.annotations.Unretained; // TODO OFFHEAP: no need to copy to heap. Just get the address of each and compare each byte. No need to call incReads when reading from address. int i; // inc it twice since we are reading two different objects - SimpleMemoryAllocatorImpl.getAllocator().getStats().incReads(); - SimpleMemoryAllocatorImpl.getAllocator().getStats().incReads(); + MemoryAllocatorImpl.getAllocator().getStats().incReads(); + MemoryAllocatorImpl.getAllocator().getStats().incReads(); for (i=0; i < mySize-(dataCache1.length-1); i+=dataCache1.length) { this.readDataBytes(i, dataCache1); other.readDataBytes(i, dataCache2); @@ -206,7 +206,7 @@ import com.gemstone.gemfire.internal.offheap.annotations.Unretained; final byte[] dataCache = new byte[1024]; int idx=0; int i; - SimpleMemoryAllocatorImpl.getAllocator().getStats().incReads(); + MemoryAllocatorImpl.getAllocator().getStats().incReads(); for (i=0; i < mySize-(dataCache.length-1); i+=dataCache.length) { this.readDataBytes(i, dataCache); for (int j=0; j < dataCache.length; j++) { @@ -416,7 +416,7 @@ import com.gemstone.gemfire.internal.offheap.annotations.Unretained; byte[] result = new byte[getDataSize()]; readDataBytes(0, result); //debugLog("reading", true); - SimpleMemoryAllocatorImpl.getAllocator().getStats().incReads(); + MemoryAllocatorImpl.getAllocator().getStats().incReads(); return result; } protected byte[] getRawBytes() { @@ -487,19 +487,19 @@ import com.gemstone.gemfire.internal.offheap.annotations.Unretained; } public static int getSize(long memAddr) { - SimpleMemoryAllocatorImpl.validateAddress(memAddr); + MemoryAllocatorImpl.validateAddress(memAddr); return AddressableMemoryManager.readInt(memAddr+CHUNK_SIZE_OFFSET); } public static void setSize(long memAddr, int size) { - SimpleMemoryAllocatorImpl.validateAddressAndSize(memAddr, size); + MemoryAllocatorImpl.validateAddressAndSize(memAddr, size); AddressableMemoryManager.writeInt(memAddr+CHUNK_SIZE_OFFSET, size); } public static long getNext(long memAddr) { - SimpleMemoryAllocatorImpl.validateAddress(memAddr); + MemoryAllocatorImpl.validateAddress(memAddr); return AddressableMemoryManager.readLong(memAddr+HEADER_SIZE); } public static void setNext(long memAddr, long next) { - SimpleMemoryAllocatorImpl.validateAddress(memAddr); + MemoryAllocatorImpl.validateAddress(memAddr); AddressableMemoryManager.writeLong(memAddr+HEADER_SIZE, next); } @@ -595,7 +595,7 @@ import com.gemstone.gemfire.internal.offheap.annotations.Unretained; } public static boolean retain(long memAddr) { - SimpleMemoryAllocatorImpl.validateAddress(memAddr); + MemoryAllocatorImpl.validateAddress(memAddr); int uc; int rawBits; int retryCount = 0; @@ -628,7 +628,7 @@ import com.gemstone.gemfire.internal.offheap.annotations.Unretained; release(memAddr, null); } static void release(final long memAddr, FreeListManager freeListManager) { - SimpleMemoryAllocatorImpl.validateAddress(memAddr); + MemoryAllocatorImpl.validateAddress(memAddr); int newCount; int rawBits; boolean returnToAllocator; @@ -661,7 +661,7 @@ import com.gemstone.gemfire.internal.offheap.annotations.Unretained; ReferenceCountHelper.freeRefCountInfo(memAddr); } if (freeListManager == null) { - freeListManager = SimpleMemoryAllocatorImpl.getAllocator().getFreeListManager(); + freeListManager = MemoryAllocatorImpl.getAllocator().getFreeListManager(); } freeListManager.free(memAddr); } else { http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStoredObjectAddressStack.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStoredObjectAddressStack.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStoredObjectAddressStack.java index 40d0143..b69d3a6 100644 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStoredObjectAddressStack.java +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStoredObjectAddressStack.java @@ -29,7 +29,7 @@ public class OffHeapStoredObjectAddressStack { private volatile long topAddr; public OffHeapStoredObjectAddressStack(long addr) { - if (addr != 0L) SimpleMemoryAllocatorImpl.validateAddress(addr); + if (addr != 0L) MemoryAllocatorImpl.validateAddress(addr); this.topAddr = addr; } public OffHeapStoredObjectAddressStack() { @@ -40,7 +40,7 @@ public class OffHeapStoredObjectAddressStack { } public void offer(long e) { assert e != 0; - SimpleMemoryAllocatorImpl.validateAddress(e); + MemoryAllocatorImpl.validateAddress(e); synchronized (this) { OffHeapStoredObject.setNext(e, this.topAddr); this.topAddr = e; http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/RefCountChangeInfo.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/RefCountChangeInfo.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/RefCountChangeInfo.java index e3b4b1f..cc67a58 100644 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/RefCountChangeInfo.java +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/RefCountChangeInfo.java @@ -23,7 +23,7 @@ import com.gemstone.gemfire.internal.shared.StringPrintWriter; @SuppressWarnings("serial") /** - * Used by SimpleMemoryAllocatorImpl to debug off-heap memory leaks. + * Used by MemoryAllocatorImpl to debug off-heap memory leaks. */ public class RefCountChangeInfo extends Throwable { private final String threadName; http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/ReferenceCountHelper.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/ReferenceCountHelper.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/ReferenceCountHelper.java index 778b329..f6696b0 100644 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/ReferenceCountHelper.java +++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/ReferenceCountHelper.java @@ -214,7 +214,7 @@ public class ReferenceCountHelper { } } if (list == LOCKED) { - SimpleMemoryAllocatorImpl.debugLog("refCount " + (decRefCount ? "deced" : "inced") + " after orphan detected for @" + Long.toHexString(address), true); + MemoryAllocatorImpl.debugLog("refCount " + (decRefCount ? "deced" : "inced") + " after orphan detected for @" + Long.toHexString(address), true); return; } RefCountChangeInfo info = new RefCountChangeInfo(decRefCount, rc, owner); @@ -242,7 +242,7 @@ public class ReferenceCountHelper { if (!trackReferenceCounts()) return; List freedInfo = stacktraces.remove(address); if (freedInfo == LOCKED) { - SimpleMemoryAllocatorImpl.debugLog("freed after orphan detected for @" + Long.toHexString(address), true); + MemoryAllocatorImpl.debugLog("freed after orphan detected for @" + Long.toHexString(address), true); } else if (trackFreedReferenceCounts()) { if (freedInfo != null) { freedStacktraces.put(address, freedInfo); http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java deleted file mode 100644 index f7fa888..0000000 --- a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java +++ /dev/null @@ -1,507 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.gemstone.gemfire.internal.offheap; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.logging.log4j.Logger; - -import com.gemstone.gemfire.cache.CacheClosedException; -import com.gemstone.gemfire.cache.Region; -import com.gemstone.gemfire.cache.RegionService; -import com.gemstone.gemfire.internal.cache.BucketRegion; -import com.gemstone.gemfire.internal.cache.GemFireCacheImpl; -import com.gemstone.gemfire.internal.cache.LocalRegion; -import com.gemstone.gemfire.internal.cache.PartitionedRegion; -import com.gemstone.gemfire.internal.cache.PartitionedRegionDataStore; -import com.gemstone.gemfire.internal.cache.RegionEntry; -import com.gemstone.gemfire.internal.logging.LogService; -import com.gemstone.gemfire.internal.offheap.annotations.OffHeapIdentifier; -import com.gemstone.gemfire.internal.offheap.annotations.Unretained; - -/** - * This allocator is somewhat like an Arena allocator. - * We start out with an array of multiple large chunks of memory. - * We also keep lists of any chunk that have been allocated and freed. - * An allocation will always try to find a chunk in a free list that is a close fit to the requested size. - * If no close fits exist then it allocates the next slice from the front of one the original large chunks. - * If we can not find enough free memory then all the existing free memory is compacted. - * If we still do not have enough to make the allocation an exception is thrown. - * - * @author darrel - * @author Kirk Lund - * @since 9.0 - */ -public class SimpleMemoryAllocatorImpl implements MemoryAllocator { - - static final Logger logger = LogService.getLogger(); - - public static final String FREE_OFF_HEAP_MEMORY_PROPERTY = "gemfire.free-off-heap-memory"; - - private volatile OffHeapMemoryStats stats; - - private volatile OutOfOffHeapMemoryListener ooohml; - - OutOfOffHeapMemoryListener getOutOfOffHeapMemoryListener() { - return this.ooohml; - } - - public final FreeListManager freeList; - - private MemoryInspector memoryInspector; - - private volatile MemoryUsageListener[] memoryUsageListeners = new MemoryUsageListener[0]; - - private static SimpleMemoryAllocatorImpl singleton = null; - - public static SimpleMemoryAllocatorImpl getAllocator() { - SimpleMemoryAllocatorImpl result = singleton; - if (result == null) { - throw new CacheClosedException("Off Heap memory allocator does not exist."); - } - return result; - } - - private static final boolean DO_EXPENSIVE_VALIDATION = Boolean.getBoolean("gemfire.OFF_HEAP_DO_EXPENSIVE_VALIDATION"); - - public static MemoryAllocator create(OutOfOffHeapMemoryListener ooohml, OffHeapMemoryStats stats, int slabCount, - long offHeapMemorySize, long maxSlabSize) { - return create(ooohml, stats, slabCount, offHeapMemorySize, maxSlabSize, null, - new SlabFactory() { - @Override - public Slab create(int size) { - return new SlabImpl(size); - } - }); - } - - private static SimpleMemoryAllocatorImpl create(OutOfOffHeapMemoryListener ooohml, OffHeapMemoryStats stats, int slabCount, - long offHeapMemorySize, long maxSlabSize, Slab[] slabs, - SlabFactory slabFactory) { - SimpleMemoryAllocatorImpl result = singleton; - boolean created = false; - try { - if (result != null) { - result.reuse(ooohml, stats, offHeapMemorySize, slabs); - logger.info("Reusing {} bytes of off-heap memory. The maximum size of a single off-heap object is {} bytes.", result.getTotalMemory(), result.freeList.getLargestSlabSize()); - created = true; - LifecycleListener.invokeAfterReuse(result); - } else { - if (slabs == null) { - // allocate memory chunks - logger.info("Allocating {} bytes of off-heap memory. The maximum size of a single off-heap object is {} bytes.", offHeapMemorySize, maxSlabSize); - slabs = new SlabImpl[slabCount]; - long uncreatedMemory = offHeapMemorySize; - for (int i=0; i < slabCount; i++) { - try { - if (uncreatedMemory >= maxSlabSize) { - slabs[i] = slabFactory.create((int) maxSlabSize); - uncreatedMemory -= maxSlabSize; - } else { - // the last slab can be smaller then maxSlabSize - slabs[i] = slabFactory.create((int) uncreatedMemory); - } - } catch (OutOfMemoryError err) { - if (i > 0) { - logger.error("Off-heap memory creation failed after successfully allocating {} bytes of off-heap memory.", (i*maxSlabSize)); - } - for (int j=0; j < i; j++) { - if (slabs[j] != null) { - slabs[j].free(); - } - } - throw err; - } - } - } - - result = new SimpleMemoryAllocatorImpl(ooohml, stats, slabs); - singleton = result; - LifecycleListener.invokeAfterCreate(result); - created = true; - } - } finally { - if (!created) { - if (stats != null) { - stats.close(); - } - if (ooohml != null) { - ooohml.close(); - } - } - } - return result; - } - static SimpleMemoryAllocatorImpl createForUnitTest(OutOfOffHeapMemoryListener ooohml, OffHeapMemoryStats stats, int slabCount, - long offHeapMemorySize, long maxSlabSize, SlabFactory memChunkFactory) { - return create(ooohml, stats, slabCount, offHeapMemorySize, maxSlabSize, null, - memChunkFactory); - } - public static SimpleMemoryAllocatorImpl createForUnitTest(OutOfOffHeapMemoryListener oooml, OffHeapMemoryStats stats, Slab[] slabs) { - int slabCount = 0; - long offHeapMemorySize = 0; - long maxSlabSize = 0; - if (slabs != null) { - slabCount = slabs.length; - for (int i=0; i < slabCount; i++) { - int slabSize = slabs[i].getSize(); - offHeapMemorySize += slabSize; - if (slabSize > maxSlabSize) { - maxSlabSize = slabSize; - } - } - } - return create(oooml, stats, slabCount, offHeapMemorySize, maxSlabSize, slabs, null); - } - - - private void reuse(OutOfOffHeapMemoryListener oooml, OffHeapMemoryStats newStats, long offHeapMemorySize, Slab[] slabs) { - if (isClosed()) { - throw new IllegalStateException("Can not reuse a closed off-heap memory manager."); - } - if (oooml == null) { - throw new IllegalArgumentException("OutOfOffHeapMemoryListener is null"); - } - if (getTotalMemory() != offHeapMemorySize) { - logger.warn("Using {} bytes of existing off-heap memory instead of the requested {}.", getTotalMemory(), offHeapMemorySize); - } - if (!this.freeList.okToReuse(slabs)) { - throw new IllegalStateException("attempted to reuse existing off-heap memory even though new off-heap memory was allocated"); - } - this.ooohml = oooml; - newStats.initialize(this.stats); - this.stats = newStats; - } - - private SimpleMemoryAllocatorImpl(final OutOfOffHeapMemoryListener oooml, final OffHeapMemoryStats stats, final Slab[] slabs) { - if (oooml == null) { - throw new IllegalArgumentException("OutOfOffHeapMemoryListener is null"); - } - - this.ooohml = oooml; - this.stats = stats; - - this.stats.setFragments(slabs.length); - this.stats.setLargestFragment(slabs[0].getSize()); - - this.freeList = new FreeListManager(this, slabs); - this.memoryInspector = new MemoryInspectorImpl(this.freeList); - - this.stats.incMaxMemory(this.freeList.getTotalMemory()); - this.stats.incFreeMemory(this.freeList.getTotalMemory()); - } - - public List getLostChunks() { - List liveChunks = this.freeList.getLiveChunks(); - List regionChunks = getRegionLiveChunks(); - Set liveChunksSet = new HashSet<>(liveChunks); - Set regionChunksSet = new HashSet<>(regionChunks); - liveChunksSet.removeAll(regionChunksSet); - return new ArrayList(liveChunksSet); - } - - /** - * Returns a possibly empty list that contains all the Chunks used by regions. - */ - private List getRegionLiveChunks() { - ArrayList result = new ArrayList(); - RegionService gfc = GemFireCacheImpl.getInstance(); - if (gfc != null) { - Iterator> rootIt = gfc.rootRegions().iterator(); - while (rootIt.hasNext()) { - Region rr = rootIt.next(); - getRegionLiveChunks(rr, result); - Iterator> srIt = rr.subregions(true).iterator(); - while (srIt.hasNext()) { - getRegionLiveChunks(srIt.next(), result); - } - } - } - return result; - } - - private void getRegionLiveChunks(Region r, List result) { - if (r.getAttributes().getOffHeap()) { - - if (r instanceof PartitionedRegion) { - PartitionedRegionDataStore prs = ((PartitionedRegion) r).getDataStore(); - if (prs != null) { - Set brs = prs.getAllLocalBucketRegions(); - if (brs != null) { - for (BucketRegion br : brs) { - if (br != null && !br.isDestroyed()) { - this.basicGetRegionLiveChunks(br, result); - } - - } - } - } - } else { - this.basicGetRegionLiveChunks((LocalRegion) r, result); - } - - } - - } - - private void basicGetRegionLiveChunks(LocalRegion r, List result) { - for (Object key : r.keySet()) { - RegionEntry re = ((LocalRegion) r).getRegionEntry(key); - if (re != null) { - /** - * value could be GATEWAY_SENDER_EVENT_IMPL_VALUE or region entry value. - */ - @Unretained(OffHeapIdentifier.GATEWAY_SENDER_EVENT_IMPL_VALUE) - Object value = re._getValue(); - if (value instanceof OffHeapStoredObject) { - result.add((OffHeapStoredObject) value); - } - } - } - } - - private OffHeapStoredObject allocateOffHeapStoredObject(int size) { - OffHeapStoredObject result = this.freeList.allocate(size); - int resultSize = result.getSize(); - stats.incObjects(1); - stats.incUsedMemory(resultSize); - stats.incFreeMemory(-resultSize); - notifyListeners(); - if (ReferenceCountHelper.trackReferenceCounts()) { - ReferenceCountHelper.refCountChanged(result.getAddress(), false, 1); - } - return result; - } - - @Override - public StoredObject allocate(int size) { - //System.out.println("allocating " + size); - OffHeapStoredObject result = allocateOffHeapStoredObject(size); - //("allocated off heap object of size " + size + " @" + Long.toHexString(result.getMemoryAddress()), true); - return result; - } - - public static void debugLog(String msg, boolean logStack) { - if (logStack) { - logger.info(msg, new RuntimeException(msg)); - } else { - logger.info(msg); - } - } - - @Override - public StoredObject allocateAndInitialize(byte[] v, boolean isSerialized, boolean isCompressed) { - return allocateAndInitialize(v, isSerialized, isCompressed, null); - } - @Override - public StoredObject allocateAndInitialize(byte[] v, boolean isSerialized, boolean isCompressed, byte[] originalHeapData) { - long addr = OffHeapRegionEntryHelper.encodeDataAsAddress(v, isSerialized, isCompressed); - if (addr != 0L) { - return new TinyStoredObject(addr); - } - OffHeapStoredObject result = allocateOffHeapStoredObject(v.length); - //debugLog("allocated off heap object of size " + v.length + " @" + Long.toHexString(result.getMemoryAddress()), true); - //debugLog("allocated off heap object of size " + v.length + " @" + Long.toHexString(result.getMemoryAddress()) + "chunkSize=" + result.getSize() + " isSerialized=" + isSerialized + " v=" + Arrays.toString(v), true); - result.setSerializedValue(v); - result.setSerialized(isSerialized); - result.setCompressed(isCompressed); - if (originalHeapData != null) { - result = new OffHeapStoredObjectWithHeapForm(result, originalHeapData); - } - return result; - } - - @Override - public long getFreeMemory() { - return this.freeList.getFreeMemory(); - } - - @Override - public long getUsedMemory() { - return this.freeList.getUsedMemory(); - } - - @Override - public long getTotalMemory() { - return this.freeList.getTotalMemory(); - } - - @Override - public void close() { - try { - LifecycleListener.invokeBeforeClose(this); - } finally { - this.ooohml.close(); - if (Boolean.getBoolean(FREE_OFF_HEAP_MEMORY_PROPERTY)) { - realClose(); - } - } - } - - public static void freeOffHeapMemory() { - SimpleMemoryAllocatorImpl ma = singleton; - if (ma != null) { - ma.realClose(); - } - } - - private void realClose() { - // Removing this memory immediately can lead to a SEGV. See 47885. - if (setClosed()) { - this.freeList.freeSlabs(); - this.stats.close(); - singleton = null; - } - } - - private final AtomicBoolean closed = new AtomicBoolean(); - private boolean isClosed() { - return this.closed.get(); - } - /** - * Returns true if caller is the one who should close; false if some other thread - * is already closing. - */ - private boolean setClosed() { - return this.closed.compareAndSet(false, true); - } - - - FreeListManager getFreeListManager() { - return this.freeList; - } - - /** - * Return the slabId of the slab that contains the given addr. - */ - int findSlab(long addr) { - return this.freeList.findSlab(addr); - } - - public OffHeapMemoryStats getStats() { - return this.stats; - } - - @Override - public void addMemoryUsageListener(final MemoryUsageListener listener) { - synchronized (this.memoryUsageListeners) { - final MemoryUsageListener[] newMemoryUsageListeners = Arrays.copyOf(this.memoryUsageListeners, this.memoryUsageListeners.length + 1); - newMemoryUsageListeners[this.memoryUsageListeners.length] = listener; - this.memoryUsageListeners = newMemoryUsageListeners; - } - } - - @Override - public void removeMemoryUsageListener(final MemoryUsageListener listener) { - synchronized (this.memoryUsageListeners) { - int listenerIndex = -1; - for (int i = 0; i < this.memoryUsageListeners.length; i++) { - if (this.memoryUsageListeners[i] == listener) { - listenerIndex = i; - break; - } - } - - if (listenerIndex != -1) { - final MemoryUsageListener[] newMemoryUsageListeners = new MemoryUsageListener[this.memoryUsageListeners.length - 1]; - System.arraycopy(this.memoryUsageListeners, 0, newMemoryUsageListeners, 0, listenerIndex); - System.arraycopy(this.memoryUsageListeners, listenerIndex + 1, newMemoryUsageListeners, listenerIndex, - this.memoryUsageListeners.length - listenerIndex - 1); - this.memoryUsageListeners = newMemoryUsageListeners; - } - } - } - - void notifyListeners() { - final MemoryUsageListener[] savedListeners = this.memoryUsageListeners; - - if (savedListeners.length == 0) { - return; - } - - final long bytesUsed = getUsedMemory(); - for (int i = 0; i < savedListeners.length; i++) { - savedListeners[i].updateMemoryUsed(bytesUsed); - } - } - - static void validateAddress(long addr) { - validateAddressAndSize(addr, -1); - } - - static void validateAddressAndSize(long addr, int size) { - // if the caller does not have a "size" to provide then use -1 - if ((addr & 7) != 0) { - StringBuilder sb = new StringBuilder(); - sb.append("address was not 8 byte aligned: 0x").append(Long.toString(addr, 16)); - SimpleMemoryAllocatorImpl ma = SimpleMemoryAllocatorImpl.singleton; - if (ma != null) { - sb.append(". Valid addresses must be in one of the following ranges: "); - ma.freeList.getSlabDescriptions(sb); - } - throw new IllegalStateException(sb.toString()); - } - if (addr >= 0 && addr < 1024) { - throw new IllegalStateException("addr was smaller than expected 0x" + addr); - } - validateAddressAndSizeWithinSlab(addr, size, DO_EXPENSIVE_VALIDATION); - } - - static void validateAddressAndSizeWithinSlab(long addr, int size, boolean doExpensiveValidation) { - if (doExpensiveValidation) { - SimpleMemoryAllocatorImpl ma = SimpleMemoryAllocatorImpl.singleton; - if (ma != null) { - if (!ma.freeList.validateAddressAndSizeWithinSlab(addr, size)) { - throw new IllegalStateException(" address 0x" + Long.toString(addr, 16) + " does not address the original slab memory"); - } - } - } - } - - public synchronized List getOrphans() { - List liveChunks = this.freeList.getLiveChunks(); - List regionChunks = getRegionLiveChunks(); - liveChunks.removeAll(regionChunks); - List orphans = new ArrayList(); - for (OffHeapStoredObject chunk: liveChunks) { - orphans.add(new MemoryBlockNode(this, chunk)); - } - Collections.sort(orphans, - new Comparator() { - @Override - public int compare(MemoryBlock o1, MemoryBlock o2) { - return Long.valueOf(o1.getAddress()).compareTo(o2.getAddress()); - } - }); - //this.memoryBlocks = new WeakReference>(orphans); - return orphans; - } - - @Override - public MemoryInspector getMemoryInspector() { - return this.memoryInspector; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/test/java/com/gemstone/gemfire/cache30/MultiVMRegionTestCase.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/com/gemstone/gemfire/cache30/MultiVMRegionTestCase.java b/geode-core/src/test/java/com/gemstone/gemfire/cache30/MultiVMRegionTestCase.java index dce68cf..8127eea 100644 --- a/geode-core/src/test/java/com/gemstone/gemfire/cache30/MultiVMRegionTestCase.java +++ b/geode-core/src/test/java/com/gemstone/gemfire/cache30/MultiVMRegionTestCase.java @@ -110,7 +110,7 @@ import com.gemstone.gemfire.internal.cache.versions.RegionVersionVector; import com.gemstone.gemfire.internal.cache.versions.VMRegionVersionVector; import com.gemstone.gemfire.internal.cache.versions.VersionTag; import com.gemstone.gemfire.internal.logging.LogService; -import com.gemstone.gemfire.internal.offheap.SimpleMemoryAllocatorImpl; +import com.gemstone.gemfire.internal.offheap.MemoryAllocatorImpl; import com.gemstone.gemfire.internal.offheap.StoredObject; import com.gemstone.gemfire.test.dunit.AsyncInvocation; import com.gemstone.gemfire.test.dunit.DistributedTestCase; @@ -2000,7 +2000,7 @@ public abstract class MultiVMRegionTestCase extends RegionTestCase { assertEquals(1, region.size()); if (region.getAttributes().getOffHeap() && !(region instanceof PartitionedRegion)) { GemFireCacheImpl gfc = (GemFireCacheImpl) getCache(); - SimpleMemoryAllocatorImpl ma = (SimpleMemoryAllocatorImpl) gfc.getOffHeapStore(); + MemoryAllocatorImpl ma = (MemoryAllocatorImpl) gfc.getOffHeapStore(); LocalRegion reRegion; reRegion = (LocalRegion) region; RegionEntry re = reRegion.getRegionEntry(key2); @@ -2066,7 +2066,7 @@ public abstract class MultiVMRegionTestCase extends RegionTestCase { assertEquals(1, region.size()); if (region.getAttributes().getOffHeap() && !(region instanceof PartitionedRegion)) { GemFireCacheImpl gfc = (GemFireCacheImpl) getCache(); - SimpleMemoryAllocatorImpl ma = (SimpleMemoryAllocatorImpl) gfc.getOffHeapStore(); + MemoryAllocatorImpl ma = (MemoryAllocatorImpl) gfc.getOffHeapStore(); assertEquals(1, ma.getStats().getObjects()); } } @@ -2087,7 +2087,7 @@ public abstract class MultiVMRegionTestCase extends RegionTestCase { assertEquals(2, region.size()); if (region.getAttributes().getOffHeap() && !(region instanceof PartitionedRegion)) { GemFireCacheImpl gfc = (GemFireCacheImpl) getCache(); - SimpleMemoryAllocatorImpl ma = (SimpleMemoryAllocatorImpl) gfc.getOffHeapStore(); + MemoryAllocatorImpl ma = (MemoryAllocatorImpl) gfc.getOffHeapStore(); assertEquals(2, ma.getStats().getObjects()); LocalRegion reRegion; reRegion = (LocalRegion) region; @@ -2153,7 +2153,7 @@ public abstract class MultiVMRegionTestCase extends RegionTestCase { assertEquals(2, region.size()); if (region.getAttributes().getOffHeap() && !(region instanceof PartitionedRegion)) { GemFireCacheImpl gfc = (GemFireCacheImpl) getCache(); - SimpleMemoryAllocatorImpl ma = (SimpleMemoryAllocatorImpl) gfc.getOffHeapStore(); + MemoryAllocatorImpl ma = (MemoryAllocatorImpl) gfc.getOffHeapStore(); assertEquals(2, ma.getStats().getObjects()); LocalRegion reRegion; reRegion = (LocalRegion) region; @@ -2177,7 +2177,7 @@ public abstract class MultiVMRegionTestCase extends RegionTestCase { assertEquals(2, region.size()); if (region.getAttributes().getOffHeap() && !(region instanceof PartitionedRegion)) { GemFireCacheImpl gfc = (GemFireCacheImpl) getCache(); - SimpleMemoryAllocatorImpl ma = (SimpleMemoryAllocatorImpl) gfc.getOffHeapStore(); + MemoryAllocatorImpl ma = (MemoryAllocatorImpl) gfc.getOffHeapStore(); assertEquals(2, ma.getStats().getObjects()); } } @@ -2237,7 +2237,7 @@ public abstract class MultiVMRegionTestCase extends RegionTestCase { assertEquals(2, region.size()); if (region.getAttributes().getOffHeap() && !(region instanceof PartitionedRegion)) { GemFireCacheImpl gfc = (GemFireCacheImpl) getCache(); - SimpleMemoryAllocatorImpl ma = (SimpleMemoryAllocatorImpl) gfc.getOffHeapStore(); + MemoryAllocatorImpl ma = (MemoryAllocatorImpl) gfc.getOffHeapStore(); assertEquals(2, ma.getStats().getObjects()); } } @@ -2257,7 +2257,7 @@ public abstract class MultiVMRegionTestCase extends RegionTestCase { assertEquals(1, region.size()); if (region.getAttributes().getOffHeap() && !(region instanceof PartitionedRegion)) { GemFireCacheImpl gfc = (GemFireCacheImpl) getCache(); - SimpleMemoryAllocatorImpl ma = (SimpleMemoryAllocatorImpl) gfc.getOffHeapStore(); + MemoryAllocatorImpl ma = (MemoryAllocatorImpl) gfc.getOffHeapStore(); assertEquals(1, ma.getStats().getObjects()); } } @@ -2312,7 +2312,7 @@ public abstract class MultiVMRegionTestCase extends RegionTestCase { assertEquals(1, region.size()); if (region.getAttributes().getOffHeap() && !(region instanceof PartitionedRegion)) { GemFireCacheImpl gfc = (GemFireCacheImpl) getCache(); - SimpleMemoryAllocatorImpl ma = (SimpleMemoryAllocatorImpl) gfc.getOffHeapStore(); + MemoryAllocatorImpl ma = (MemoryAllocatorImpl) gfc.getOffHeapStore(); assertEquals(1, ma.getStats().getObjects()); } } @@ -2331,13 +2331,13 @@ public abstract class MultiVMRegionTestCase extends RegionTestCase { assertEquals(1, region.size()); if (region.getAttributes().getOffHeap() && !(region instanceof PartitionedRegion)) { GemFireCacheImpl gfc = (GemFireCacheImpl) getCache(); - SimpleMemoryAllocatorImpl ma = (SimpleMemoryAllocatorImpl) gfc.getOffHeapStore(); + MemoryAllocatorImpl ma = (MemoryAllocatorImpl) gfc.getOffHeapStore(); assertEquals(1, ma.getStats().getObjects()); } region.destroyRegion(arg); if (region.getAttributes().getOffHeap() && !(region instanceof PartitionedRegion)) { GemFireCacheImpl gfc = (GemFireCacheImpl) getCache(); - final SimpleMemoryAllocatorImpl ma = (SimpleMemoryAllocatorImpl) gfc.getOffHeapStore(); + final MemoryAllocatorImpl ma = (MemoryAllocatorImpl) gfc.getOffHeapStore(); WaitCriterion waitForStatChange = new WaitCriterion() { public boolean done() { return ma.getStats().getObjects() == 0; http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/ClientServerGetAllDUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/ClientServerGetAllDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/ClientServerGetAllDUnitTest.java index f7f633c..803593e 100755 --- a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/ClientServerGetAllDUnitTest.java +++ b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/ClientServerGetAllDUnitTest.java @@ -39,7 +39,7 @@ import com.gemstone.gemfire.cache30.ClientServerTestCase; import com.gemstone.gemfire.distributed.internal.DistributionConfig; import com.gemstone.gemfire.internal.AvailablePort; import com.gemstone.gemfire.internal.AvailablePortHelper; -import com.gemstone.gemfire.internal.offheap.SimpleMemoryAllocatorImpl; +import com.gemstone.gemfire.internal.offheap.MemoryAllocatorImpl; import com.gemstone.gemfire.test.dunit.Assert; import com.gemstone.gemfire.test.dunit.AsyncInvocation; import com.gemstone.gemfire.test.dunit.DistributedTestUtils; @@ -270,7 +270,7 @@ import com.gemstone.gemfire.test.dunit.VM; server.invoke(new CacheSerializableRunnable("Dump OffHeap Stats") { @Override public void run2() throws CacheException { - SimpleMemoryAllocatorImpl ma = SimpleMemoryAllocatorImpl.getAllocator(); + MemoryAllocatorImpl ma = MemoryAllocatorImpl.getAllocator(); System.out.println("STATS: objects=" + ma.getStats().getObjects() + " usedMemory=" + ma.getStats().getUsedMemory() + " reads=" + ma.getStats().getReads()); } }); http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OffHeapTestUtil.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OffHeapTestUtil.java b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OffHeapTestUtil.java index 8fd6895..f9d2c2a 100644 --- a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OffHeapTestUtil.java +++ b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OffHeapTestUtil.java @@ -25,15 +25,15 @@ import com.gemstone.gemfire.cache.CacheClosedException; import com.gemstone.gemfire.internal.offheap.MemoryBlock; import com.gemstone.gemfire.internal.offheap.RefCountChangeInfo; import com.gemstone.gemfire.internal.offheap.ReferenceCountHelper; -import com.gemstone.gemfire.internal.offheap.SimpleMemoryAllocatorImpl; +import com.gemstone.gemfire.internal.offheap.MemoryAllocatorImpl; @SuppressWarnings("deprecation") public class OffHeapTestUtil { public static void checkOrphans() { // TODO:KIRK: need to do something special to guarantee proper tearDown - SimpleMemoryAllocatorImpl allocator = null; + MemoryAllocatorImpl allocator = null; try { - allocator = SimpleMemoryAllocatorImpl.getAllocator(); + allocator = MemoryAllocatorImpl.getAllocator(); } catch (CacheClosedException ignore) { // no off-heap memory so no orphans return; http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OffHeapValueWrapperJUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OffHeapValueWrapperJUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OffHeapValueWrapperJUnitTest.java index 0829009..550d133 100644 --- a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OffHeapValueWrapperJUnitTest.java +++ b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OffHeapValueWrapperJUnitTest.java @@ -31,7 +31,7 @@ import com.gemstone.gemfire.internal.cache.DiskEntry.Helper.OffHeapValueWrapper; import com.gemstone.gemfire.internal.cache.DiskEntry.Helper.Flushable; import com.gemstone.gemfire.internal.offheap.NullOffHeapMemoryStats; import com.gemstone.gemfire.internal.offheap.NullOutOfOffHeapMemoryListener; -import com.gemstone.gemfire.internal.offheap.SimpleMemoryAllocatorImpl; +import com.gemstone.gemfire.internal.offheap.MemoryAllocatorImpl; import com.gemstone.gemfire.internal.offheap.SlabImpl; import com.gemstone.gemfire.internal.offheap.StoredObject; import com.gemstone.gemfire.test.junit.categories.UnitTest; @@ -40,18 +40,18 @@ import com.gemstone.gemfire.test.junit.categories.UnitTest; public class OffHeapValueWrapperJUnitTest { private static OffHeapValueWrapper createChunkValueWrapper(byte[] bytes, boolean isSerialized) { - StoredObject c = SimpleMemoryAllocatorImpl.getAllocator().allocateAndInitialize(bytes, isSerialized, false); + StoredObject c = MemoryAllocatorImpl.getAllocator().allocateAndInitialize(bytes, isSerialized, false); return new OffHeapValueWrapper(c); } @Before public void setUp() throws Exception { - SimpleMemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(), new NullOffHeapMemoryStats(), new SlabImpl[]{new SlabImpl(1024*1024)}); + MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(), new NullOffHeapMemoryStats(), new SlabImpl[]{new SlabImpl(1024*1024)}); } @After public void tearDown() throws Exception { - SimpleMemoryAllocatorImpl.freeOffHeapMemory(); + MemoryAllocatorImpl.freeOffHeapMemory(); } @Test http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OldValueImporterTestBase.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OldValueImporterTestBase.java b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OldValueImporterTestBase.java index 84d7fc7..6dbe100 100644 --- a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OldValueImporterTestBase.java +++ b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/OldValueImporterTestBase.java @@ -30,7 +30,7 @@ import com.gemstone.gemfire.internal.offheap.OffHeapStoredObject; import com.gemstone.gemfire.internal.offheap.TinyStoredObject; import com.gemstone.gemfire.internal.offheap.NullOffHeapMemoryStats; import com.gemstone.gemfire.internal.offheap.NullOutOfOffHeapMemoryListener; -import com.gemstone.gemfire.internal.offheap.SimpleMemoryAllocatorImpl; +import com.gemstone.gemfire.internal.offheap.MemoryAllocatorImpl; import com.gemstone.gemfire.internal.offheap.SlabImpl; import com.gemstone.gemfire.internal.util.BlobHelper; @@ -109,8 +109,8 @@ public abstract class OldValueImporterTestBase { // off-heap DataAsAddress byte array { - SimpleMemoryAllocatorImpl sma = - SimpleMemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(), new NullOffHeapMemoryStats(), new SlabImpl[]{new SlabImpl(1024*1024)}); + MemoryAllocatorImpl sma = + MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(), new NullOffHeapMemoryStats(), new SlabImpl[]{new SlabImpl(1024*1024)}); try { byte[] baValue = new byte[] {1,2}; TinyStoredObject baValueSO = (TinyStoredObject) sma.allocateAndInitialize(baValue, false, false); @@ -121,13 +121,13 @@ public abstract class OldValueImporterTestBase { fromData(imsg, bytes); assertArrayEquals(baValue, (byte[])getOldValueFromImporter(imsg)); } finally { - SimpleMemoryAllocatorImpl.freeOffHeapMemory(); + MemoryAllocatorImpl.freeOffHeapMemory(); } } // off-heap Chunk byte array { - SimpleMemoryAllocatorImpl sma = - SimpleMemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(), new NullOffHeapMemoryStats(), new SlabImpl[]{new SlabImpl(1024*1024)}); + MemoryAllocatorImpl sma = + MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(), new NullOffHeapMemoryStats(), new SlabImpl[]{new SlabImpl(1024*1024)}); try { byte[] baValue = new byte[] {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17}; OffHeapStoredObject baValueSO = (OffHeapStoredObject) sma.allocateAndInitialize(baValue, false, false); @@ -138,13 +138,13 @@ public abstract class OldValueImporterTestBase { fromData(imsg, bytes); assertArrayEquals(baValue, (byte[])getOldValueFromImporter(imsg)); } finally { - SimpleMemoryAllocatorImpl.freeOffHeapMemory(); + MemoryAllocatorImpl.freeOffHeapMemory(); } } // off-heap DataAsAddress String { - SimpleMemoryAllocatorImpl sma = - SimpleMemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(), new NullOffHeapMemoryStats(), new SlabImpl[]{new SlabImpl(1024*1024)}); + MemoryAllocatorImpl sma = + MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(), new NullOffHeapMemoryStats(), new SlabImpl[]{new SlabImpl(1024*1024)}); try { String baValue = "12"; byte[] baValueBlob = BlobHelper.serializeToBlob(baValue); @@ -156,13 +156,13 @@ public abstract class OldValueImporterTestBase { fromData(imsg, bytes); assertArrayEquals(baValueBlob, ((VMCachedDeserializable)getOldValueFromImporter(imsg)).getSerializedValue()); } finally { - SimpleMemoryAllocatorImpl.freeOffHeapMemory(); + MemoryAllocatorImpl.freeOffHeapMemory(); } } // off-heap Chunk String { - SimpleMemoryAllocatorImpl sma = - SimpleMemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(), new NullOffHeapMemoryStats(), new SlabImpl[]{new SlabImpl(1024*1024)}); + MemoryAllocatorImpl sma = + MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(), new NullOffHeapMemoryStats(), new SlabImpl[]{new SlabImpl(1024*1024)}); try { String baValue = "12345678"; byte[] baValueBlob = BlobHelper.serializeToBlob(baValue); @@ -174,7 +174,7 @@ public abstract class OldValueImporterTestBase { fromData(imsg, bytes); assertArrayEquals(baValueBlob, ((VMCachedDeserializable)getOldValueFromImporter(imsg)).getSerializedValue()); } finally { - SimpleMemoryAllocatorImpl.freeOffHeapMemory(); + MemoryAllocatorImpl.freeOffHeapMemory(); } } } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/82faa8af/geode-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java b/geode-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java index 382bd98..950d90b 100644 --- a/geode-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java +++ b/geode-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java @@ -43,7 +43,7 @@ public class FreeListManagerTest { } private final int DEFAULT_SLAB_SIZE = 1024*1024*5; - private final SimpleMemoryAllocatorImpl ma = mock(SimpleMemoryAllocatorImpl.class); + private final MemoryAllocatorImpl ma = mock(MemoryAllocatorImpl.class); private final OffHeapMemoryStats stats = mock(OffHeapMemoryStats.class); private TestableFreeListManager freeListManager; @@ -68,7 +68,7 @@ public class FreeListManagerTest { } } - private static TestableFreeListManager createFreeListManager(SimpleMemoryAllocatorImpl ma, Slab[] slabs) { + private static TestableFreeListManager createFreeListManager(MemoryAllocatorImpl ma, Slab[] slabs) { return new TestableFreeListManager(ma, slabs); } @@ -871,7 +871,7 @@ public class FreeListManagerTest { } } - public TestableFreeListManager(SimpleMemoryAllocatorImpl ma, Slab[] slabs) { + public TestableFreeListManager(MemoryAllocatorImpl ma, Slab[] slabs) { super(ma, slabs); }