Return-Path: X-Original-To: apmail-geode-commits-archive@minotaur.apache.org Delivered-To: apmail-geode-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 440D118015 for ; Mon, 27 Jul 2015 23:19:35 +0000 (UTC) Received: (qmail 24124 invoked by uid 500); 27 Jul 2015 23:19:32 -0000 Delivered-To: apmail-geode-commits-archive@geode.apache.org Received: (qmail 24094 invoked by uid 500); 27 Jul 2015 23:19:32 -0000 Mailing-List: contact commits-help@geode.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@geode.incubator.apache.org Delivered-To: mailing list commits@geode.incubator.apache.org Received: (qmail 24085 invoked by uid 99); 27 Jul 2015 23:19:32 -0000 Received: from Unknown (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 27 Jul 2015 23:19:32 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id B5C00D80E4 for ; Mon, 27 Jul 2015 23:19:31 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 0.512 X-Spam-Level: X-Spam-Status: No, score=0.512 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RP_MATCHES_RCVD=-1.289, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-us-west.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id IZkNJHR23YO7 for ; Mon, 27 Jul 2015 23:19:25 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-us-west.apache.org (ASF Mail Server at mx1-us-west.apache.org) with SMTP id E74492D629 for ; Mon, 27 Jul 2015 23:19:24 +0000 (UTC) Received: (qmail 23718 invoked by uid 99); 27 Jul 2015 23:19:24 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 27 Jul 2015 23:19:24 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A9D6BE0F7D; Mon, 27 Jul 2015 23:19:24 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: bschuchardt@apache.org To: commits@geode.incubator.apache.org Date: Mon, 27 Jul 2015 23:19:28 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [05/50] [abbrv] incubator-geode git commit: GEODE-102: Stats for bytes overflowed on disk, local-max-memory GEODE-102: Stats for bytes overflowed on disk, local-max-memory These data are used in rebalancing, so we should have a stat that tracks them. Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/8c411a86 Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/8c411a86 Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/8c411a86 Branch: refs/heads/feature/GEODE-77 Commit: 8c411a861617a1b6988155150229ccef3d8b2000 Parents: f7242d2 Author: Dan Smith Authored: Tue Jun 9 18:14:31 2015 -0700 Committer: Dan Smith Committed: Tue Jul 14 12:12:50 2015 -0700 ---------------------------------------------------------------------- .../gemfire/internal/cache/DiskEntry.java | 15 +++++++++++-- .../gemfire/internal/cache/DiskRegion.java | 9 ++++++++ .../gemfire/internal/cache/DiskRegionStats.java | 22 ++++++++++++++++++++ .../internal/cache/PartitionedRegion.java | 1 + .../internal/cache/PartitionedRegionStats.java | 12 ++++++++++- 5 files changed, 56 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/8c411a86/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskEntry.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskEntry.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskEntry.java index b67d0f1..d02085f 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskEntry.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskEntry.java @@ -514,6 +514,7 @@ public interface DiskEntry extends RegionEntry { did.setValueLength(re.getValueLength()); if (re.getRecoveredKeyId() < 0) { drv.incNumOverflowOnDisk(1L); + drv.incNumOverflowBytesOnDisk(did.getValueLength()); incrementBucketStats(r, 0/*InVM*/, 1/*OnDisk*/, did.getValueLength()); } else { @@ -947,6 +948,7 @@ public interface DiskEntry extends RegionEntry { // First, undo the stats done for the previous recovered value if (oldKeyId < 0) { dr.incNumOverflowOnDisk(-1L); + dr.incNumOverflowBytesOnDisk(-oldValueLength); incrementBucketStats(region, 0/*InVM*/, -1/*OnDisk*/, -oldValueLength); } else { dr.incNumEntriesInVM(-1L); @@ -964,6 +966,7 @@ public interface DiskEntry extends RegionEntry { } dr.incNumOverflowOnDisk(1L); + dr.incNumOverflowBytesOnDisk(did.getValueLength()); incrementBucketStats(region, 0/*InVM*/, 1/*OnDisk*/, did.getValueLength()); } else { @@ -1054,6 +1057,7 @@ public interface DiskEntry extends RegionEntry { // done by lruEntryUpdate dr.incNumEntriesInVM(1L); dr.incNumOverflowOnDisk(-1L); + dr.incNumOverflowBytesOnDisk(-oldValueLength); incrementBucketStats(region, 1/*InVM*/, -1/*OnDisk*/, -oldValueLength); } } @@ -1092,6 +1096,7 @@ public interface DiskEntry extends RegionEntry { DiskId did = entry.getDiskId(); synchronized (did) { boolean oldValueWasNull = entry.isValueNull(); + int oldValueLength = did.getValueLength(); // Now that oplog creates are immediately put in cache // a later oplog modify will get us here long oldOplogId = did.getOplogId(); @@ -1126,6 +1131,9 @@ public interface DiskEntry extends RegionEntry { // done by lruEntryUpdate drv.incNumEntriesInVM(1L); drv.incNumOverflowOnDisk(-1L); + drv.incNumOverflowBytesOnDisk(-oldValueLength); + //No need to call incrementBucketStats here because we don't have + //a real bucket region, this is during recovery from disk. } } } @@ -1419,6 +1427,7 @@ public interface DiskEntry extends RegionEntry { entry.setValueWithContext((RegionEntryContext) region, preparedValue); dr.incNumEntriesInVM(1L); dr.incNumOverflowOnDisk(-1L); + dr.incNumOverflowBytesOnDisk(-bytesOnDisk); incrementBucketStats(region, 1/*InVM*/, -1/*OnDisk*/, -bytesOnDisk); return preparedValue; } @@ -1546,12 +1555,13 @@ public interface DiskEntry extends RegionEntry { movedValueToDisk = true; change = ((LRUClockNode)entry).updateEntrySize(ccHelper); } - dr.incNumEntriesInVM(-1L); - dr.incNumOverflowOnDisk(1L); int valueLength = 0; if (movedValueToDisk) { valueLength = getValueLength(did); } + dr.incNumEntriesInVM(-1L); + dr.incNumOverflowOnDisk(1L); + dr.incNumOverflowBytesOnDisk(valueLength); incrementBucketStats(region, -1/*InVM*/, 1/*OnDisk*/, valueLength); } } finally { @@ -1844,6 +1854,7 @@ public interface DiskEntry extends RegionEntry { } if (valueWasNull) { dr.incNumOverflowOnDisk(-1L); + dr.incNumOverflowBytesOnDisk(-oldValueLength); incrementBucketStats(region, 0/*InVM*/, -1/*OnDisk*/, -oldValueLength); } else { http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/8c411a86/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegion.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegion.java index e8548ed..1ca8f7f 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegion.java @@ -235,6 +235,12 @@ public class DiskRegion extends AbstractDiskRegion { } @Override + public void incNumOverflowBytesOnDisk(long delta) { + getStats().incNumOverflowBytesOnDisk(delta); + super.incNumOverflowBytesOnDisk(delta); + } + + @Override public void incNumOverflowOnDisk(long delta) { getStats().incNumOverflowOnDisk(delta); super.incNumOverflowOnDisk(delta); @@ -514,10 +520,13 @@ public class DiskRegion extends AbstractDiskRegion { BucketRegion owner=(BucketRegion)region; long curInVM = owner.getNumEntriesInVM()*-1; long curOnDisk = owner.getNumOverflowOnDisk()*-1; + long curOnDiskBytes = owner.getNumOverflowBytesOnDisk()*-1; incNumEntriesInVM(curInVM); incNumOverflowOnDisk(curOnDisk); + incNumOverflowBytesOnDisk(curOnDiskBytes); owner.incNumEntriesInVM(curInVM); owner.incNumOverflowOnDisk(curOnDisk); + owner.incNumOverflowBytesOnDisk(curOnDiskBytes); } else { // set them both to zero incNumEntriesInVM(getNumEntriesInVM()*-1); http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/8c411a86/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegionStats.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegionStats.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegionStats.java index c2f6eee..cc53438 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegionStats.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegionStats.java @@ -38,6 +38,7 @@ public class DiskRegionStats { private static final int removeTimeId; private static final int numOverflowOnDiskId; private static final int numEntriesInVMId; + private static final int numOverflowBytesOnDiskId; private static final int localInitializationsId; private static final int remoteInitializationsId; @@ -67,6 +68,8 @@ public class DiskRegionStats { "The total amount of time spent removing from disk"; final String numOverflowOnDiskDesc = "The current number of entries whose value is on disk and is not in memory. This is true of overflowed entries. It is also true of recovered entries that have not yet been faulted in."; + final String numOverflowBytesOnDiskDesc = + "The current number bytes on disk and not in memory. This is true of overflowed entries. It is also true of recovered entries that have not yet been faulted in."; final String numEntriesInVMDesc = "The current number of entries whose value resides in the VM. The value may also have been written to disk."; final String localInitializationsDesc = @@ -87,6 +90,7 @@ public class DiskRegionStats { f.createLongCounter("removes", removesDesc, "ops"), f.createLongCounter("removeTime", removeTimeDesc, "nanoseconds"), f.createLongGauge("entriesOnlyOnDisk", numOverflowOnDiskDesc, "entries"), + f.createLongGauge("bytesOnlyOnDisk", numOverflowBytesOnDiskDesc, "bytes"), f.createLongGauge("entriesInVM", numEntriesInVMDesc, "entries"), f.createIntGauge("writesInProgress", "current number of oplog writes that are in progress", "writes"), f.createIntGauge("localInitializations", localInitializationsDesc, "initializations"), @@ -104,6 +108,7 @@ public class DiskRegionStats { removesId = type.nameToId("removes"); removeTimeId = type.nameToId("removeTime"); numOverflowOnDiskId = type.nameToId("entriesOnlyOnDisk"); + numOverflowBytesOnDiskId = type.nameToId("bytesOnlyOnDisk"); numEntriesInVMId = type.nameToId("entriesInVM"); localInitializationsId = type.nameToId("localInitializations"); @@ -198,6 +203,15 @@ public class DiskRegionStats { public long getNumOverflowOnDisk() { return this.stats.getLong(numOverflowOnDiskId); } + + /** + * Returns the current number of entries whose value has been + * overflowed to disk. This value will decrease when a value is + * faulted in. + */ + public long getNumOverflowBytesOnDisk() { + return this.stats.getLong(numOverflowBytesOnDiskId); + } /** * Returns the current number of entries whose value resides in the @@ -223,6 +237,14 @@ public class DiskRegionStats { public void incNumEntriesInVM(long delta) { this.stats.incLong(numEntriesInVMId, delta); } + + /** + * Increments the current number of entries whose value has been + * overflowed to disk by a given amount. + */ + public void incNumOverflowBytesOnDisk(long delta) { + this.stats.incLong(numOverflowBytesOnDiskId, delta); + } /** * Invoked before data is written to disk. http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/8c411a86/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java index bc164a8..003b691 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java @@ -764,6 +764,7 @@ public class PartitionedRegion extends LocalRegion implements // this.scope = Scope.LOCAL; this.redundantCopies = ra.getPartitionAttributes().getRedundantCopies(); this.prStats.setConfiguredRedundantCopies(ra.getPartitionAttributes().getRedundantCopies()); + this.prStats.setLocalMaxMemory(ra.getPartitionAttributes().getLocalMaxMemory() * 1024L * 1024); // No redundancy required for writes this.minimumWriteRedundancy = Integer.getInteger( http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/8c411a86/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionStats.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionStats.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionStats.java index 563c38b..eda4761 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionStats.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionStats.java @@ -165,6 +165,8 @@ public class PartitionedRegionStats { private static final int prMetaDataSentCountId; + private static final int localMaxMemoryId; + static { final boolean largerIsBetter = true; StatisticsTypeFactory f = StatisticsTypeFactoryImpl.singleton(); @@ -549,6 +551,10 @@ public class PartitionedRegionStats { "prMetaDataSentCount", "total number of times meta data refreshed sent on client's request.", "operation", false), + + f.createLongGauge("localMaxMemory", + "local max memory in bytes for this region on this member", + "bytes") }); @@ -651,6 +657,8 @@ public class PartitionedRegionStats { putLocalTimeId = type.nameToId("putLocalTime"); prMetaDataSentCountId = type.nameToId("prMetaDataSentCount"); + + localMaxMemoryId = type.nameToId("localMaxMemory"); } private final Statistics stats; @@ -999,7 +1007,9 @@ type, name /* fixes bug 42343 */); public void setConfiguredRedundantCopies(int val) { this.stats.setInt(configuredRedundantCopiesId, val); } - + public void setLocalMaxMemory(long l) { + this.stats.setLong(localMaxMemoryId, l); + } public int getActualRedundantCopies() { return this.stats.getInt(actualRedundantCopiesId); }