hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject hbase git commit: HBASE-15740 Replication source.shippedKBs metric is undercounting because it is in KB
Date Fri, 27 May 2016 00:35:35 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 c7445edd6 -> 1822bd3ec


HBASE-15740 Replication source.shippedKBs metric is undercounting because it is in KB


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1822bd3e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1822bd3e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1822bd3e

Branch: refs/heads/0.98
Commit: 1822bd3ec57d40a2cb682500a8a5cff5df9c9e08
Parents: c7445ed
Author: Enis Soztutar <enis@apache.org>
Authored: Mon May 9 10:25:49 2016 -0700
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Thu May 26 17:26:38 2016 -0700

----------------------------------------------------------------------
 .../MetricsReplicationSourceSource.java         |  5 +++-
 .../MetricsReplicationGlobalSourceSource.java   | 19 ++++++++++++++--
 .../MetricsReplicationSourceSourceImpl.java     | 11 +++++++--
 .../MetricsReplicationGlobalSourceSource.java   | 24 ++++++++++++++++++--
 .../MetricsReplicationSourceSourceImpl.java     | 12 ++++++++--
 .../replication/regionserver/MetricsSource.java |  6 ++---
 .../regionserver/ReplicationSource.java         |  2 +-
 7 files changed, 66 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/1822bd3e/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
index fecf191..8611e15 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
@@ -24,7 +24,10 @@ public interface MetricsReplicationSourceSource {
   public static final String SOURCE_AGE_OF_LAST_SHIPPED_OP = "source.ageOfLastShippedOp";
   public static final String SOURCE_SHIPPED_BATCHES = "source.shippedBatches";
 
+  @Deprecated
+  /** @deprecated Use SOURCE_SHIPPED_BYTES instead */
   public static final String SOURCE_SHIPPED_KBS = "source.shippedKBs";
+  public static final String SOURCE_SHIPPED_BYTES = "source.shippedBytes";
   public static final String SOURCE_SHIPPED_OPS = "source.shippedOps";
 
   public static final String SOURCE_LOG_READ_IN_BYTES = "source.logReadInBytes";
@@ -39,7 +42,7 @@ public interface MetricsReplicationSourceSource {
   void incrLogEditsFiltered(long size);
   void incrBatchesShipped(int batches);
   void incrOpsShipped(long ops);
-  void incrShippedKBs(long size);
+  void incrShippedBytes(long size);
   void incrLogReadInBytes(long size);
   void incrLogReadInEdits(long size);
   void clear();

http://git-wip-us.apache.org/repos/asf/hbase/blob/1822bd3e/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
index 8c47fa5..620b764 100644
--- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
+++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
@@ -30,6 +30,8 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
   private final MetricMutableCounterLong logEditsFilteredCounter;
   private final MetricMutableCounterLong shippedBatchesCounter;
   private final MetricMutableCounterLong shippedOpsCounter;
+  private final MetricMutableCounterLong shippedBytesCounter;
+  @Deprecated
   private final MetricMutableCounterLong shippedKBsCounter;
   private final MetricMutableCounterLong logReadInBytesCounter;
 
@@ -45,6 +47,8 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
 
     shippedKBsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_KBS, 0L);
 
+    shippedBytesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_BYTES, 0L);
+
     logReadInBytesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_READ_IN_BYTES,
0L);
 
     logReadInEditsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_READ_IN_EDITS,
0L);
@@ -85,8 +89,19 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
     shippedOpsCounter.incr(ops);
   }
 
-  @Override public void incrShippedKBs(long size) {
-    shippedKBsCounter.incr(size);
+  @Override public void incrShippedBytes(long size) {
+    incrementKBsCounter(size, shippedBytesCounter, shippedKBsCounter);
+  }
+
+  static void incrementKBsCounter(long size, MetricMutableCounterLong bytesCounter,
+      MetricMutableCounterLong kbsCounter) {
+    // Not as smart as the implementation for Hadoop 2 metrics but we care
+    // little about Hadoop 1 at this point
+    bytesCounter.incr(size);
+    long delta = size / 1024;
+    if (delta > 0) {
+      kbsCounter.incr(delta);
+    }
   }
 
   @Override public void incrLogReadInBytes(long size) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/1822bd3e/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
index b3acd6c..21a6cb3 100644
--- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
+++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
@@ -31,7 +31,9 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
   private final String logEditsFilteredKey;
   private final String shippedBatchesKey;
   private final String shippedOpsKey;
+  @Deprecated
   private final String shippedKBsKey;
+  private final String shippedBytesKey;
   private final String logReadInBytesKey;
 
   private final MetricMutableGaugeLong ageOfLastShippedOpGauge;
@@ -42,6 +44,7 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
   private final MetricMutableCounterLong shippedBatchesCounter;
   private final MetricMutableCounterLong shippedOpsCounter;
   private final MetricMutableCounterLong shippedKBsCounter;
+  private final MetricMutableCounterLong shippedBytesCounter;
   private final MetricMutableCounterLong logReadInBytesCounter;
 
   public MetricsReplicationSourceSourceImpl(MetricsReplicationSourceImpl rms, String id)
{
@@ -63,6 +66,9 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
     shippedKBsKey = "source." + this.id + ".shippedKBs";
     shippedKBsCounter = rms.getMetricsRegistry().getLongCounter(shippedKBsKey, 0L);
 
+    shippedBytesKey = "source." + this.id + ".shippedBytes";
+    shippedBytesCounter = rms.getMetricsRegistry().getLongCounter(shippedBytesKey, 0L);
+
     logReadInBytesKey = "source." + this.id + ".logReadInBytes";
     logReadInBytesCounter = rms.getMetricsRegistry().getLongCounter(logReadInBytesKey, 0L);
 
@@ -106,8 +112,9 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
     shippedOpsCounter.incr(ops);
   }
 
-  @Override public void incrShippedKBs(long size) {
-    shippedKBsCounter.incr(size);
+  @Override public void incrShippedBytes(long size) {
+    MetricsReplicationGlobalSourceSource
+      .incrementKBsCounter(size, shippedBytesCounter, shippedKBsCounter);
   }
 
   @Override public void incrLogReadInBytes(long size) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/1822bd3e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
index 08baf23..b3e1766 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
@@ -29,6 +29,8 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
   private final MutableCounterLong logEditsFilteredCounter;
   private final MutableCounterLong shippedBatchesCounter;
   private final MutableCounterLong shippedOpsCounter;
+  private final MutableCounterLong shippedBytesCounter;
+  @Deprecated
   private final MutableCounterLong shippedKBsCounter;
   private final MutableCounterLong logReadInBytesCounter;
 
@@ -44,6 +46,8 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
 
     shippedKBsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_KBS, 0L);
 
+    shippedBytesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_BYTES, 0L);
+
     logReadInBytesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_READ_IN_BYTES,
0L);
 
     logReadInEditsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_READ_IN_EDITS,
0L);
@@ -83,8 +87,24 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
     shippedOpsCounter.incr(ops);
   }
 
-  @Override public void incrShippedKBs(long size) {
-    shippedKBsCounter.incr(size);
+  @Override public void incrShippedBytes(long size) {
+    incrementKBsCounter(size, shippedBytesCounter, shippedKBsCounter);
+  }
+
+  static void incrementKBsCounter(long size, MutableCounterLong bytesCounter,
+      MutableCounterLong kbsCounter) {
+    bytesCounter.incr(size);
+    // Following code should be thread-safe.
+    long delta = 0;
+    while(true) {
+      long bytes = bytesCounter.value();
+      delta = (bytes / 1024) - kbsCounter.value();
+      if (delta > 0) {
+        kbsCounter.incr(delta);
+      } else {
+        break;
+      }
+    }
   }
 
   @Override public void incrLogReadInBytes(long size) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/1822bd3e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
index d5e52b8..d6a9128 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
@@ -31,7 +31,9 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
   private final String logEditsFilteredKey;
   private final String shippedBatchesKey;
   private final String shippedOpsKey;
+  @Deprecated
   private final String shippedKBsKey;
+  private final String shippedBytesKey;
   private final String logReadInBytesKey;
 
   private final MutableGaugeLong ageOfLastShippedOpGauge;
@@ -41,6 +43,7 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
   private final MutableCounterLong shippedBatchesCounter;
   private final MutableCounterLong shippedOpsCounter;
   private final MutableCounterLong shippedKBsCounter;
+  private final MutableCounterLong shippedBytesCounter;
   private final MutableCounterLong logReadInBytesCounter;
 
   public MetricsReplicationSourceSourceImpl(MetricsReplicationSourceImpl rms, String id)
{
@@ -62,6 +65,9 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
     shippedKBsKey = "source." + this.id + ".shippedKBs";
     shippedKBsCounter = rms.getMetricsRegistry().getLongCounter(shippedKBsKey, 0L);
 
+    shippedBytesKey = "source." + this.id + ".shippedBytes";
+    shippedBytesCounter = rms.getMetricsRegistry().getLongCounter(shippedBytesKey, 0L);
+
     logReadInBytesKey = "source." + this.id + ".logReadInBytes";
     logReadInBytesCounter = rms.getMetricsRegistry().getLongCounter(logReadInBytesKey, 0L);
 
@@ -104,8 +110,9 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
     shippedOpsCounter.incr(ops);
   }
 
-  @Override public void incrShippedKBs(long size) {
-    shippedKBsCounter.incr(size);
+  @Override public void incrShippedBytes(long size) {
+    MetricsReplicationGlobalSourceSource
+      .incrementKBsCounter(size, shippedBytesCounter, shippedKBsCounter);
   }
 
   @Override public void incrLogReadInBytes(long size) {
@@ -120,6 +127,7 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
     rms.removeMetric(shippedBatchesKey);
     rms.removeMetric(shippedOpsKey);
     rms.removeMetric(shippedKBsKey);
+    rms.removeMetric(shippedBytesKey);
 
     rms.removeMetric(logReadInBytesKey);
     rms.removeMetric(logReadInEditsKey);

http://git-wip-us.apache.org/repos/asf/hbase/blob/1822bd3e/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java
index b5dee6f..113bc5b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java
@@ -141,15 +141,15 @@ public class MetricsSource {
    *
    * @param batchSize the size of the batch that was shipped to sinks.
    */
-  public void shipBatch(long batchSize, int sizeInKB) {
+  public void shipBatch(long batchSize, int sizeInBytes) {
     singleSourceSource.incrBatchesShipped(1);
     globalSourceSource.incrBatchesShipped(1);
 
     singleSourceSource.incrOpsShipped(batchSize);
     globalSourceSource.incrOpsShipped(batchSize);
 
-    singleSourceSource.incrShippedKBs(sizeInKB);
-    globalSourceSource.incrShippedKBs(sizeInKB);
+    singleSourceSource.incrShippedBytes(sizeInBytes);
+    globalSourceSource.incrShippedBytes(sizeInBytes);
   }
 
   /** increase the byte number read by source from log file */

http://git-wip-us.apache.org/repos/asf/hbase/blob/1822bd3e/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index b7330aa..f545129 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -725,7 +725,7 @@ public class ReplicationSource extends Thread
         }
         this.totalReplicatedEdits += entries.size();
         this.totalReplicatedOperations += currentNbOperations;
-        this.metrics.shipBatch(this.currentNbOperations, this.currentSize/1024);
+        this.metrics.shipBatch(currentNbOperations, currentSize);
         this.metrics.setAgeOfLastShippedOp(entries.get(entries.size()-1).getKey().getWriteTime());
         if (LOG.isTraceEnabled()) {
           LOG.trace("Replicated " + this.totalReplicatedEdits + " entries in total, or "


Mime
View raw message