hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [3/9] hbase git commit: HBASE-9531 a command line (hbase shell) interface to retreive the replication metrics and show replication lag
Date Thu, 12 Feb 2015 22:49:44 GMT
HBASE-9531 a command line (hbase shell) interface to retreive the replication metrics and show
replication lag

Signed-off-by: Andrew Purtell <apurtell@apache.org>

Conflicts:
	hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
	hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClusterStatusProtos.java
	hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c0e26c3a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c0e26c3a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c0e26c3a

Branch: refs/heads/branch-1.0
Commit: c0e26c3a0704deb69473bf3daec851afd17688a3
Parents: bdb938b
Author: Ashish Singhi <ashish.singhi@huawei.com>
Authored: Thu Feb 12 14:00:19 2015 -0800
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Thu Feb 12 14:16:22 2015 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/ServerLoad.java     |   27 +-
 .../hadoop/hbase/protobuf/ProtobufUtil.java     |   24 +
 .../hbase/replication/ReplicationLoadSink.java  |   36 +
 .../replication/ReplicationLoadSource.java      |   53 +
 .../MetricsReplicationSinkSource.java           |    1 +
 .../MetricsReplicationSourceSource.java         |    1 +
 .../MetricsReplicationGlobalSourceSource.java   |    5 +
 .../MetricsReplicationSinkSourceImpl.java       |    5 +
 .../MetricsReplicationSourceSourceImpl.java     |    5 +
 .../protobuf/generated/ClusterStatusProtos.java | 5535 +++++++++++++-----
 .../src/main/protobuf/ClusterStatus.proto       |   23 +
 .../hbase/regionserver/HRegionServer.java       |   17 +
 .../hbase/regionserver/ReplicationService.java  |    8 +-
 .../replication/regionserver/MetricsSink.java   |   17 +
 .../replication/regionserver/MetricsSource.java |   34 +
 .../replication/regionserver/Replication.java   |   32 +-
 .../regionserver/ReplicationLoad.java           |  151 +
 .../regionserver/ReplicationSink.java           |    8 +
 .../regionserver/ReplicationSource.java         |    8 +
 .../replication/TestReplicationSmallTests.java  |   49 +-
 hbase-shell/src/main/ruby/hbase/admin.rb        |   42 +-
 .../src/main/ruby/shell/commands/status.rb      |    9 +-
 hbase-shell/src/test/ruby/hbase/admin_test.rb   |   12 +
 hbase-shell/src/test/ruby/test_helper.rb        |    4 +
 24 files changed, 4472 insertions(+), 1634 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
index 06a61c0..370862a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
@@ -22,8 +22,11 @@ package org.apache.hadoop.hbase;
 
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.Coprocessor;
+import org.apache.hadoop.hbase.replication.ReplicationLoadSink;
+import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Strings;
 
@@ -52,7 +55,7 @@ public class ServerLoad {
   private int totalStaticBloomSizeKB = 0;
   private long totalCompactingKVs = 0;
   private long currentCompactedKVs = 0;
-  
+
   public ServerLoad(ClusterStatusProtos.ServerLoad serverLoad) {
     this.serverLoad = serverLoad;
     for (ClusterStatusProtos.RegionLoad rl: serverLoad.getRegionLoadsList()) {
@@ -70,7 +73,7 @@ public class ServerLoad {
       totalCompactingKVs += rl.getTotalCompactingKVs();
       currentCompactedKVs += rl.getCurrentCompactedKVs();
     }
-    
+
   }
 
   // NOTE: Function name cannot start with "get" because then an OpenDataException is thrown
because
@@ -178,6 +181,26 @@ public class ServerLoad {
   }
 
   /**
+   * Call directly from client such as hbase shell
+   * @return the list of ReplicationLoadSource
+   */
+  public List<ReplicationLoadSource> getReplicationLoadSourceList() {
+    return ProtobufUtil.toReplicationLoadSourceList(serverLoad.getReplLoadSourceList());
+  }
+
+  /**
+   * Call directly from client such as hbase shell
+   * @return ReplicationLoadSink
+   */
+  public ReplicationLoadSink getReplicationLoadSink() {
+    if (serverLoad.hasReplLoadSink()) {
+      return ProtobufUtil.toReplicationLoadSink(serverLoad.getReplLoadSink());
+    } else {
+      return null;
+    }
+  }
+
+  /**
    * Originally, this method factored in the effect of requests going to the
    * server as well. However, this does not interact very well with the current
    * region rebalancing code, which only factors number of regions. For the

http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 9893825..56d9a64 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -102,6 +102,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.Col
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
+import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
 import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
 import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
@@ -123,6 +124,8 @@ import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;
+import org.apache.hadoop.hbase.replication.ReplicationLoadSink;
+import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
 import org.apache.hadoop.hbase.security.access.Permission;
 import org.apache.hadoop.hbase.security.access.TablePermission;
 import org.apache.hadoop.hbase.security.access.UserPermission;
@@ -2828,4 +2831,25 @@ public final class ProtobufUtil {
     }
     return result;
   }
+
+  public static ReplicationLoadSink toReplicationLoadSink(
+      ClusterStatusProtos.ReplicationLoadSink cls) {
+    return new ReplicationLoadSink(cls.getAgeOfLastAppliedOp(), cls.getTimeStampsOfLastAppliedOp());
+  }
+
+  public static ReplicationLoadSource toReplicationLoadSource(
+      ClusterStatusProtos.ReplicationLoadSource cls) {
+    return new ReplicationLoadSource(cls.getPeerID(), cls.getAgeOfLastShippedOp(),
+        cls.getSizeOfLogQueue(), cls.getTimeStampOfLastShippedOp(), cls.getReplicationLag());
+  }
+
+  public static List<ReplicationLoadSource> toReplicationLoadSourceList(
+      List<ClusterStatusProtos.ReplicationLoadSource> clsList) {
+    ArrayList<ReplicationLoadSource> rlsList = new ArrayList<ReplicationLoadSource>();
+    for (ClusterStatusProtos.ReplicationLoadSource cls : clsList) {
+      rlsList.add(toReplicationLoadSource(cls));
+    }
+    return rlsList;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java
new file mode 100644
index 0000000..63fe334
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information
regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version
2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain
a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on
an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License
+ * for the specific language governing permissions and limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * A HBase ReplicationLoad to present MetricsSink information
+ */
+@InterfaceAudience.Private
+public class ReplicationLoadSink {
+  private long ageOfLastAppliedOp;
+  private long timeStampsOfLastAppliedOp;
+
+  public ReplicationLoadSink(long age, long timeStamp) {
+    this.ageOfLastAppliedOp = age;
+    this.timeStampsOfLastAppliedOp = timeStamp;
+  }
+
+  public long getAgeOfLastAppliedOp() {
+    return this.ageOfLastAppliedOp;
+  }
+
+  public long getTimeStampsOfLastAppliedOp() {
+    return this.timeStampsOfLastAppliedOp;
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSource.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSource.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSource.java
new file mode 100644
index 0000000..bfd1599
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSource.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information
regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version
2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain
a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on
an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License
+ * for the specific language governing permissions and limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * A HBase ReplicationLoad to present MetricsSource information
+ */
+@InterfaceAudience.Private
+public class ReplicationLoadSource {
+  private String peerID;
+  private long ageOfLastShippedOp;
+  private int sizeOfLogQueue;
+  private long timeStampOfLastShippedOp;
+  private long replicationLag;
+
+  public ReplicationLoadSource(String id, long age, int size, long timeStamp, long lag) {
+    this.peerID = id;
+    this.ageOfLastShippedOp = age;
+    this.sizeOfLogQueue = size;
+    this.timeStampOfLastShippedOp = timeStamp;
+    this.replicationLag = lag;
+  }
+
+  public String getPeerID() {
+    return this.peerID;
+  }
+
+  public long getAgeOfLastShippedOp() {
+    return this.ageOfLastShippedOp;
+  }
+
+  public long getSizeOfLogQueue() {
+    return this.sizeOfLogQueue;
+  }
+
+  public long getTimeStampOfLastShippedOp() {
+    return this.timeStampOfLastShippedOp;
+  }
+
+  public long getReplicationLag() {
+    return this.replicationLag;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
index 805dfca..698a59a 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
@@ -26,4 +26,5 @@ public interface MetricsReplicationSinkSource {
   void setLastAppliedOpAge(long age);
   void incrAppliedBatches(long batches);
   void incrAppliedOps(long batchsize);
+  long getLastAppliedOpAge();
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
index 66d265a..fecf191 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
@@ -43,4 +43,5 @@ public interface MetricsReplicationSourceSource {
   void incrLogReadInBytes(long size);
   void incrLogReadInEdits(long size);
   void clear();
+  long getLastShippedAge();
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
index a210171..6dace10 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
@@ -95,4 +95,9 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
 
   @Override public void clear() {
   }
+
+  @Override
+  public long getLastShippedAge() {
+    return ageOfLastShippedOpGauge.value();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java
index 3025e3e..14212ba 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java
@@ -44,4 +44,9 @@ public class MetricsReplicationSinkSourceImpl implements MetricsReplicationSinkS
   @Override public void incrAppliedOps(long batchsize) {
     opsCounter.incr(batchsize);
   }
+
+  @Override
+  public long getLastAppliedOpAge() {
+    return ageGauge.value();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
index 89ef4de..1422e7e 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
@@ -125,4 +125,9 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
 
     rms.removeMetric(logEditsFilteredKey);
   }
+
+  @Override
+  public long getLastShippedAge() {
+    return ageOfLastShippedOpGauge.value();
+  }
 }


Mime
View raw message