Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B6CA3174F7 for ; Thu, 12 Feb 2015 22:49:45 +0000 (UTC) Received: (qmail 64614 invoked by uid 500); 12 Feb 2015 22:49:42 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 64549 invoked by uid 500); 12 Feb 2015 22:49:42 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 64536 invoked by uid 99); 12 Feb 2015 22:49:42 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Feb 2015 22:49:42 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 36D64E0540; Thu, 12 Feb 2015 22:49:42 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: apurtell@apache.org To: commits@hbase.apache.org Date: Thu, 12 Feb 2015 22:49:44 -0000 Message-Id: <515e452c7efa4dec855d045e0bd596ea@git.apache.org> In-Reply-To: <14ccfff27d0145ad8f4550a60e319a6e@git.apache.org> References: <14ccfff27d0145ad8f4550a60e319a6e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/9] hbase git commit: HBASE-9531 a command line (hbase shell) interface to retreive the replication metrics and show replication lag HBASE-9531 a command line (hbase shell) interface to retreive the replication metrics and show replication lag Signed-off-by: Andrew Purtell Conflicts: hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClusterStatusProtos.java hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c0e26c3a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c0e26c3a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c0e26c3a Branch: refs/heads/branch-1.0 Commit: c0e26c3a0704deb69473bf3daec851afd17688a3 Parents: bdb938b Author: Ashish Singhi Authored: Thu Feb 12 14:00:19 2015 -0800 Committer: Andrew Purtell Committed: Thu Feb 12 14:16:22 2015 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/ServerLoad.java | 27 +- .../hadoop/hbase/protobuf/ProtobufUtil.java | 24 + .../hbase/replication/ReplicationLoadSink.java | 36 + .../replication/ReplicationLoadSource.java | 53 + .../MetricsReplicationSinkSource.java | 1 + .../MetricsReplicationSourceSource.java | 1 + .../MetricsReplicationGlobalSourceSource.java | 5 + .../MetricsReplicationSinkSourceImpl.java | 5 + .../MetricsReplicationSourceSourceImpl.java | 5 + .../protobuf/generated/ClusterStatusProtos.java | 5535 +++++++++++++----- .../src/main/protobuf/ClusterStatus.proto | 23 + .../hbase/regionserver/HRegionServer.java | 17 + .../hbase/regionserver/ReplicationService.java | 8 +- .../replication/regionserver/MetricsSink.java | 17 + .../replication/regionserver/MetricsSource.java | 34 + .../replication/regionserver/Replication.java | 32 +- .../regionserver/ReplicationLoad.java | 151 + .../regionserver/ReplicationSink.java | 8 + .../regionserver/ReplicationSource.java | 8 + .../replication/TestReplicationSmallTests.java | 49 +- hbase-shell/src/main/ruby/hbase/admin.rb | 42 +- .../src/main/ruby/shell/commands/status.rb | 9 +- hbase-shell/src/test/ruby/hbase/admin_test.rb | 12 + hbase-shell/src/test/ruby/test_helper.rb | 4 + 24 files changed, 4472 insertions(+), 1634 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java index 06a61c0..370862a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java @@ -22,8 +22,11 @@ package org.apache.hadoop.hbase; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.Coprocessor; +import org.apache.hadoop.hbase.replication.ReplicationLoadSink; +import org.apache.hadoop.hbase.replication.ReplicationLoadSource; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Strings; @@ -52,7 +55,7 @@ public class ServerLoad { private int totalStaticBloomSizeKB = 0; private long totalCompactingKVs = 0; private long currentCompactedKVs = 0; - + public ServerLoad(ClusterStatusProtos.ServerLoad serverLoad) { this.serverLoad = serverLoad; for (ClusterStatusProtos.RegionLoad rl: serverLoad.getRegionLoadsList()) { @@ -70,7 +73,7 @@ public class ServerLoad { totalCompactingKVs += rl.getTotalCompactingKVs(); currentCompactedKVs += rl.getCurrentCompactedKVs(); } - + } // NOTE: Function name cannot start with "get" because then an OpenDataException is thrown because @@ -178,6 +181,26 @@ public class ServerLoad { } /** + * Call directly from client such as hbase shell + * @return the list of ReplicationLoadSource + */ + public List getReplicationLoadSourceList() { + return ProtobufUtil.toReplicationLoadSourceList(serverLoad.getReplLoadSourceList()); + } + + /** + * Call directly from client such as hbase shell + * @return ReplicationLoadSink + */ + public ReplicationLoadSink getReplicationLoadSink() { + if (serverLoad.hasReplLoadSink()) { + return ProtobufUtil.toReplicationLoadSink(serverLoad.getReplLoadSink()); + } else { + return null; + } + } + + /** * Originally, this method factored in the effect of requests going to the * server as well. However, this does not interact very well with the current * region rebalancing code, which only factors number of regions. For the http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index 9893825..56d9a64 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -102,6 +102,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.Col import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest; +import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos; import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad; import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; @@ -123,6 +124,8 @@ import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor; import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction; import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor; import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor.EventType; +import org.apache.hadoop.hbase.replication.ReplicationLoadSink; +import org.apache.hadoop.hbase.replication.ReplicationLoadSource; import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.access.TablePermission; import org.apache.hadoop.hbase.security.access.UserPermission; @@ -2828,4 +2831,25 @@ public final class ProtobufUtil { } return result; } + + public static ReplicationLoadSink toReplicationLoadSink( + ClusterStatusProtos.ReplicationLoadSink cls) { + return new ReplicationLoadSink(cls.getAgeOfLastAppliedOp(), cls.getTimeStampsOfLastAppliedOp()); + } + + public static ReplicationLoadSource toReplicationLoadSource( + ClusterStatusProtos.ReplicationLoadSource cls) { + return new ReplicationLoadSource(cls.getPeerID(), cls.getAgeOfLastShippedOp(), + cls.getSizeOfLogQueue(), cls.getTimeStampOfLastShippedOp(), cls.getReplicationLag()); + } + + public static List toReplicationLoadSourceList( + List clsList) { + ArrayList rlsList = new ArrayList(); + for (ClusterStatusProtos.ReplicationLoadSource cls : clsList) { + rlsList.add(toReplicationLoadSource(cls)); + } + return rlsList; + } + } http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java new file mode 100644 index 0000000..63fe334 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable + * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License + * for the specific language governing permissions and limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * A HBase ReplicationLoad to present MetricsSink information + */ +@InterfaceAudience.Private +public class ReplicationLoadSink { + private long ageOfLastAppliedOp; + private long timeStampsOfLastAppliedOp; + + public ReplicationLoadSink(long age, long timeStamp) { + this.ageOfLastAppliedOp = age; + this.timeStampsOfLastAppliedOp = timeStamp; + } + + public long getAgeOfLastAppliedOp() { + return this.ageOfLastAppliedOp; + } + + public long getTimeStampsOfLastAppliedOp() { + return this.timeStampsOfLastAppliedOp; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSource.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSource.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSource.java new file mode 100644 index 0000000..bfd1599 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSource.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable + * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License + * for the specific language governing permissions and limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * A HBase ReplicationLoad to present MetricsSource information + */ +@InterfaceAudience.Private +public class ReplicationLoadSource { + private String peerID; + private long ageOfLastShippedOp; + private int sizeOfLogQueue; + private long timeStampOfLastShippedOp; + private long replicationLag; + + public ReplicationLoadSource(String id, long age, int size, long timeStamp, long lag) { + this.peerID = id; + this.ageOfLastShippedOp = age; + this.sizeOfLogQueue = size; + this.timeStampOfLastShippedOp = timeStamp; + this.replicationLag = lag; + } + + public String getPeerID() { + return this.peerID; + } + + public long getAgeOfLastShippedOp() { + return this.ageOfLastShippedOp; + } + + public long getSizeOfLogQueue() { + return this.sizeOfLogQueue; + } + + public long getTimeStampOfLastShippedOp() { + return this.timeStampOfLastShippedOp; + } + + public long getReplicationLag() { + return this.replicationLag; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java index 805dfca..698a59a 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java @@ -26,4 +26,5 @@ public interface MetricsReplicationSinkSource { void setLastAppliedOpAge(long age); void incrAppliedBatches(long batches); void incrAppliedOps(long batchsize); + long getLastAppliedOpAge(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java index 66d265a..fecf191 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java @@ -43,4 +43,5 @@ public interface MetricsReplicationSourceSource { void incrLogReadInBytes(long size); void incrLogReadInEdits(long size); void clear(); + long getLastShippedAge(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java index a210171..6dace10 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java @@ -95,4 +95,9 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS @Override public void clear() { } + + @Override + public long getLastShippedAge() { + return ageOfLastShippedOpGauge.value(); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java index 3025e3e..14212ba 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java @@ -44,4 +44,9 @@ public class MetricsReplicationSinkSourceImpl implements MetricsReplicationSinkS @Override public void incrAppliedOps(long batchsize) { opsCounter.incr(batchsize); } + + @Override + public long getLastAppliedOpAge() { + return ageGauge.value(); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/c0e26c3a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java index 89ef4de..1422e7e 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java @@ -125,4 +125,9 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou rms.removeMetric(logEditsFilteredKey); } + + @Override + public long getLastShippedAge() { + return ageOfLastShippedOpGauge.value(); + } }