Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D7FAE200C8C for ; Mon, 22 May 2017 19:09:00 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id D3F15160BAD; Mon, 22 May 2017 17:09:00 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 241EE160BD8 for ; Mon, 22 May 2017 19:08:58 +0200 (CEST) Received: (qmail 10376 invoked by uid 500); 22 May 2017 17:08:52 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 7365 invoked by uid 99); 22 May 2017 17:08:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 22 May 2017 17:08:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 464B9F4A5B; Mon, 22 May 2017 17:08:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@hbase.apache.org Date: Mon, 22 May 2017 17:09:35 -0000 Message-Id: In-Reply-To: <1f1a6ae0ef894c62b7c847848fb42c57@git.apache.org> References: <1f1a6ae0ef894c62b7c847848fb42c57@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [48/50] [abbrv] hbase git commit: HBASE-17002 JMX metrics and some UI additions for space quotas archived-at: Mon, 22 May 2017 17:09:01 -0000 http://git-wip-us.apache.org/repos/asf/hbase/blob/1e843b4d/hbase-protocol-shaded/src/main/protobuf/Master.proto ---------------------------------------------------------------------- diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index 4706e90..3d6ae1b 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -943,7 +943,11 @@ service MasterService { rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest) returns(RemoveDrainFromRegionServersResponse); - /** Fetches the Master's view of space quotas */ + /** Fetches the Master's view of space utilization */ rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest) returns(GetSpaceQuotaRegionSizesResponse); + + /** Fetches the Master's view of quotas */ + rpc GetQuotaStates(GetQuotaStatesRequest) + returns(GetQuotaStatesResponse); } http://git-wip-us.apache.org/repos/asf/hbase/blob/1e843b4d/hbase-protocol-shaded/src/main/protobuf/Quota.proto ---------------------------------------------------------------------- diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto b/hbase-protocol-shaded/src/main/protobuf/Quota.proto index 2d7e5f5..1a6d5ed 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto @@ -119,6 +119,7 @@ message GetSpaceQuotaRegionSizesResponse { message RegionSizes { optional TableName table_name = 1; optional uint64 size = 2; + } repeated RegionSizes sizes = 1; } @@ -146,3 +147,19 @@ message GetSpaceQuotaEnforcementsResponse { } repeated TableViolationPolicy violation_policies = 1; } + +message GetQuotaStatesRequest { +} + +message GetQuotaStatesResponse { + message TableQuotaSnapshot { + optional TableName table_name = 1; + optional SpaceQuotaSnapshot snapshot = 2; + } + message NamespaceQuotaSnapshot { + optional string namespace = 1; + optional SpaceQuotaSnapshot snapshot = 2; + } + repeated TableQuotaSnapshot table_snapshots = 1; + repeated NamespaceQuotaSnapshot ns_snapshots = 2; +} http://git-wip-us.apache.org/repos/asf/hbase/blob/1e843b4d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 78268a8..33927de 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -912,7 +912,7 @@ public class HMaster extends HRegionServer implements MasterServices { // Create the quota snapshot notifier spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier(); spaceQuotaSnapshotNotifier.initialize(getClusterConnection()); - this.quotaObserverChore = new QuotaObserverChore(this); + this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics()); // Start the chore to read the region FS space reports and act on them getChoreService().scheduleChore(quotaObserverChore); } http://git-wip-us.apache.org/repos/asf/hbase/blob/1e843b4d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 50a75b9..2ac6fee 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -62,7 +62,9 @@ import org.apache.hadoop.hbase.procedure.MasterProcedureManager; import org.apache.hadoop.hbase.procedure2.LockInfo; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.quotas.MasterQuotaManager; +import org.apache.hadoop.hbase.quotas.QuotaObserverChore; import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; import org.apache.hadoop.hbase.regionserver.RSRpcServices; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; @@ -112,8 +114,12 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.NamespaceQuotaSnapshot; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.TableQuotaSnapshot; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.RegionSizes; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse; @@ -1986,4 +1992,36 @@ public class MasterRpcServices extends RSRpcServices throw new ServiceException(e); } } + + @Override + public GetQuotaStatesResponse getQuotaStates( + RpcController controller, GetQuotaStatesRequest request) throws ServiceException { + try { + master.checkInitialized(); + QuotaObserverChore quotaChore = this.master.getQuotaObserverChore(); + GetQuotaStatesResponse.Builder builder = GetQuotaStatesResponse.newBuilder(); + if (null != quotaChore) { + // The "current" view of all tables with quotas + Map tableSnapshots = quotaChore.getTableQuotaSnapshots(); + for (Entry entry : tableSnapshots.entrySet()) { + builder.addTableSnapshots( + TableQuotaSnapshot.newBuilder() + .setTableName(ProtobufUtil.toProtoTableName(entry.getKey())) + .setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(entry.getValue())).build()); + } + // The "current" view of all namespaces with quotas + Map nsSnapshots = quotaChore.getNamespaceQuotaSnapshots(); + for (Entry entry : nsSnapshots.entrySet()) { + builder.addNsSnapshots( + NamespaceQuotaSnapshot.newBuilder() + .setNamespace(entry.getKey()) + .setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(entry.getValue())).build()); + } + return builder.build(); + } + return builder.build(); + } catch (Exception e) { + throw new ServiceException(e); + } + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/1e843b4d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java index d055853..b5bc3d7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java @@ -37,11 +37,14 @@ public class MetricsMaster { private static final Log LOG = LogFactory.getLog(MetricsMaster.class); private MetricsMasterSource masterSource; private MetricsMasterProcSource masterProcSource; + private MetricsMasterQuotaSource masterQuotaSource; public MetricsMaster(MetricsMasterWrapper masterWrapper) { masterSource = CompatibilitySingletonFactory.getInstance(MetricsMasterSourceFactory.class).create(masterWrapper); masterProcSource = CompatibilitySingletonFactory.getInstance(MetricsMasterProcSourceFactory.class).create(masterWrapper); + masterQuotaSource = + CompatibilitySingletonFactory.getInstance(MetricsMasterQuotaSourceFactory.class).create(masterWrapper); } // for unit-test usage @@ -53,10 +56,49 @@ public class MetricsMaster { return masterProcSource; } + public MetricsMasterQuotaSource getMetricsQuotaSource() { + return masterQuotaSource; + } + /** * @param inc How much to add to requests. */ public void incrementRequests(final long inc) { masterSource.incRequests(inc); } + + /** + * Sets the number of space quotas defined. + */ + public void setNumSpaceQuotas(final long numSpaceQuotas) { + masterQuotaSource.updateNumSpaceQuotas(numSpaceQuotas); + } + + /** + * Sets the number of table in violation of a space quota. + */ + public void setNumTableInSpaceQuotaViolation(final long numTablesInViolation) { + masterQuotaSource.updateNumTablesInSpaceQuotaViolation(numTablesInViolation); + } + + /** + * Sets the number of namespaces in violation of a space quota. + */ + public void setNumNamespacesInSpaceQuotaViolation(final long numNamespacesInViolation) { + masterQuotaSource.updateNumNamespacesInSpaceQuotaViolation(numNamespacesInViolation); + } + + /** + * Sets the number of region size reports the master has seen. + */ + public void setNumRegionSizeReports(final long numRegionReports) { + masterQuotaSource.updateNumCurrentSpaceQuotaRegionSizeReports(numRegionReports); + } + + /** + * Sets the execution time of a period of the QuotaObserverChore. + */ + public void incrementQuotaObserverTime(final long executionTime) { + masterQuotaSource.incrementSpaceQuotaObserverChoreTime(executionTime); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/1e843b4d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java index 4cff28b..cbf7ba5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java @@ -17,9 +17,18 @@ */ package org.apache.hadoop.hbase.master; +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.quotas.QuotaObserverChore; +import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; /** @@ -134,4 +143,35 @@ public class MetricsMasterWrapperImpl implements MetricsMasterWrapper { return master.getNumWALFiles(); } + @Override + public Map> getTableSpaceUtilization() { + QuotaObserverChore quotaChore = master.getQuotaObserverChore(); + if (null == quotaChore) { + return Collections.emptyMap(); + } + Map tableSnapshots = quotaChore.getTableQuotaSnapshots(); + Map> convertedData = new HashMap<>(); + for (Entry entry : tableSnapshots.entrySet()) { + convertedData.put(entry.getKey().toString(), convertSnapshot(entry.getValue())); + } + return convertedData; + } + + @Override + public Map> getNamespaceSpaceUtilization() { + QuotaObserverChore quotaChore = master.getQuotaObserverChore(); + if (null == quotaChore) { + return Collections.emptyMap(); + } + Map namespaceSnapshots = quotaChore.getNamespaceQuotaSnapshots(); + Map> convertedData = new HashMap<>(); + for (Entry entry : namespaceSnapshots.entrySet()) { + convertedData.put(entry.getKey(), convertSnapshot(entry.getValue())); + } + return convertedData; + } + + Entry convertSnapshot(SpaceQuotaSnapshot snapshot) { + return new SimpleImmutableEntry(snapshot.getUsage(), snapshot.getLimit()); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/1e843b4d/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java index 79effe7..94c5c87 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java @@ -18,12 +18,12 @@ package org.apache.hadoop.hbase.quotas; import java.io.IOException; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MetricsMaster; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota; @@ -78,6 +79,7 @@ public class QuotaObserverChore extends ScheduledChore { private final Connection conn; private final Configuration conf; private final MasterQuotaManager quotaManager; + private final MetricsMaster metrics; /* * Callback that changes in quota snapshots are passed to. */ @@ -87,7 +89,9 @@ public class QuotaObserverChore extends ScheduledChore { * Preserves the state of quota snapshots for tables and namespaces */ private final Map tableQuotaSnapshots; + private final Map readOnlyTableQuotaSnapshots; private final Map namespaceQuotaSnapshots; + private final Map readOnlyNamespaceSnapshots; // The time, in millis, that region reports should be kept by the master private final long regionReportLifetimeMillis; @@ -98,25 +102,28 @@ public class QuotaObserverChore extends ScheduledChore { private QuotaSnapshotStore tableSnapshotStore; private QuotaSnapshotStore namespaceSnapshotStore; - public QuotaObserverChore(HMaster master) { + public QuotaObserverChore(HMaster master, MetricsMaster metrics) { this( master.getConnection(), master.getConfiguration(), master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(), - master); + master, metrics); } QuotaObserverChore( Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier snapshotNotifier, - MasterQuotaManager quotaManager, Stoppable stopper) { + MasterQuotaManager quotaManager, Stoppable stopper, MetricsMaster metrics) { super( QuotaObserverChore.class.getSimpleName(), stopper, getPeriod(conf), getInitialDelay(conf), getTimeUnit(conf)); this.conn = conn; this.conf = conf; + this.metrics = metrics; this.quotaManager = quotaManager; this.snapshotNotifier = Objects.requireNonNull(snapshotNotifier); - this.tableQuotaSnapshots = new HashMap<>(); - this.namespaceQuotaSnapshots = new HashMap<>(); + this.tableQuotaSnapshots = new ConcurrentHashMap<>(); + this.readOnlyTableQuotaSnapshots = Collections.unmodifiableMap(tableQuotaSnapshots); + this.namespaceQuotaSnapshots = new ConcurrentHashMap<>(); + this.readOnlyNamespaceSnapshots = Collections.unmodifiableMap(namespaceQuotaSnapshots); this.regionReportLifetimeMillis = conf.getLong( REGION_REPORT_RETENTION_DURATION_KEY, REGION_REPORT_RETENTION_DURATION_DEFAULT); } @@ -127,7 +134,11 @@ public class QuotaObserverChore extends ScheduledChore { if (LOG.isTraceEnabled()) { LOG.trace("Refreshing space quotas in RegionServer"); } + long start = System.nanoTime(); _chore(); + if (null != metrics) { + metrics.incrementQuotaObserverTime((System.nanoTime() - start) / 1_000_000); + } } catch (IOException e) { LOG.warn("Failed to process quota reports and update quota state. Will retry.", e); } @@ -141,6 +152,12 @@ public class QuotaObserverChore extends ScheduledChore { LOG.trace("Found following tables with quotas: " + tablesWithQuotas); } + if (null != metrics) { + // Set the number of namespaces and tables with quotas defined + metrics.setNumSpaceQuotas(tablesWithQuotas.getTableQuotaTables().size() + + tablesWithQuotas.getNamespacesWithQuotas().size()); + } + // The current "view" of region space use. Used henceforth. final Map reportedRegionSpaceUse = quotaManager.snapshotRegionSizes(); if (LOG.isTraceEnabled()) { @@ -152,6 +169,10 @@ public class QuotaObserverChore extends ScheduledChore { // Create the stores to track table and namespace snapshots initializeSnapshotStores(reportedRegionSpaceUse); + // Report the number of (non-expired) region size reports + if (null != metrics) { + metrics.setNumRegionSizeReports(reportedRegionSpaceUse.size()); + } // Filter out tables for which we don't have adequate regionspace reports yet. // Important that we do this after we instantiate the stores above @@ -215,6 +236,7 @@ public class QuotaObserverChore extends ScheduledChore { * @param tablesWithTableQuotas The HBase tables which have quotas defined */ void processTablesWithQuotas(final Set tablesWithTableQuotas) throws IOException { + long numTablesInViolation = 0L; for (TableName table : tablesWithTableQuotas) { final SpaceQuota spaceQuota = tableSnapshotStore.getSpaceQuota(table); if (null == spaceQuota) { @@ -227,9 +249,18 @@ public class QuotaObserverChore extends ScheduledChore { final SpaceQuotaSnapshot currentSnapshot = tableSnapshotStore.getCurrentState(table); final SpaceQuotaSnapshot targetSnapshot = tableSnapshotStore.getTargetState(table, spaceQuota); if (LOG.isTraceEnabled()) { - LOG.trace("Processing " + table + " with current=" + currentSnapshot + ", target=" + targetSnapshot); + LOG.trace("Processing " + table + " with current=" + currentSnapshot + ", target=" + + targetSnapshot); } updateTableQuota(table, currentSnapshot, targetSnapshot); + + if (targetSnapshot.getQuotaStatus().isInViolation()) { + numTablesInViolation++; + } + } + // Report the number of tables in violation + if (null != metrics) { + metrics.setNumTableInSpaceQuotaViolation(numTablesInViolation); } } @@ -246,6 +277,7 @@ public class QuotaObserverChore extends ScheduledChore { void processNamespacesWithQuotas( final Set namespacesWithQuotas, final Multimap tablesByNamespace) throws IOException { + long numNamespacesInViolation = 0L; for (String namespace : namespacesWithQuotas) { // Get the quota definition for the namespace final SpaceQuota spaceQuota = namespaceSnapshotStore.getSpaceQuota(namespace); @@ -257,8 +289,22 @@ public class QuotaObserverChore extends ScheduledChore { continue; } final SpaceQuotaSnapshot currentSnapshot = namespaceSnapshotStore.getCurrentState(namespace); - final SpaceQuotaSnapshot targetSnapshot = namespaceSnapshotStore.getTargetState(namespace, spaceQuota); + final SpaceQuotaSnapshot targetSnapshot = namespaceSnapshotStore.getTargetState( + namespace, spaceQuota); + if (LOG.isTraceEnabled()) { + LOG.trace("Processing " + namespace + " with current=" + currentSnapshot + ", target=" + + targetSnapshot); + } updateNamespaceQuota(namespace, currentSnapshot, targetSnapshot, tablesByNamespace); + + if (targetSnapshot.getQuotaStatus().isInViolation()) { + numNamespacesInViolation++; + } + } + + // Report the number of namespaces in violation + if (null != metrics) { + metrics.setNumNamespacesInSpaceQuotaViolation(numNamespacesInViolation); } } @@ -285,14 +331,16 @@ public class QuotaObserverChore extends ScheduledChore { } } else if (LOG.isDebugEnabled()) { // We're either moving into violation or changing violation policies - LOG.debug(table + " moving into violation of table space quota with policy of " + targetStatus.getPolicy()); + LOG.debug(table + " moving into violation of table space quota with policy of " + + targetStatus.getPolicy()); } this.snapshotNotifier.transitionTable(table, targetSnapshot); // Update it in memory tableSnapshotStore.setCurrentState(table, targetSnapshot); } else if (LOG.isTraceEnabled()) { - // Policies are the same, so we have nothing to do except log this. Don't need to re-update the quota table + // Policies are the same, so we have nothing to do except log this. Don't need to re-update + // the quota table if (!currentStatus.isInViolation()) { LOG.trace(table + " remains in observance of quota."); } else { @@ -347,11 +395,14 @@ public class QuotaObserverChore extends ScheduledChore { } } else { // No table quota present or a table quota present that is not in violation - LOG.info(tableInNS + " moving into violation of namespace space quota with policy " + targetStatus.getPolicy()); + LOG.info(tableInNS + " moving into violation of namespace space quota with policy " + + targetStatus.getPolicy()); this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot); } } } + // Update the new state in memory for this namespace + namespaceSnapshotStore.setCurrentState(namespace, targetSnapshot); } else { // Policies are the same if (!targetStatus.isInViolation()) { @@ -360,7 +411,8 @@ public class QuotaObserverChore extends ScheduledChore { LOG.trace(namespace + " remains in observance of quota."); } } else { - // Namespace quota is still in violation, need to enact if the table quota is not taking priority. + // Namespace quota is still in violation, need to enact if the table quota is not + // taking priority. for (TableName tableInNS : tablesByNamespace.get(namespace)) { // Does a table policy exist if (tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation()) { @@ -451,6 +503,22 @@ public class QuotaObserverChore extends ScheduledChore { } /** + * Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects + * for each HBase table with a quota. + */ + public Map getTableQuotaSnapshots() { + return readOnlyTableQuotaSnapshots; + } + + /** + * Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects + * for each HBase namespace with a quota. + */ + public Map getNamespaceQuotaSnapshots() { + return readOnlyNamespaceSnapshots; + } + + /** * Fetches the {@link SpaceQuotaSnapshot} for the given table. */ SpaceQuotaSnapshot getTableQuotaSnapshot(TableName table) { http://git-wip-us.apache.org/repos/asf/hbase/blob/1e843b4d/hbase-server/src/main/resources/hbase-webapps/master/table.jsp ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index 0e1d1cf..c62d3a6 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -42,10 +42,14 @@ import="org.apache.hadoop.hbase.HConstants" import="org.apache.hadoop.hbase.master.HMaster" import="org.apache.hadoop.hbase.zookeeper.MetaTableLocator" + import="org.apache.hadoop.hbase.quotas.QuotaTableUtil" + import="org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot" import="org.apache.hadoop.hbase.util.Bytes" import="org.apache.hadoop.hbase.util.FSUtils" import="org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos" import="org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos" + import="org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas" + import="org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota" import="org.apache.hadoop.hbase.TableName" import="org.apache.hadoop.hbase.HColumnDescriptor" import="org.apache.hadoop.hbase.HBaseConfiguration" @@ -87,6 +91,7 @@ if (showFragmentation) { frags = FSUtils.getTableFragmentation(master); } + boolean quotasEnabled = conf.getBoolean("hbase.quota.enabled", false); String action = request.getParameter("action"); String key = request.getParameter("key"); String left = request.getParameter("left"); @@ -328,6 +333,60 @@ if ( fqtn != null ) { How fragmented is the table. After a major compaction it is 0%. <% } %> +<% + if (quotasEnabled) { + TableName tn = TableName.valueOf(fqtn); + SpaceQuotaSnapshot masterSnapshot = null; + Quotas quota = QuotaTableUtil.getTableQuota(master.getConnection(), tn); + if (null == quota || !quota.hasSpace()) { + quota = QuotaTableUtil.getNamespaceQuota(master.getConnection(), tn.getNamespaceAsString()); + if (null != quota) { + masterSnapshot = QuotaTableUtil.getCurrentSnapshot(master.getConnection(), tn.getNamespaceAsString()); + } + } else { + masterSnapshot = QuotaTableUtil.getCurrentSnapshot(master.getConnection(), tn); + } + if (null != quota && quota.hasSpace()) { + SpaceQuota spaceQuota = quota.getSpace(); +%> + + Space Quota + + + + + + + + + + + + + + +<% + if (null != masterSnapshot) { +%> + + + + + + + + +<% + } +%> +
PropertyValue
Limit<%= StringUtils.byteDesc(spaceQuota.getSoftLimit()) %>
Policy<%= spaceQuota.getViolationPolicy() %>
Usage<%= StringUtils.byteDesc(masterSnapshot.getUsage()) %>
State<%= masterSnapshot.getQuotaStatus().isInViolation() ? "In Violation" : "In Observance" %>
+ + Information about a Space Quota on this table, if set. + +<% + } + } +%>

Table Schema

http://git-wip-us.apache.org/repos/asf/hbase/blob/1e843b4d/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java index 02f3721..8c1138a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java @@ -19,9 +19,14 @@ package org.apache.hadoop.hbase.master; import static org.junit.Assert.*; +import java.util.AbstractMap.SimpleImmutableEntry; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; +import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus; +import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Threads; @@ -77,4 +82,16 @@ public class TestMasterMetricsWrapper { assertEquals(1, info.getNumDeadRegionServers()); assertEquals(1, info.getNumWALFiles()); } + + @Test + public void testQuotaSnapshotConversion() { + MetricsMasterWrapperImpl info = new MetricsMasterWrapperImpl( + TEST_UTIL.getHBaseCluster().getMaster()); + assertEquals(new SimpleImmutableEntry(1024L, 2048L), + info.convertSnapshot(new SpaceQuotaSnapshot( + SpaceQuotaStatus.notInViolation(), 1024L, 2048L))); + assertEquals(new SimpleImmutableEntry(4096L, 2048L), + info.convertSnapshot(new SpaceQuotaSnapshot( + new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 2048L))); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/1e843b4d/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java index 5a00aaf..38dbf66 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -33,6 +34,7 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.Waiter.Predicate; +import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus; import org.apache.hadoop.hbase.regionserver.HRegionServer; @@ -187,6 +189,87 @@ public class TestQuotaStatusRPCs { assertEquals(SpaceViolationPolicy.NO_INSERTS, policy); } + @Test + public void testQuotaStatusFromMaster() throws Exception { + final long sizeLimit = 1024L * 10L; // 10KB + final long tableSize = 1024L * 5; // 5KB + final long nsLimit = Long.MAX_VALUE; + final int numRegions = 10; + final TableName tn = helper.createTableWithRegions(numRegions); + + // Define the quota + QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( + tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS); + TEST_UTIL.getAdmin().setQuota(settings); + QuotaSettings nsSettings = QuotaSettingsFactory.limitNamespaceSpace( + tn.getNamespaceAsString(), nsLimit, SpaceViolationPolicy.NO_INSERTS); + TEST_UTIL.getAdmin().setQuota(nsSettings); + + // Write at least `tableSize` data + helper.writeData(tn, tableSize); + + final Connection conn = TEST_UTIL.getConnection(); + // Make sure the master has a snapshot for our table + Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate() { + @Override + public boolean evaluate() throws Exception { + SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(conn, tn); + LOG.info("Table snapshot after initial ingest: " + snapshot); + if (null == snapshot) { + return false; + } + return snapshot.getLimit() == sizeLimit && snapshot.getUsage() > 0L; + } + }); + final AtomicReference nsUsage = new AtomicReference<>(); + // If we saw the table snapshot, we should also see the namespace snapshot + Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000 * 1000, new Predicate() { + @Override + public boolean evaluate() throws Exception { + SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot( + conn, tn.getNamespaceAsString()); + LOG.debug("Namespace snapshot after initial ingest: " + snapshot); + if (null == snapshot) { + return false; + } + nsUsage.set(snapshot.getUsage()); + return snapshot.getLimit() == nsLimit && snapshot.getUsage() > 0; + } + }); + + try { + helper.writeData(tn, tableSize * 2L); + } catch (SpaceLimitingException e) { + // Pass + } + + // Wait for the status to move to violation + Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate() { + @Override + public boolean evaluate() throws Exception { + SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(conn, tn); + LOG.info("Table snapshot after second ingest: " + snapshot); + if (null == snapshot) { + return false; + } + return snapshot.getQuotaStatus().isInViolation(); + } + }); + // The namespace should still not be in violation, but have a larger usage than previously + Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate() { + @Override + public boolean evaluate() throws Exception { + SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot( + conn, tn.getNamespaceAsString()); + LOG.debug("Namespace snapshot after second ingest: " + snapshot); + if (null == snapshot) { + return false; + } + return snapshot.getUsage() > nsUsage.get() && !snapshot.getQuotaStatus().isInViolation(); + } + }); + } + private int countRegionsForTable(TableName tn, Map regionSizes) { int size = 0; for (HRegionInfo regionInfo : regionSizes.keySet()) {