hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [47/54] [abbrv] hbase git commit: HBASE-17002 JMX metrics and some UI additions for space quotas
Date Tue, 11 Apr 2017 17:31:52 GMT
http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-protocol-shaded/src/main/protobuf/Master.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto
index 58e6f77..90a7f07 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Master.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto
@@ -930,7 +930,11 @@ service MasterService {
   rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest)
     returns(RemoveDrainFromRegionServersResponse);
 
-  /** Fetches the Master's view of space quotas */
+  /** Fetches the Master's view of space utilization */
   rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest)
     returns(GetSpaceQuotaRegionSizesResponse);
+
+  /** Fetches the Master's view of quotas */
+  rpc GetQuotaStates(GetQuotaStatesRequest)
+    returns(GetQuotaStatesResponse);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-protocol-shaded/src/main/protobuf/Quota.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
index 2d7e5f5..1a6d5ed 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
@@ -119,6 +119,7 @@ message GetSpaceQuotaRegionSizesResponse {
   message RegionSizes {
     optional TableName table_name = 1;
     optional uint64 size = 2;
+
   }
   repeated RegionSizes sizes = 1;
 }
@@ -146,3 +147,19 @@ message GetSpaceQuotaEnforcementsResponse {
   }
   repeated TableViolationPolicy violation_policies = 1;
 }
+
+message GetQuotaStatesRequest {
+}
+
+message GetQuotaStatesResponse {
+  message TableQuotaSnapshot {
+    optional TableName table_name = 1;
+    optional SpaceQuotaSnapshot snapshot = 2;
+  }
+  message NamespaceQuotaSnapshot {
+    optional string namespace = 1;
+    optional SpaceQuotaSnapshot snapshot = 2;
+  }
+  repeated TableQuotaSnapshot table_snapshots = 1;
+  repeated NamespaceQuotaSnapshot ns_snapshots = 2;
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 0243934..92c1461 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -908,7 +908,7 @@ public class HMaster extends HRegionServer implements MasterServices {
       // Create the quota snapshot notifier
       spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
       spaceQuotaSnapshotNotifier.initialize(getClusterConnection());
-      this.quotaObserverChore = new QuotaObserverChore(this);
+      this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());
       // Start the chore to read the region FS space reports and act on them
       getChoreService().scheduleChore(quotaObserverChore);
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index eb2711c..1c4abc0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -62,7 +62,9 @@ import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
 import org.apache.hadoop.hbase.quotas.QuotaUtil;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
 import org.apache.hadoop.hbase.replication.ReplicationException;
 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -214,8 +216,12 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.NamespaceQuotaSnapshot;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.TableQuotaSnapshot;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.RegionSizes;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
@@ -2071,4 +2077,36 @@ public class MasterRpcServices extends RSRpcServices
       throw new ServiceException(e);
     }
   }
+
+  @Override
+  public GetQuotaStatesResponse getQuotaStates(
+      RpcController controller, GetQuotaStatesRequest request) throws ServiceException {
+    try {
+      master.checkInitialized();
+      QuotaObserverChore quotaChore = this.master.getQuotaObserverChore();
+      GetQuotaStatesResponse.Builder builder = GetQuotaStatesResponse.newBuilder();
+      if (null != quotaChore) {
+        // The "current" view of all tables with quotas
+        Map<TableName, SpaceQuotaSnapshot> tableSnapshots = quotaChore.getTableQuotaSnapshots();
+        for (Entry<TableName, SpaceQuotaSnapshot> entry : tableSnapshots.entrySet())
{
+          builder.addTableSnapshots(
+              TableQuotaSnapshot.newBuilder()
+                  .setTableName(ProtobufUtil.toProtoTableName(entry.getKey()))
+                  .setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(entry.getValue())).build());
+        }
+        // The "current" view of all namespaces with quotas
+        Map<String, SpaceQuotaSnapshot> nsSnapshots = quotaChore.getNamespaceQuotaSnapshots();
+        for (Entry<String, SpaceQuotaSnapshot> entry : nsSnapshots.entrySet()) {
+          builder.addNsSnapshots(
+              NamespaceQuotaSnapshot.newBuilder()
+                  .setNamespace(entry.getKey())
+                  .setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(entry.getValue())).build());
+        }
+        return builder.build();
+      }
+      return builder.build();
+    } catch (Exception e) {
+      throw new ServiceException(e);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java
index d055853..b5bc3d7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java
@@ -37,11 +37,14 @@ public class MetricsMaster {
   private static final Log LOG = LogFactory.getLog(MetricsMaster.class);
   private MetricsMasterSource masterSource;
   private MetricsMasterProcSource masterProcSource;
+  private MetricsMasterQuotaSource masterQuotaSource;
 
   public MetricsMaster(MetricsMasterWrapper masterWrapper) {
     masterSource = CompatibilitySingletonFactory.getInstance(MetricsMasterSourceFactory.class).create(masterWrapper);
     masterProcSource =
             CompatibilitySingletonFactory.getInstance(MetricsMasterProcSourceFactory.class).create(masterWrapper);
+    masterQuotaSource =
+            CompatibilitySingletonFactory.getInstance(MetricsMasterQuotaSourceFactory.class).create(masterWrapper);
   }
 
   // for unit-test usage
@@ -53,10 +56,49 @@ public class MetricsMaster {
     return masterProcSource;
   }
 
+  public MetricsMasterQuotaSource getMetricsQuotaSource() {
+    return masterQuotaSource;
+  }
+
   /**
    * @param inc How much to add to requests.
    */
   public void incrementRequests(final long inc) {
     masterSource.incRequests(inc);
   }
+
+  /**
+   * Sets the number of space quotas defined.
+   */
+  public void setNumSpaceQuotas(final long numSpaceQuotas) {
+    masterQuotaSource.updateNumSpaceQuotas(numSpaceQuotas);
+  }
+
+  /**
+   * Sets the number of table in violation of a space quota.
+   */
+  public void setNumTableInSpaceQuotaViolation(final long numTablesInViolation) {
+    masterQuotaSource.updateNumTablesInSpaceQuotaViolation(numTablesInViolation);
+  }
+
+  /**
+   * Sets the number of namespaces in violation of a space quota.
+   */
+  public void setNumNamespacesInSpaceQuotaViolation(final long numNamespacesInViolation)
{
+    masterQuotaSource.updateNumNamespacesInSpaceQuotaViolation(numNamespacesInViolation);
+  }
+
+  /**
+   * Sets the number of region size reports the master has seen.
+   */
+  public void setNumRegionSizeReports(final long numRegionReports) {
+    masterQuotaSource.updateNumCurrentSpaceQuotaRegionSizeReports(numRegionReports);
+  }
+
+  /**
+   * Sets the execution time of a period of the QuotaObserverChore.
+   */
+  public void incrementQuotaObserverTime(final long executionTime) {
+    masterQuotaSource.incrementSpaceQuotaObserverChoreTime(executionTime);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java
index 4cff28b..cbf7ba5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java
@@ -17,9 +17,18 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import java.util.AbstractMap.SimpleImmutableEntry;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 
 /**
@@ -134,4 +143,35 @@ public class MetricsMasterWrapperImpl implements MetricsMasterWrapper
{
     return master.getNumWALFiles();
   }
 
+  @Override
+  public Map<String,Entry<Long,Long>> getTableSpaceUtilization() {
+    QuotaObserverChore quotaChore = master.getQuotaObserverChore();
+    if (null == quotaChore) {
+      return Collections.emptyMap();
+    }
+    Map<TableName,SpaceQuotaSnapshot> tableSnapshots = quotaChore.getTableQuotaSnapshots();
+    Map<String,Entry<Long,Long>> convertedData = new HashMap<>();
+    for (Entry<TableName,SpaceQuotaSnapshot> entry : tableSnapshots.entrySet()) {
+      convertedData.put(entry.getKey().toString(), convertSnapshot(entry.getValue()));
+    }
+    return convertedData;
+  }
+
+  @Override
+  public Map<String,Entry<Long,Long>> getNamespaceSpaceUtilization() {
+    QuotaObserverChore quotaChore = master.getQuotaObserverChore();
+    if (null == quotaChore) {
+      return Collections.emptyMap();
+    }
+    Map<String,SpaceQuotaSnapshot> namespaceSnapshots = quotaChore.getNamespaceQuotaSnapshots();
+    Map<String,Entry<Long,Long>> convertedData = new HashMap<>();
+    for (Entry<String,SpaceQuotaSnapshot> entry : namespaceSnapshots.entrySet()) {
+      convertedData.put(entry.getKey(), convertSnapshot(entry.getValue()));
+    }
+    return convertedData;
+  }
+
+  Entry<Long,Long> convertSnapshot(SpaceQuotaSnapshot snapshot) {
+    return new SimpleImmutableEntry<Long,Long>(snapshot.getUsage(), snapshot.getLimit());
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 79effe7..94c5c87 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -18,12 +18,12 @@ package org.apache.hadoop.hbase.quotas;
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MetricsMaster;
 import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
 import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
@@ -78,6 +79,7 @@ public class QuotaObserverChore extends ScheduledChore {
   private final Connection conn;
   private final Configuration conf;
   private final MasterQuotaManager quotaManager;
+  private final MetricsMaster metrics;
   /*
    * Callback that changes in quota snapshots are passed to.
    */
@@ -87,7 +89,9 @@ public class QuotaObserverChore extends ScheduledChore {
    * Preserves the state of quota snapshots for tables and namespaces
    */
   private final Map<TableName,SpaceQuotaSnapshot> tableQuotaSnapshots;
+  private final Map<TableName,SpaceQuotaSnapshot> readOnlyTableQuotaSnapshots;
   private final Map<String,SpaceQuotaSnapshot> namespaceQuotaSnapshots;
+  private final Map<String,SpaceQuotaSnapshot> readOnlyNamespaceSnapshots;
 
   // The time, in millis, that region reports should be kept by the master
   private final long regionReportLifetimeMillis;
@@ -98,25 +102,28 @@ public class QuotaObserverChore extends ScheduledChore {
   private QuotaSnapshotStore<TableName> tableSnapshotStore;
   private QuotaSnapshotStore<String> namespaceSnapshotStore;
 
-  public QuotaObserverChore(HMaster master) {
+  public QuotaObserverChore(HMaster master, MetricsMaster metrics) {
     this(
         master.getConnection(), master.getConfiguration(),
         master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(),
-        master);
+        master, metrics);
   }
 
   QuotaObserverChore(
       Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier snapshotNotifier,
-      MasterQuotaManager quotaManager, Stoppable stopper) {
+      MasterQuotaManager quotaManager, Stoppable stopper, MetricsMaster metrics) {
     super(
         QuotaObserverChore.class.getSimpleName(), stopper, getPeriod(conf),
         getInitialDelay(conf), getTimeUnit(conf));
     this.conn = conn;
     this.conf = conf;
+    this.metrics = metrics;
     this.quotaManager = quotaManager;
     this.snapshotNotifier = Objects.requireNonNull(snapshotNotifier);
-    this.tableQuotaSnapshots = new HashMap<>();
-    this.namespaceQuotaSnapshots = new HashMap<>();
+    this.tableQuotaSnapshots = new ConcurrentHashMap<>();
+    this.readOnlyTableQuotaSnapshots = Collections.unmodifiableMap(tableQuotaSnapshots);
+    this.namespaceQuotaSnapshots = new ConcurrentHashMap<>();
+    this.readOnlyNamespaceSnapshots = Collections.unmodifiableMap(namespaceQuotaSnapshots);
     this.regionReportLifetimeMillis = conf.getLong(
         REGION_REPORT_RETENTION_DURATION_KEY, REGION_REPORT_RETENTION_DURATION_DEFAULT);
   }
@@ -127,7 +134,11 @@ public class QuotaObserverChore extends ScheduledChore {
       if (LOG.isTraceEnabled()) {
         LOG.trace("Refreshing space quotas in RegionServer");
       }
+      long start = System.nanoTime();
       _chore();
+      if (null != metrics) {
+        metrics.incrementQuotaObserverTime((System.nanoTime() - start) / 1_000_000);
+      }
     } catch (IOException e) {
       LOG.warn("Failed to process quota reports and update quota state. Will retry.", e);
     }
@@ -141,6 +152,12 @@ public class QuotaObserverChore extends ScheduledChore {
       LOG.trace("Found following tables with quotas: " + tablesWithQuotas);
     }
 
+    if (null != metrics) {
+      // Set the number of namespaces and tables with quotas defined
+      metrics.setNumSpaceQuotas(tablesWithQuotas.getTableQuotaTables().size()
+          + tablesWithQuotas.getNamespacesWithQuotas().size());
+    }
+
     // The current "view" of region space use. Used henceforth.
     final Map<HRegionInfo,Long> reportedRegionSpaceUse = quotaManager.snapshotRegionSizes();
     if (LOG.isTraceEnabled()) {
@@ -152,6 +169,10 @@ public class QuotaObserverChore extends ScheduledChore {
 
     // Create the stores to track table and namespace snapshots
     initializeSnapshotStores(reportedRegionSpaceUse);
+    // Report the number of (non-expired) region size reports
+    if (null != metrics) {
+      metrics.setNumRegionSizeReports(reportedRegionSpaceUse.size());
+    }
 
     // Filter out tables for which we don't have adequate regionspace reports yet.
     // Important that we do this after we instantiate the stores above
@@ -215,6 +236,7 @@ public class QuotaObserverChore extends ScheduledChore {
    * @param tablesWithTableQuotas The HBase tables which have quotas defined
    */
   void processTablesWithQuotas(final Set<TableName> tablesWithTableQuotas) throws IOException
{
+    long numTablesInViolation = 0L;
     for (TableName table : tablesWithTableQuotas) {
       final SpaceQuota spaceQuota = tableSnapshotStore.getSpaceQuota(table);
       if (null == spaceQuota) {
@@ -227,9 +249,18 @@ public class QuotaObserverChore extends ScheduledChore {
       final SpaceQuotaSnapshot currentSnapshot = tableSnapshotStore.getCurrentState(table);
       final SpaceQuotaSnapshot targetSnapshot = tableSnapshotStore.getTargetState(table,
spaceQuota);
       if (LOG.isTraceEnabled()) {
-        LOG.trace("Processing " + table + " with current=" + currentSnapshot + ", target="
+ targetSnapshot);
+        LOG.trace("Processing " + table + " with current=" + currentSnapshot + ", target="
+            + targetSnapshot);
       }
       updateTableQuota(table, currentSnapshot, targetSnapshot);
+
+      if (targetSnapshot.getQuotaStatus().isInViolation()) {
+        numTablesInViolation++;
+      }
+    }
+    // Report the number of tables in violation
+    if (null != metrics) {
+      metrics.setNumTableInSpaceQuotaViolation(numTablesInViolation);
     }
   }
 
@@ -246,6 +277,7 @@ public class QuotaObserverChore extends ScheduledChore {
   void processNamespacesWithQuotas(
       final Set<String> namespacesWithQuotas,
       final Multimap<String,TableName> tablesByNamespace) throws IOException {
+    long numNamespacesInViolation = 0L;
     for (String namespace : namespacesWithQuotas) {
       // Get the quota definition for the namespace
       final SpaceQuota spaceQuota = namespaceSnapshotStore.getSpaceQuota(namespace);
@@ -257,8 +289,22 @@ public class QuotaObserverChore extends ScheduledChore {
         continue;
       }
       final SpaceQuotaSnapshot currentSnapshot = namespaceSnapshotStore.getCurrentState(namespace);
-      final SpaceQuotaSnapshot targetSnapshot = namespaceSnapshotStore.getTargetState(namespace,
spaceQuota);
+      final SpaceQuotaSnapshot targetSnapshot = namespaceSnapshotStore.getTargetState(
+          namespace, spaceQuota);
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Processing " + namespace + " with current=" + currentSnapshot + ", target="
+            + targetSnapshot);
+      }
       updateNamespaceQuota(namespace, currentSnapshot, targetSnapshot, tablesByNamespace);
+
+      if (targetSnapshot.getQuotaStatus().isInViolation()) {
+        numNamespacesInViolation++;
+      }
+    }
+
+    // Report the number of namespaces in violation
+    if (null != metrics) {
+      metrics.setNumNamespacesInSpaceQuotaViolation(numNamespacesInViolation);
     }
   }
 
@@ -285,14 +331,16 @@ public class QuotaObserverChore extends ScheduledChore {
         }
       } else if (LOG.isDebugEnabled()) {
         // We're either moving into violation or changing violation policies
-        LOG.debug(table + " moving into violation of table space quota with policy of " +
targetStatus.getPolicy());
+        LOG.debug(table + " moving into violation of table space quota with policy of "
+            + targetStatus.getPolicy());
       }
 
       this.snapshotNotifier.transitionTable(table, targetSnapshot);
       // Update it in memory
       tableSnapshotStore.setCurrentState(table, targetSnapshot);
     } else if (LOG.isTraceEnabled()) {
-      // Policies are the same, so we have nothing to do except log this. Don't need to re-update
the quota table
+      // Policies are the same, so we have nothing to do except log this. Don't need to re-update
+      // the quota table
       if (!currentStatus.isInViolation()) {
         LOG.trace(table + " remains in observance of quota.");
       } else {
@@ -347,11 +395,14 @@ public class QuotaObserverChore extends ScheduledChore {
             }
           } else {
             // No table quota present or a table quota present that is not in violation
-            LOG.info(tableInNS + " moving into violation of namespace space quota with policy
" + targetStatus.getPolicy());
+            LOG.info(tableInNS + " moving into violation of namespace space quota with policy
"
+                + targetStatus.getPolicy());
             this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
           }
         }
       }
+      // Update the new state in memory for this namespace
+      namespaceSnapshotStore.setCurrentState(namespace, targetSnapshot);
     } else {
       // Policies are the same
       if (!targetStatus.isInViolation()) {
@@ -360,7 +411,8 @@ public class QuotaObserverChore extends ScheduledChore {
           LOG.trace(namespace + " remains in observance of quota.");
         }
       } else {
-        // Namespace quota is still in violation, need to enact if the table quota is not
taking priority.
+        // Namespace quota is still in violation, need to enact if the table quota is not
+        // taking priority.
         for (TableName tableInNS : tablesByNamespace.get(namespace)) {
           // Does a table policy exist
           if (tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation())
{
@@ -451,6 +503,22 @@ public class QuotaObserverChore extends ScheduledChore {
   }
 
   /**
+   * Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects
+   * for each HBase table with a quota.
+   */
+  public Map<TableName,SpaceQuotaSnapshot> getTableQuotaSnapshots() {
+    return readOnlyTableQuotaSnapshots;
+  }
+
+  /**
+   * Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects
+   * for each HBase namespace with a quota.
+   */
+  public Map<String,SpaceQuotaSnapshot> getNamespaceQuotaSnapshots() {
+    return readOnlyNamespaceSnapshots;
+  }
+
+  /**
    * Fetches the {@link SpaceQuotaSnapshot} for the given table.
    */
   SpaceQuotaSnapshot getTableQuotaSnapshot(TableName table) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
index 0f8a289..5c52601 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
@@ -42,10 +42,14 @@
   import="org.apache.hadoop.hbase.HConstants"
   import="org.apache.hadoop.hbase.master.HMaster"
   import="org.apache.hadoop.hbase.zookeeper.MetaTableLocator"
+  import="org.apache.hadoop.hbase.quotas.QuotaTableUtil"
+  import="org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot"
   import="org.apache.hadoop.hbase.util.Bytes"
   import="org.apache.hadoop.hbase.util.FSUtils"
   import="org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos"
   import="org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos"
+  import="org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas"
+  import="org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota"
   import="org.apache.hadoop.hbase.TableName"
   import="org.apache.hadoop.hbase.HColumnDescriptor"
   import="org.apache.hadoop.hbase.HBaseConfiguration"
@@ -87,6 +91,7 @@
   if (showFragmentation) {
       frags = FSUtils.getTableFragmentation(master);
   }
+  boolean quotasEnabled = conf.getBoolean("hbase.quota.enabled", false);
   String action = request.getParameter("action");
   String key = request.getParameter("key");
   String left = request.getParameter("left");
@@ -328,6 +333,60 @@ if ( fqtn != null ) {
       <td>How fragmented is the table. After a major compaction it is 0%.</td>
   </tr>
 <%  } %>
+<%
+  if (quotasEnabled) {
+    TableName tn = TableName.valueOf(fqtn);
+    SpaceQuotaSnapshot masterSnapshot = null;
+    Quotas quota = QuotaTableUtil.getTableQuota(master.getConnection(), tn);
+    if (null == quota || !quota.hasSpace()) {
+      quota = QuotaTableUtil.getNamespaceQuota(master.getConnection(), tn.getNamespaceAsString());
+      if (null != quota) {
+        masterSnapshot = QuotaTableUtil.getCurrentSnapshot(master.getConnection(), tn.getNamespaceAsString());
+      }
+    } else {
+      masterSnapshot = QuotaTableUtil.getCurrentSnapshot(master.getConnection(), tn);
+    }
+    if (null != quota && quota.hasSpace()) {
+      SpaceQuota spaceQuota = quota.getSpace();
+%>
+  <tr>
+    <td>Space Quota</td>
+    <td>
+      <table>
+        <tr>
+          <th>Property</th>
+          <th>Value</th>
+        </tr>
+        <tr>
+          <td>Limit</td>
+          <td><%= StringUtils.byteDesc(spaceQuota.getSoftLimit()) %></td>
+        </tr>
+        <tr>
+          <td>Policy</td>
+          <td><%= spaceQuota.getViolationPolicy() %></td>
+        </tr>
+<%
+      if (null != masterSnapshot) {
+%>
+        <tr>
+          <td>Usage</td>
+          <td><%= StringUtils.byteDesc(masterSnapshot.getUsage()) %></td>
+        </tr>
+        <tr>
+          <td>State</td>
+          <td><%= masterSnapshot.getQuotaStatus().isInViolation() ? "In Violation"
: "In Observance" %></td>
+        </tr>
+<%
+      }
+%>
+      </table>
+    </td>
+    <td>Information about a Space Quota on this table, if set.</td>
+  </tr>
+<%
+    }
+  }
+%>
 </table>
 <h2>Table Schema</h2>
 <table class="table table-striped">

http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java
index 02f3721..8c1138a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java
@@ -19,9 +19,14 @@ package org.apache.hadoop.hbase.master;
 
 import static org.junit.Assert.*;
 
+import java.util.AbstractMap.SimpleImmutableEntry;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
+import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Threads;
@@ -77,4 +82,16 @@ public class TestMasterMetricsWrapper {
     assertEquals(1, info.getNumDeadRegionServers());
     assertEquals(1, info.getNumWALFiles());
   }
+
+  @Test
+  public void testQuotaSnapshotConversion() {
+    MetricsMasterWrapperImpl info = new MetricsMasterWrapperImpl(
+        TEST_UTIL.getHBaseCluster().getMaster());
+    assertEquals(new SimpleImmutableEntry<Long,Long>(1024L, 2048L),
+        info.convertSnapshot(new SpaceQuotaSnapshot(
+            SpaceQuotaStatus.notInViolation(), 1024L, 2048L)));
+    assertEquals(new SimpleImmutableEntry<Long,Long>(4096L, 2048L),
+        info.convertSnapshot(new SpaceQuotaSnapshot(
+            new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 2048L)));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java
index 5a00aaf..38dbf66 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -33,6 +34,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.Waiter;
 import org.apache.hadoop.hbase.Waiter.Predicate;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -187,6 +189,87 @@ public class TestQuotaStatusRPCs {
     assertEquals(SpaceViolationPolicy.NO_INSERTS, policy);
   }
 
+  @Test
+  public void testQuotaStatusFromMaster() throws Exception {
+    final long sizeLimit = 1024L * 10L; // 10KB
+    final long tableSize = 1024L * 5; // 5KB
+    final long nsLimit = Long.MAX_VALUE;
+    final int numRegions = 10;
+    final TableName tn = helper.createTableWithRegions(numRegions);
+
+    // Define the quota
+    QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
+        tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS);
+    TEST_UTIL.getAdmin().setQuota(settings);
+    QuotaSettings nsSettings = QuotaSettingsFactory.limitNamespaceSpace(
+        tn.getNamespaceAsString(), nsLimit, SpaceViolationPolicy.NO_INSERTS);
+    TEST_UTIL.getAdmin().setQuota(nsSettings);
+
+    // Write at least `tableSize` data
+    helper.writeData(tn, tableSize);
+
+    final Connection conn = TEST_UTIL.getConnection();
+    // Make sure the master has a snapshot for our table
+    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>()
{
+      @Override
+      public boolean evaluate() throws Exception {
+        SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(conn, tn);
+        LOG.info("Table snapshot after initial ingest: " + snapshot);
+        if (null == snapshot) {
+          return false;
+        }
+        return snapshot.getLimit() == sizeLimit && snapshot.getUsage() > 0L;
+      }
+    });
+    final AtomicReference<Long> nsUsage = new AtomicReference<>();
+    // If we saw the table snapshot, we should also see the namespace snapshot
+    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000 * 1000, new Predicate<Exception>()
{
+      @Override
+      public boolean evaluate() throws Exception {
+        SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(
+            conn, tn.getNamespaceAsString());
+        LOG.debug("Namespace snapshot after initial ingest: " + snapshot);
+        if (null == snapshot) {
+          return false;
+        }
+        nsUsage.set(snapshot.getUsage());
+        return snapshot.getLimit() == nsLimit && snapshot.getUsage() > 0;
+      }
+    });
+
+    try {
+      helper.writeData(tn, tableSize * 2L);
+    } catch (SpaceLimitingException e) {
+      // Pass
+    }
+
+    // Wait for the status to move to violation
+    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>()
{
+      @Override
+      public boolean evaluate() throws Exception {
+        SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(conn, tn);
+        LOG.info("Table snapshot after second ingest: " + snapshot);
+        if (null == snapshot) {
+          return false;
+        }
+        return snapshot.getQuotaStatus().isInViolation();
+      }
+    });
+    // The namespace should still not be in violation, but have a larger usage than previously
+    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>()
{
+      @Override
+      public boolean evaluate() throws Exception {
+        SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(
+            conn, tn.getNamespaceAsString());
+        LOG.debug("Namespace snapshot after second ingest: " + snapshot);
+        if (null == snapshot) {
+          return false;
+        }
+        return snapshot.getUsage() > nsUsage.get() && !snapshot.getQuotaStatus().isInViolation();
+      }
+    });
+  }
+
   private int countRegionsForTable(TableName tn, Map<HRegionInfo,Long> regionSizes)
{
     int size = 0;
     for (HRegionInfo regionInfo : regionSizes.keySet()) {


Mime
View raw message