hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [24/54] [abbrv] hbase git commit: HBASE-16998 Implement Master-side analysis of region space reports
Date Tue, 11 Apr 2017 17:31:29 GMT
http://git-wip-us.apache.org/repos/asf/hbase/blob/acf21536/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
new file mode 100644
index 0000000..98236c2
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -0,0 +1,596 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceNotFoundException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+
+/**
+ * Test class for {@link QuotaObserverChore} that uses a live HBase cluster.
+ */
+@Category(LargeTests.class)
+public class TestQuotaObserverChoreWithMiniCluster {
+  private static final Log LOG = LogFactory.getLog(TestQuotaObserverChoreWithMiniCluster.class);
+  private static final int SIZE_PER_VALUE = 256;
+  private static final String F1 = "f1";
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final AtomicLong COUNTER = new AtomicLong(0);
+  private static final long ONE_MEGABYTE = 1024L * 1024L;
+  private static final long DEFAULT_WAIT_MILLIS = 500;
+
+  @Rule
+  public TestName testName = new TestName();
+
+  private HMaster master;
+  private QuotaObserverChore chore;
+  private SpaceQuotaViolationNotifierForTest violationNotifier;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 1000);
+    conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 1000);
+    conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_DELAY_KEY, 1000);
+    conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_PERIOD_KEY, 1000);
+    conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
+    TEST_UTIL.startMiniCluster(1);
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Before
+  public void removeAllQuotas() throws Exception {
+    final Connection conn = TEST_UTIL.getConnection();
+    // Wait for the quota table to be created
+    if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) {
+      do {
+        LOG.debug("Quota table does not yet exist");
+        Thread.sleep(DEFAULT_WAIT_MILLIS);
+      } while (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME));
+    } else {
+      // Or, clean up any quotas from previous test runs.
+      QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration());
+      for (QuotaSettings quotaSettings : scanner) {
+        final String namespace = quotaSettings.getNamespace();
+        final TableName tableName = quotaSettings.getTableName();
+        if (null != namespace) {
+          LOG.debug("Deleting quota for namespace: " + namespace);
+          QuotaUtil.deleteNamespaceQuota(conn, namespace);
+        } else {
+          assert null != tableName;
+          LOG.debug("Deleting quota for table: "+ tableName);
+          QuotaUtil.deleteTableQuota(conn, tableName);
+        }
+      }
+    }
+
+    master = TEST_UTIL.getMiniHBaseCluster().getMaster();
+    violationNotifier =
+        (SpaceQuotaViolationNotifierForTest) master.getSpaceQuotaViolationNotifier();
+    violationNotifier.clearTableViolations();
+    chore = master.getQuotaObserverChore();
+  }
+
+  @Test
+  public void testTableViolatesQuota() throws Exception {
+    TableName tn = createTableWithRegions(10);
+
+    final long sizeLimit = 2L * ONE_MEGABYTE;
+    final SpaceViolationPolicy violationPolicy = SpaceViolationPolicy.NO_INSERTS;
+    QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, violationPolicy);
+    TEST_UTIL.getAdmin().setQuota(settings);
+
+    // Write more data than should be allowed
+    writeData(tn, 3L * ONE_MEGABYTE);
+
+    Map<TableName,SpaceViolationPolicy> violatedQuotas = violationNotifier.snapshotTablesInViolation();
+    while (violatedQuotas.isEmpty()) {
+      LOG.info("Found no violated quotas, sleeping and retrying. Current reports: "
+          + master.getMasterQuotaManager().snapshotRegionSizes());
+      try {
+        Thread.sleep(DEFAULT_WAIT_MILLIS);
+      } catch (InterruptedException e) {
+        LOG.debug("Interrupted while sleeping.", e);
+        Thread.currentThread().interrupt();
+      }
+      violatedQuotas = violationNotifier.snapshotTablesInViolation();
+    }
+
+    Entry<TableName,SpaceViolationPolicy> entry = Iterables.getOnlyElement(violatedQuotas.entrySet());
+    assertEquals(tn, entry.getKey());
+    assertEquals(violationPolicy, entry.getValue());
+  }
+
+  @Test
+  public void testNamespaceViolatesQuota() throws Exception {
+    final String namespace = testName.getMethodName();
+    final Admin admin = TEST_UTIL.getAdmin();
+    // Ensure the namespace exists
+    try {
+      admin.getNamespaceDescriptor(namespace);
+    } catch (NamespaceNotFoundException e) {
+      NamespaceDescriptor desc = NamespaceDescriptor.create(namespace).build();
+      admin.createNamespace(desc);
+    }
+
+    TableName tn1 = createTableWithRegions(namespace, 5);
+    TableName tn2 = createTableWithRegions(namespace, 5);
+    TableName tn3 = createTableWithRegions(namespace, 5);
+
+    final long sizeLimit = 5L * ONE_MEGABYTE;
+    final SpaceViolationPolicy violationPolicy = SpaceViolationPolicy.DISABLE;
+    QuotaSettings settings = QuotaSettingsFactory.limitNamespaceSpace(namespace, sizeLimit,
violationPolicy);
+    admin.setQuota(settings);
+
+    writeData(tn1, 2L * ONE_MEGABYTE);
+    admin.flush(tn1);
+    Map<TableName,SpaceViolationPolicy> violatedQuotas = violationNotifier.snapshotTablesInViolation();
+    for (int i = 0; i < 5; i++) {
+      // Check a few times to make sure we don't prematurely move to violation
+      assertEquals("Should not see any quota violations after writing 2MB of data", 0, violatedQuotas.size());
+      try {
+        Thread.sleep(DEFAULT_WAIT_MILLIS);
+      } catch (InterruptedException e) {
+        LOG.debug("Interrupted while sleeping." , e);
+      }
+      violatedQuotas = violationNotifier.snapshotTablesInViolation();
+    }
+
+    writeData(tn2, 2L * ONE_MEGABYTE);
+    admin.flush(tn2);
+    violatedQuotas = violationNotifier.snapshotTablesInViolation();
+    for (int i = 0; i < 5; i++) {
+      // Check a few times to make sure we don't prematurely move to violation
+      assertEquals("Should not see any quota violations after writing 4MB of data", 0,
+          violatedQuotas.size());
+      try {
+        Thread.sleep(DEFAULT_WAIT_MILLIS);
+      } catch (InterruptedException e) {
+        LOG.debug("Interrupted while sleeping." , e);
+      }
+      violatedQuotas = violationNotifier.snapshotTablesInViolation();
+    }
+
+    // Writing the final 2MB of data will push the namespace over the 5MB limit (6MB in total)
+    // and should push all three tables in the namespace into violation.
+    writeData(tn3, 2L * ONE_MEGABYTE);
+    admin.flush(tn3);
+    violatedQuotas = violationNotifier.snapshotTablesInViolation();
+    while (violatedQuotas.size() < 3) {
+      LOG.debug("Saw fewer violations than desired (expected 3): " + violatedQuotas
+          + ". Current reports: " + master.getMasterQuotaManager().snapshotRegionSizes());
+      try {
+        Thread.sleep(DEFAULT_WAIT_MILLIS);
+      } catch (InterruptedException e) {
+        LOG.debug("Interrupted while sleeping.", e);
+        Thread.currentThread().interrupt();
+      }
+      violatedQuotas = violationNotifier.snapshotTablesInViolation();
+    }
+
+    SpaceViolationPolicy vp1 = violatedQuotas.remove(tn1);
+    assertNotNull("tn1 should be in violation", vp1);
+    assertEquals(violationPolicy, vp1);
+    SpaceViolationPolicy vp2 = violatedQuotas.remove(tn2);
+    assertNotNull("tn2 should be in violation", vp2);
+    assertEquals(violationPolicy, vp2);
+    SpaceViolationPolicy vp3 = violatedQuotas.remove(tn3);
+    assertNotNull("tn3 should be in violation", vp3);
+    assertEquals(violationPolicy, vp3);
+    assertTrue("Unexpected additional quota violations: " + violatedQuotas, violatedQuotas.isEmpty());
+  }
+
+  @Test
+  public void testTableQuotaOverridesNamespaceQuota() throws Exception {
+    final String namespace = testName.getMethodName();
+    final Admin admin = TEST_UTIL.getAdmin();
+    // Ensure the namespace exists
+    try {
+      admin.getNamespaceDescriptor(namespace);
+    } catch (NamespaceNotFoundException e) {
+      NamespaceDescriptor desc = NamespaceDescriptor.create(namespace).build();
+      admin.createNamespace(desc);
+    }
+
+    TableName tn1 = createTableWithRegions(namespace, 5);
+    TableName tn2 = createTableWithRegions(namespace, 5);
+
+    final long namespaceSizeLimit = 3L * ONE_MEGABYTE;
+    final SpaceViolationPolicy namespaceViolationPolicy = SpaceViolationPolicy.DISABLE;
+    QuotaSettings namespaceSettings = QuotaSettingsFactory.limitNamespaceSpace(namespace,
+        namespaceSizeLimit, namespaceViolationPolicy);
+    admin.setQuota(namespaceSettings);
+
+    writeData(tn1, 2L * ONE_MEGABYTE);
+    admin.flush(tn1);
+    Map<TableName,SpaceViolationPolicy> violatedQuotas = violationNotifier.snapshotTablesInViolation();
+    for (int i = 0; i < 5; i++) {
+      // Check a few times to make sure we don't prematurely move to violation
+      assertEquals("Should not see any quota violations after writing 2MB of data", 0,
+          violatedQuotas.size());
+      try {
+        Thread.sleep(DEFAULT_WAIT_MILLIS);
+      } catch (InterruptedException e) {
+        LOG.debug("Interrupted while sleeping." , e);
+      }
+      violatedQuotas = violationNotifier.snapshotTablesInViolation();
+    }
+
+    writeData(tn2, 2L * ONE_MEGABYTE);
+    admin.flush(tn2);
+    violatedQuotas = violationNotifier.snapshotTablesInViolation();
+    while (violatedQuotas.size() < 2) {
+      LOG.debug("Saw fewer violations than desired (expected 2): " + violatedQuotas
+          + ". Current reports: " + master.getMasterQuotaManager().snapshotRegionSizes());
+      try {
+        Thread.sleep(DEFAULT_WAIT_MILLIS);
+      } catch (InterruptedException e) {
+        LOG.debug("Interrupted while sleeping.", e);
+        Thread.currentThread().interrupt();
+      }
+      violatedQuotas = violationNotifier.snapshotTablesInViolation();
+    }
+
+    SpaceViolationPolicy actualPolicyTN1 = violatedQuotas.get(tn1);
+    assertNotNull("Expected to see violation policy for tn1", actualPolicyTN1);
+    assertEquals(namespaceViolationPolicy, actualPolicyTN1);
+    SpaceViolationPolicy actualPolicyTN2 = violatedQuotas.get(tn2);
+    assertNotNull("Expected to see violation policy for tn2", actualPolicyTN2);
+    assertEquals(namespaceViolationPolicy, actualPolicyTN2);
+
+    // Override the namespace quota with a table quota
+    final long tableSizeLimit = ONE_MEGABYTE;
+    final SpaceViolationPolicy tableViolationPolicy = SpaceViolationPolicy.NO_INSERTS;
+    QuotaSettings tableSettings = QuotaSettingsFactory.limitTableSpace(tn1, tableSizeLimit,
+        tableViolationPolicy);
+    admin.setQuota(tableSettings);
+
+    // Keep checking for the table quota policy to override the namespace quota
+    while (true) {
+      violatedQuotas = violationNotifier.snapshotTablesInViolation();
+      SpaceViolationPolicy actualTableViolationPolicy = violatedQuotas.get(tn1);
+      assertNotNull("Violation policy should never be null", actualTableViolationPolicy);
+      if (tableViolationPolicy != actualTableViolationPolicy) {
+        LOG.debug("Saw unexpected table violation policy, waiting and re-checking.");
+        try {
+          Thread.sleep(DEFAULT_WAIT_MILLIS);
+        } catch (InterruptedException e) {
+          LOG.debug("Interrupted while sleeping");
+          Thread.currentThread().interrupt();
+        }
+        continue;
+      }
+      assertEquals(tableViolationPolicy, actualTableViolationPolicy);
+      break;
+    }
+
+    // This should not change with the introduction of the table quota for tn1
+    actualPolicyTN2 = violatedQuotas.get(tn2);
+    assertNotNull("Expected to see violation policy for tn2", actualPolicyTN2);
+    assertEquals(namespaceViolationPolicy, actualPolicyTN2);
+  }
+
+  @Test
+  public void testGetAllTablesWithQuotas() throws Exception {
+    final Multimap<TableName, QuotaSettings> quotas = createTablesWithSpaceQuotas();
+    Set<TableName> tablesWithQuotas = new HashSet<>();
+    Set<TableName> namespaceTablesWithQuotas = new HashSet<>();
+    // Partition the tables with quotas by table and ns quota
+    partitionTablesByQuotaTarget(quotas, tablesWithQuotas, namespaceTablesWithQuotas);
+
+    TablesWithQuotas tables = chore.fetchAllTablesWithQuotasDefined();
+    assertEquals("Found tables: " + tables, tablesWithQuotas, tables.getTableQuotaTables());
+    assertEquals("Found tables: " + tables, namespaceTablesWithQuotas, tables.getNamespaceQuotaTables());
+  }
+
+  @Test
+  public void testRpcQuotaTablesAreFiltered() throws Exception {
+    final Multimap<TableName, QuotaSettings> quotas = createTablesWithSpaceQuotas();
+    Set<TableName> tablesWithQuotas = new HashSet<>();
+    Set<TableName> namespaceTablesWithQuotas = new HashSet<>();
+    // Partition the tables with quotas by table and ns quota
+    partitionTablesByQuotaTarget(quotas, tablesWithQuotas, namespaceTablesWithQuotas);
+
+    TableName rpcQuotaTable = createTable();
+    TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory
+      .throttleTable(rpcQuotaTable, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
+
+    // The `rpcQuotaTable` should not be included in this Set
+    TablesWithQuotas tables = chore.fetchAllTablesWithQuotasDefined();
+    assertEquals("Found tables: " + tables, tablesWithQuotas, tables.getTableQuotaTables());
+    assertEquals("Found tables: " + tables, namespaceTablesWithQuotas, tables.getNamespaceQuotaTables());
+  }
+
+  @Test
+  public void testFilterRegions() throws Exception {
+    Map<TableName,Integer> mockReportedRegions = new HashMap<>();
+    // Can't mock because of primitive int as a return type -- Mockito
+    // can only handle an Integer.
+    TablesWithQuotas tables = new TablesWithQuotas(TEST_UTIL.getConnection(),
+        TEST_UTIL.getConfiguration()) {
+      @Override
+      int getNumReportedRegions(TableName table, QuotaViolationStore<TableName> tableStore)
{
+        Integer i = mockReportedRegions.get(table);
+        if (null == i) {
+          return 0;
+        }
+        return i;
+      }
+    };
+
+    // Create the tables
+    TableName tn1 = createTableWithRegions(20);
+    TableName tn2 = createTableWithRegions(20);
+    TableName tn3 = createTableWithRegions(20);
+
+    // Add them to the Tables with Quotas object
+    tables.addTableQuotaTable(tn1);
+    tables.addTableQuotaTable(tn2);
+    tables.addTableQuotaTable(tn3);
+
+    // Mock the number of regions reported
+    mockReportedRegions.put(tn1, 10); // 50%
+    mockReportedRegions.put(tn2, 19); // 95%
+    mockReportedRegions.put(tn3, 20); // 100%
+
+    // Argument is un-used
+    tables.filterInsufficientlyReportedTables(null);
+    // The default of 95% reported should prevent tn1 from appearing
+    assertEquals(new HashSet<>(Arrays.asList(tn2, tn3)), tables.getTableQuotaTables());
+  }
+
+  @Test
+  public void testFetchSpaceQuota() throws Exception {
+    Multimap<TableName,QuotaSettings> tables = createTablesWithSpaceQuotas();
+    // Can pass in an empty map, we're not consulting it.
+    chore.initializeViolationStores(Collections.emptyMap());
+    // All tables that were created should have a quota defined.
+    for (Entry<TableName,QuotaSettings> entry : tables.entries()) {
+      final TableName table = entry.getKey();
+      final QuotaSettings qs = entry.getValue();
+
+      assertTrue("QuotaSettings was an instance of " + qs.getClass(),
+          qs instanceof SpaceLimitSettings);
+
+      SpaceQuota spaceQuota = null;
+      if (null != qs.getTableName()) {
+        spaceQuota = chore.getTableViolationStore().getSpaceQuota(table);
+        assertNotNull("Could not find table space quota for " + table, spaceQuota);
+      } else if (null != qs.getNamespace()) {
+        spaceQuota = chore.getNamespaceViolationStore().getSpaceQuota(table.getNamespaceAsString());
+        assertNotNull("Could not find namespace space quota for " + table.getNamespaceAsString(),
spaceQuota);
+      } else {
+        fail("Expected table or namespace space quota");
+      }
+
+      final SpaceLimitSettings sls = (SpaceLimitSettings) qs;
+      assertEquals(sls.getProto().getQuota(), spaceQuota);
+    }
+
+    TableName tableWithoutQuota = createTable();
+    assertNull(chore.getTableViolationStore().getSpaceQuota(tableWithoutQuota));
+  }
+
+  //
+  // Helpers
+  //
+
+  void writeData(TableName tn, long sizeInBytes) throws IOException {
+    final Connection conn = TEST_UTIL.getConnection();
+    final Table table = conn.getTable(tn);
+    try {
+      List<Put> updates = new ArrayList<>();
+      long bytesToWrite = sizeInBytes;
+      long rowKeyId = 0L;
+      final StringBuilder sb = new StringBuilder();
+      final Random r = new Random();
+      while (bytesToWrite > 0L) {
+        sb.setLength(0);
+        sb.append(Long.toString(rowKeyId));
+        // Use the reverse counter as the rowKey to get even spread across all regions
+        Put p = new Put(Bytes.toBytes(sb.reverse().toString()));
+        byte[] value = new byte[SIZE_PER_VALUE];
+        r.nextBytes(value);
+        p.addColumn(Bytes.toBytes(F1), Bytes.toBytes("q1"), value);
+        updates.add(p);
+
+        // Batch 50K worth of updates
+        if (updates.size() > 50) {
+          table.put(updates);
+          updates.clear();
+        }
+
+        // Just count the value size, ignore the size of rowkey + column
+        bytesToWrite -= SIZE_PER_VALUE;
+        rowKeyId++;
+      }
+
+      // Write the final batch
+      if (!updates.isEmpty()) {
+        table.put(updates);
+      }
+
+      LOG.debug("Data was written to HBase");
+      // Push the data to disk.
+      TEST_UTIL.getAdmin().flush(tn);
+      LOG.debug("Data flushed to disk");
+    } finally {
+      table.close();
+    }
+  }
+
+  Multimap<TableName, QuotaSettings> createTablesWithSpaceQuotas() throws Exception
{
+    final Admin admin = TEST_UTIL.getAdmin();
+    final Multimap<TableName, QuotaSettings> tablesWithQuotas = HashMultimap.create();
+
+    final TableName tn1 = createTable();
+    final TableName tn2 = createTable();
+
+    NamespaceDescriptor nd = NamespaceDescriptor.create("ns" + COUNTER.getAndIncrement()).build();
+    admin.createNamespace(nd);
+    final TableName tn3 = createTableInNamespace(nd);
+    final TableName tn4 = createTableInNamespace(nd);
+    final TableName tn5 = createTableInNamespace(nd);
+
+    final long sizeLimit1 = 1024L * 1024L * 1024L * 1024L * 5L; // 5TB
+    final SpaceViolationPolicy violationPolicy1 = SpaceViolationPolicy.NO_WRITES;
+    QuotaSettings qs1 = QuotaSettingsFactory.limitTableSpace(tn1, sizeLimit1, violationPolicy1);
+    tablesWithQuotas.put(tn1, qs1);
+    admin.setQuota(qs1);
+
+    final long sizeLimit2 = 1024L * 1024L * 1024L * 200L; // 200GB
+    final SpaceViolationPolicy violationPolicy2 = SpaceViolationPolicy.NO_WRITES_COMPACTIONS;
+    QuotaSettings qs2 = QuotaSettingsFactory.limitTableSpace(tn2, sizeLimit2, violationPolicy2);
+    tablesWithQuotas.put(tn2, qs2);
+    admin.setQuota(qs2);
+
+    final long sizeLimit3 = 1024L * 1024L * 1024L * 1024L * 100L; // 100TB
+    final SpaceViolationPolicy violationPolicy3 = SpaceViolationPolicy.NO_INSERTS;
+    QuotaSettings qs3 = QuotaSettingsFactory.limitNamespaceSpace(nd.getName(), sizeLimit3,
violationPolicy3);
+    tablesWithQuotas.put(tn3, qs3);
+    tablesWithQuotas.put(tn4, qs3);
+    tablesWithQuotas.put(tn5, qs3);
+    admin.setQuota(qs3);
+
+    final long sizeLimit4 = 1024L * 1024L * 1024L * 5L; // 5GB
+    final SpaceViolationPolicy violationPolicy4 = SpaceViolationPolicy.NO_INSERTS;
+    QuotaSettings qs4 = QuotaSettingsFactory.limitTableSpace(tn5, sizeLimit4, violationPolicy4);
+    // Override the ns quota for tn5, import edge-case to catch table quota taking
+    // precedence over ns quota.
+    tablesWithQuotas.put(tn5, qs4);
+    admin.setQuota(qs4);
+
+    return tablesWithQuotas;
+  }
+
+  TableName createTable() throws Exception {
+    return createTableWithRegions(1);
+  }
+
+  TableName createTableWithRegions(int numRegions) throws Exception {
+    return createTableWithRegions(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR, numRegions);
+  }
+
+  TableName createTableWithRegions(String namespace, int numRegions) throws Exception {
+    final Admin admin = TEST_UTIL.getAdmin();
+    final TableName tn = TableName.valueOf(namespace, testName.getMethodName() + COUNTER.getAndIncrement());
+
+    // Delete the old table
+    if (admin.tableExists(tn)) {
+      admin.disableTable(tn);
+      admin.deleteTable(tn);
+    }
+
+    // Create the table
+    HTableDescriptor tableDesc = new HTableDescriptor(tn);
+    tableDesc.addFamily(new HColumnDescriptor(F1));
+    if (numRegions == 1) {
+      admin.createTable(tableDesc);
+    } else {
+      admin.createTable(tableDesc, Bytes.toBytes("0"), Bytes.toBytes("9"), numRegions);
+    }
+    return tn;
+  }
+
+  TableName createTableInNamespace(NamespaceDescriptor nd) throws Exception {
+    final Admin admin = TEST_UTIL.getAdmin();
+    final TableName tn = TableName.valueOf(nd.getName(),
+        testName.getMethodName() + COUNTER.getAndIncrement());
+
+    // Delete the old table
+    if (admin.tableExists(tn)) {
+      admin.disableTable(tn);
+      admin.deleteTable(tn);
+    }
+
+    // Create the table
+    HTableDescriptor tableDesc = new HTableDescriptor(tn);
+    tableDesc.addFamily(new HColumnDescriptor(F1));
+
+    admin.createTable(tableDesc);
+    return tn;
+  }
+
+  void partitionTablesByQuotaTarget(Multimap<TableName,QuotaSettings> quotas,
+      Set<TableName> tablesWithTableQuota, Set<TableName> tablesWithNamespaceQuota)
{
+    // Partition the tables with quotas by table and ns quota
+    for (Entry<TableName, QuotaSettings> entry : quotas.entries()) {
+      SpaceLimitSettings settings = (SpaceLimitSettings) entry.getValue();
+      TableName tn = entry.getKey();
+      if (null != settings.getTableName()) {
+        tablesWithTableQuota.add(tn);
+      }
+      if (null != settings.getNamespace()) {
+        tablesWithNamespaceQuota.add(tn);
+      }
+
+      if (null == settings.getTableName() && null == settings.getNamespace()) {
+        fail("Unexpected table name with null tableName and namespace: " + tn);
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/acf21536/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java
index 3b276ad..238c4c0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java
@@ -23,14 +23,11 @@ import static org.junit.Assert.assertEquals;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos;
 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
@@ -50,7 +47,6 @@ import org.junit.rules.TestName;
  */
 @Category({MasterTests.class, MediumTests.class})
 public class TestQuotaTableUtil {
-  private static final Log LOG = LogFactory.getLog(TestQuotaTableUtil.class);
 
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private Connection connection;

http://git-wip-us.apache.org/repos/asf/hbase/blob/acf21536/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableQuotaViolationStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableQuotaViolationStore.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableQuotaViolationStore.java
new file mode 100644
index 0000000..efc046b
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableQuotaViolationStore.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static com.google.common.collect.Iterables.size;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.quotas.QuotaViolationStore.ViolationState;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+/**
+ * Test class for {@link TableQuotaViolationStore}.
+ */
+@Category(SmallTests.class)
+public class TestTableQuotaViolationStore {
+  private static final long ONE_MEGABYTE = 1024L * 1024L;
+
+  private Connection conn;
+  private QuotaObserverChore chore;
+  private Map<HRegionInfo, Long> regionReports;
+  private TableQuotaViolationStore store;
+
+  @Before
+  public void setup() {
+    conn = mock(Connection.class);
+    chore = mock(QuotaObserverChore.class);
+    regionReports = new HashMap<>();
+    store = new TableQuotaViolationStore(conn, chore, regionReports);
+  }
+
+  @Test
+  public void testFilterRegionsByTable() throws Exception {
+    TableName tn1 = TableName.valueOf("foo");
+    TableName tn2 = TableName.valueOf("bar");
+    TableName tn3 = TableName.valueOf("ns", "foo");
+
+    assertEquals(0, size(store.filterBySubject(tn1)));
+
+    for (int i = 0; i < 5; i++) {
+      regionReports.put(new HRegionInfo(tn1, Bytes.toBytes(i), Bytes.toBytes(i+1)), 0L);
+    }
+    for (int i = 0; i < 3; i++) {
+      regionReports.put(new HRegionInfo(tn2, Bytes.toBytes(i), Bytes.toBytes(i+1)), 0L);
+    }
+    for (int i = 0; i < 10; i++) {
+      regionReports.put(new HRegionInfo(tn3, Bytes.toBytes(i), Bytes.toBytes(i+1)), 0L);
+    }
+    assertEquals(18, regionReports.size());
+    assertEquals(5, size(store.filterBySubject(tn1)));
+    assertEquals(3, size(store.filterBySubject(tn2)));
+    assertEquals(10, size(store.filterBySubject(tn3)));
+  }
+
+  @Test
+  public void testTargetViolationState() {
+    TableName tn1 = TableName.valueOf("violation1");
+    TableName tn2 = TableName.valueOf("observance1");
+    TableName tn3 = TableName.valueOf("observance2");
+    SpaceQuota quota = SpaceQuota.newBuilder()
+        .setSoftLimit(1024L * 1024L)
+        .setViolationPolicy(ProtobufUtil.toProtoViolationPolicy(SpaceViolationPolicy.DISABLE))
+        .build();
+
+    // Create some junk data to filter. Makes sure it's so large that it would
+    // immediately violate the quota.
+    for (int i = 0; i < 3; i++) {
+      regionReports.put(new HRegionInfo(tn2, Bytes.toBytes(i), Bytes.toBytes(i + 1)),
+          5L * ONE_MEGABYTE);
+      regionReports.put(new HRegionInfo(tn3, Bytes.toBytes(i), Bytes.toBytes(i + 1)),
+          5L * ONE_MEGABYTE);
+    }
+
+    regionReports.put(new HRegionInfo(tn1, Bytes.toBytes(0), Bytes.toBytes(1)), 1024L * 512L);
+    regionReports.put(new HRegionInfo(tn1, Bytes.toBytes(1), Bytes.toBytes(2)), 1024L * 256L);
+
+    // Below the quota
+    assertEquals(ViolationState.IN_OBSERVANCE, store.getTargetState(tn1, quota));
+
+    regionReports.put(new HRegionInfo(tn1, Bytes.toBytes(2), Bytes.toBytes(3)), 1024L * 256L);
+
+    // Equal to the quota is still in observance
+    assertEquals(ViolationState.IN_OBSERVANCE, store.getTargetState(tn1, quota));
+
+    regionReports.put(new HRegionInfo(tn1, Bytes.toBytes(3), Bytes.toBytes(4)), 1024L);
+
+    // Exceeds the quota, should be in violation
+    assertEquals(ViolationState.IN_VIOLATION, store.getTargetState(tn1, quota));
+  }
+
+  @Test
+  public void testGetSpaceQuota() throws Exception {
+    TableQuotaViolationStore mockStore = mock(TableQuotaViolationStore.class);
+    when(mockStore.getSpaceQuota(any(TableName.class))).thenCallRealMethod();
+
+    Quotas quotaWithSpace = Quotas.newBuilder().setSpace(
+        SpaceQuota.newBuilder()
+            .setSoftLimit(1024L)
+            .setViolationPolicy(QuotaProtos.SpaceViolationPolicy.DISABLE)
+            .build())
+        .build();
+    Quotas quotaWithoutSpace = Quotas.newBuilder().build();
+
+    AtomicReference<Quotas> quotaRef = new AtomicReference<>();
+    when(mockStore.getQuotaForTable(any(TableName.class))).then(new Answer<Quotas>()
{
+      @Override
+      public Quotas answer(InvocationOnMock invocation) throws Throwable {
+        return quotaRef.get();
+      }
+    });
+
+    quotaRef.set(quotaWithSpace);
+    assertEquals(quotaWithSpace.getSpace(), mockStore.getSpaceQuota(TableName.valueOf("foo")));
+    quotaRef.set(quotaWithoutSpace);
+    assertNull(mockStore.getSpaceQuota(TableName.valueOf("foo")));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/acf21536/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTablesWithQuotas.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTablesWithQuotas.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTablesWithQuotas.java
new file mode 100644
index 0000000..bb8d5cd
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTablesWithQuotas.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.collect.Multimap;
+
+/**
+ * Non-HBase cluster unit tests for {@link TablesWithQuotas}.
+ */
+@Category(SmallTests.class)
+public class TestTablesWithQuotas {
+  private Connection conn;
+  private Configuration conf;
+
+  @Before
+  public void setup() throws Exception {
+    conn = mock(Connection.class);
+    conf = HBaseConfiguration.create();
+  }
+
+  @Test
+  public void testImmutableGetters() {
+    Set<TableName> tablesWithTableQuotas = new HashSet<>();
+    Set<TableName> tablesWithNamespaceQuotas = new HashSet<>();
+    final TablesWithQuotas tables = new TablesWithQuotas(conn, conf);
+    for (int i = 0; i < 5; i++) {
+      TableName tn = TableName.valueOf("tn" + i);
+      tablesWithTableQuotas.add(tn);
+      tables.addTableQuotaTable(tn);
+    }
+    for (int i = 0; i < 3; i++) {
+      TableName tn = TableName.valueOf("tn_ns" + i);
+      tablesWithNamespaceQuotas.add(tn);
+      tables.addNamespaceQuotaTable(tn);
+    }
+    Set<TableName> actualTableQuotaTables = tables.getTableQuotaTables();
+    Set<TableName> actualNamespaceQuotaTables = tables.getNamespaceQuotaTables();
+    assertEquals(tablesWithTableQuotas, actualTableQuotaTables);
+    assertEquals(tablesWithNamespaceQuotas, actualNamespaceQuotaTables);
+    try {
+      actualTableQuotaTables.add(null);
+      fail("Should not be able to add an element");
+    } catch (UnsupportedOperationException e) {
+      // pass
+    }
+    try {
+      actualNamespaceQuotaTables.add(null);
+      fail("Should not be able to add an element");
+    } catch (UnsupportedOperationException e) {
+      // pass
+    }
+  }
+
+  @Test
+  public void testInsufficientlyReportedTableFiltering() throws Exception {
+    final Map<TableName,Integer> reportedRegions = new HashMap<>();
+    final Map<TableName,Integer> actualRegions = new HashMap<>();
+    final Configuration conf = HBaseConfiguration.create();
+    conf.setDouble(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_REPORT_PERCENT_KEY, 0.95);
+
+    TableName tooFewRegionsTable = TableName.valueOf("tn1");
+    TableName sufficientRegionsTable = TableName.valueOf("tn2");
+    TableName tooFewRegionsNamespaceTable = TableName.valueOf("ns1", "tn2");
+    TableName sufficientRegionsNamespaceTable = TableName.valueOf("ns1", "tn2");
+    final TablesWithQuotas tables = new TablesWithQuotas(conn, conf) {
+      @Override
+      Configuration getConfiguration() {
+        return conf;
+      }
+
+      @Override
+      int getNumRegions(TableName tableName) {
+        return actualRegions.get(tableName);
+      }
+
+      @Override
+      int getNumReportedRegions(TableName table, QuotaViolationStore<TableName> tableStore)
{
+        return reportedRegions.get(table);
+      }
+    };
+    tables.addTableQuotaTable(tooFewRegionsTable);
+    tables.addTableQuotaTable(sufficientRegionsTable);
+    tables.addNamespaceQuotaTable(tooFewRegionsNamespaceTable);
+    tables.addNamespaceQuotaTable(sufficientRegionsNamespaceTable);
+
+    reportedRegions.put(tooFewRegionsTable, 5);
+    actualRegions.put(tooFewRegionsTable, 10);
+    reportedRegions.put(sufficientRegionsTable, 19);
+    actualRegions.put(sufficientRegionsTable, 20);
+    reportedRegions.put(tooFewRegionsNamespaceTable, 9);
+    actualRegions.put(tooFewRegionsNamespaceTable, 10);
+    reportedRegions.put(sufficientRegionsNamespaceTable, 98);
+    actualRegions.put(sufficientRegionsNamespaceTable, 100);
+
+    // Unused argument
+    tables.filterInsufficientlyReportedTables(null);
+    Set<TableName> filteredTablesWithTableQuotas = tables.getTableQuotaTables();
+    assertEquals(Collections.singleton(sufficientRegionsTable), filteredTablesWithTableQuotas);
+    Set<TableName> filteredTablesWithNamespaceQutoas = tables.getNamespaceQuotaTables();
+    assertEquals(Collections.singleton(sufficientRegionsNamespaceTable), filteredTablesWithNamespaceQutoas);
+  }
+
+  @Test
+  public void testGetTablesByNamespace() {
+    final TablesWithQuotas tables = new TablesWithQuotas(conn, conf);
+    tables.addTableQuotaTable(TableName.valueOf("ignored1"));
+    tables.addTableQuotaTable(TableName.valueOf("ignored2"));
+    tables.addNamespaceQuotaTable(TableName.valueOf("ns1", "t1"));
+    tables.addNamespaceQuotaTable(TableName.valueOf("ns1", "t2"));
+    tables.addNamespaceQuotaTable(TableName.valueOf("ns1", "t3"));
+    tables.addNamespaceQuotaTable(TableName.valueOf("ns2", "t1"));
+    tables.addNamespaceQuotaTable(TableName.valueOf("ns2", "t2"));
+
+    Multimap<String,TableName> tablesByNamespace = tables.getTablesByNamespace();
+    Collection<TableName> tablesInNs = tablesByNamespace.get("ns1");
+    assertEquals(3, tablesInNs.size());
+    assertTrue("Unexpected results for ns1: " + tablesInNs,
+        tablesInNs.containsAll(Arrays.asList(
+            TableName.valueOf("ns1", "t1"),
+            TableName.valueOf("ns1", "t2"),
+            TableName.valueOf("ns1", "t3"))));
+    tablesInNs = tablesByNamespace.get("ns2");
+    assertEquals(2, tablesInNs.size());
+    assertTrue("Unexpected results for ns2: " + tablesInNs,
+        tablesInNs.containsAll(Arrays.asList(
+            TableName.valueOf("ns2", "t1"),
+            TableName.valueOf("ns2", "t2"))));
+  }
+
+  @Test
+  public void testFilteringMissingTables() throws Exception {
+    final TableName missingTable = TableName.valueOf("doesNotExist");
+    // Set up Admin to return null (match the implementation)
+    Admin admin = mock(Admin.class);
+    when(conn.getAdmin()).thenReturn(admin);
+    when(admin.getTableRegions(missingTable)).thenReturn(null);
+
+    QuotaObserverChore chore = mock(QuotaObserverChore.class);
+    Map<HRegionInfo,Long> regionUsage = new HashMap<>();
+    TableQuotaViolationStore store = new TableQuotaViolationStore(conn, chore, regionUsage);
+
+    // A super dirty hack to verify that, after getting no regions for our table,
+    // we bail out and start processing the next element (which there is none).
+    final TablesWithQuotas tables = new TablesWithQuotas(conn, conf) {
+      @Override
+      int getNumReportedRegions(TableName table, QuotaViolationStore<TableName> tableStore)
{
+        throw new RuntimeException("Should should not reach here");
+      }
+    };
+    tables.addTableQuotaTable(missingTable);
+
+    tables.filterInsufficientlyReportedTables(store);
+
+    final Set<TableName> tablesWithQuotas = tables.getTableQuotaTables();
+    assertTrue(
+        "Expected to find no tables, but found " + tablesWithQuotas, tablesWithQuotas.isEmpty());
+  }
+}


Mime
View raw message