hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-13593 Quota support for namespace should take snapshot restore and clone into account (Ashish Singhi)
Date Sat, 09 May 2015 18:08:48 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 4c4d69490 -> ec357ce83


HBASE-13593 Quota support for namespace should take snapshot restore and clone into account
(Ashish Singhi)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ec357ce8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ec357ce8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ec357ce8

Branch: refs/heads/branch-1.1
Commit: ec357ce83eb6496404b6863c02a21b75d9aec0fd
Parents: 4c4d694
Author: tedyu <yuzhihong@gmail.com>
Authored: Sat May 9 11:09:38 2015 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Sat May 9 11:09:38 2015 -0700

----------------------------------------------------------------------
 .../hbase/master/snapshot/SnapshotManager.java  |  37 +++++-
 .../hbase/namespace/NamespaceAuditor.java       |  14 ++
 .../hbase/namespace/NamespaceStateManager.java  |  27 ++++
 .../hadoop/hbase/quotas/MasterQuotaManager.java |   6 +
 .../hbase/namespace/TestNamespaceAuditor.java   | 129 +++++++++++++++++++
 5 files changed, 211 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/ec357ce8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index 59446cb..853e3cd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -734,7 +734,16 @@ public class SnapshotManager extends MasterProcedureManager implements
Stoppable
       if (cpHost != null) {
         cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
       }
-      restoreSnapshot(snapshot, snapshotTableDesc);
+      try {
+        // Table already exist. Check and update the region quota for this table namespace
+        checkAndUpdateNamespaceRegionQuota(manifest, tableName);
+        restoreSnapshot(snapshot, snapshotTableDesc);
+      } catch (IOException e) {
+        this.master.getMasterQuotaManager().removeTableFromNamespaceQuota(tableName);
+        LOG.error("Exception occurred while restoring the snapshot " + snapshot.getName()
+            + " as table " + tableName.getNameAsString(), e);
+        throw e;
+      }
       LOG.info("Restore snapshot=" + snapshot.getName() + " as table=" + tableName);
 
       if (cpHost != null) {
@@ -745,7 +754,15 @@ public class SnapshotManager extends MasterProcedureManager implements
Stoppable
       if (cpHost != null) {
         cpHost.preCloneSnapshot(reqSnapshot, htd);
       }
-      cloneSnapshot(snapshot, htd);
+      try {
+        checkAndUpdateNamespaceQuota(manifest, tableName);
+        cloneSnapshot(snapshot, htd);
+      } catch (IOException e) {
+        this.master.getMasterQuotaManager().removeTableFromNamespaceQuota(tableName);
+        LOG.error("Exception occurred while cloning the snapshot " + snapshot.getName()
+            + " as table " + tableName.getNameAsString(), e);
+        throw e;
+      }
       LOG.info("Clone snapshot=" + snapshot.getName() + " as table=" + tableName);
 
       if (cpHost != null) {
@@ -753,6 +770,22 @@ public class SnapshotManager extends MasterProcedureManager implements
Stoppable
       }
     }
   }
+  
+  private void checkAndUpdateNamespaceQuota(SnapshotManifest manifest, TableName tableName)
+      throws IOException {
+    if (this.master.getMasterQuotaManager().isQuotaEnabled()) {
+      this.master.getMasterQuotaManager().checkNamespaceTableAndRegionQuota(tableName,
+        manifest.getRegionManifestsMap().size());
+    }
+  }
+
+  private void checkAndUpdateNamespaceRegionQuota(SnapshotManifest manifest, TableName tableName)
+      throws IOException {
+    if (this.master.getMasterQuotaManager().isQuotaEnabled()) {
+      this.master.getMasterQuotaManager().checkAndUpdateNamespaceRegionQuota(tableName,
+        manifest.getRegionManifestsMap().size());
+    }
+  }
 
   /**
    * Restore the specified snapshot.

http://git-wip-us.apache.org/repos/asf/hbase/blob/ec357ce8/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java
index 2ba771d..ba9177f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java
@@ -68,6 +68,20 @@ public class NamespaceAuditor {
       checkTableTypeAndThrowException(tName);
     }
   }
+  
+  /**
+   * Check and update region count quota for an existing table.
+   * @param tName - table name for which region count to be updated.
+   * @param regions - Number of regions that will be added.
+   * @throws IOException Signals that an I/O exception has occurred.
+   */
+  public void checkQuotaToUpdateRegion(TableName tName, int regions) throws IOException {
+    if (stateManager.isInitialized()) {
+      stateManager.checkAndUpdateNamespaceRegionCount(tName, regions);
+    } else {
+      checkTableTypeAndThrowException(tName);
+    }
+  }
 
   private void checkTableTypeAndThrowException(TableName name) throws IOException {
     if (name.isSystemTable()) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/ec357ce8/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
index b2e47c3..41aa11d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
@@ -108,6 +108,33 @@ class NamespaceStateManager extends ZooKeeperListener {
     }
     return true;
   }
+  
+  /**
+   * Check and update region count for an existing table. To handle scenarios like restore
snapshot
+   * @param TableName name of the table for region count needs to be checked and updated
+   * @param incr count of regions
+   * @throws QuotaExceededException if quota exceeds for the number of regions allowed in
a
+   *           namespace
+   * @throws IOException Signals that an I/O exception has occurred.
+   */
+  synchronized void checkAndUpdateNamespaceRegionCount(TableName name, int incr) 
+      throws IOException {
+    String namespace = name.getNamespaceAsString();
+    NamespaceDescriptor nspdesc = getNamespaceDescriptor(namespace);
+    if (nspdesc != null) {
+      NamespaceTableAndRegionInfo currentStatus = getState(namespace);
+      int regionCountOfTable = currentStatus.getRegionCountOfTable(name);
+      if ((currentStatus.getRegionCount() - regionCountOfTable + incr) > TableNamespaceManager
+          .getMaxRegions(nspdesc)) {
+        throw new QuotaExceededException("The table " + name.getNameAsString()
+            + " region count cannot be updated as it would exceed maximum number "
+            + "of regions allowed in the namespace.  The total number of regions permitted
is "
+            + TableNamespaceManager.getMaxRegions(nspdesc));
+      }
+      currentStatus.removeTable(name);
+      currentStatus.addTable(name, incr);
+    }
+  }
 
   private NamespaceDescriptor getNamespaceDescriptor(String namespaceAsString) {
     try {

http://git-wip-us.apache.org/repos/asf/hbase/blob/ec357ce8/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index 3c1a4ad..61d2ec0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -326,6 +326,12 @@ public class MasterQuotaManager implements RegionStateListener {
       namespaceQuotaManager.checkQuotaToCreateTable(tName, regions);
     }
   }
+  
+  public void checkAndUpdateNamespaceRegionQuota(TableName tName, int regions) throws IOException
{
+    if (enabled) {
+      namespaceQuotaManager.checkQuotaToUpdateRegion(tName, regions);
+    }
+  }
 
   public void onRegionMerged(HRegionInfo hri) throws IOException {
     if (enabled) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/ec357ce8/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
index 05a0294..22fe916 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.RegionLocator;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
@@ -68,6 +69,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -679,4 +681,131 @@ public class TestNamespaceAuditor {
     ADMIN.createTable(tableDescOne);
     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
   }
+  
+  @Test(expected = QuotaExceededException.class, timeout = 30000)
+  public void testCloneSnapshotQuotaExceed() throws Exception {
+    String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
+    NamespaceDescriptor nspDesc =
+        NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES,
"1")
+            .build();
+    ADMIN.createNamespace(nspDesc);
+    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
+    TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
+    TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
+    HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
+    ADMIN.createTable(tableDescOne);
+    String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
+    ADMIN.snapshot(snapshot, tableName);
+    ADMIN.cloneSnapshot(snapshot, cloneTableName);
+    ADMIN.deleteSnapshot(snapshot);
+  }
+
+  @Test(timeout = 180000)
+  public void testCloneSnapshot() throws Exception {
+    String nsp = prefix + "_testCloneSnapshot";
+    NamespaceDescriptor nspDesc =
+        NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES,
"2")
+            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
+    ADMIN.createNamespace(nspDesc);
+    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
+    TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
+    TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
+    HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
+
+    ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
+    String snapshot = "snapshot_testCloneSnapshot";
+    ADMIN.snapshot(snapshot, tableName);
+    ADMIN.cloneSnapshot(snapshot, cloneTableName);
+
+    int tableLength;
+    try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
+      tableLength = locator.getStartKeys().length;
+    }
+    assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
+
+    try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName))
{
+      tableLength = locator.getStartKeys().length;
+    }
+    assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
+
+    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
+    assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
+    assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
+
+    ADMIN.deleteSnapshot(snapshot);
+  }
+
+  @Test(timeout = 180000)
+  public void testRestoreSnapshot() throws Exception {
+    String nsp = prefix + "_testRestoreSnapshot";
+    NamespaceDescriptor nspDesc =
+        NamespaceDescriptor.create(nsp)
+            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
+    ADMIN.createNamespace(nspDesc);
+    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
+    TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
+    HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
+    ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
+
+    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
+    assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
+
+    String snapshot = "snapshot_testRestoreSnapshot";
+    ADMIN.snapshot(snapshot, tableName1);
+
+    List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
+    Collections.sort(regions);
+
+    ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
+    Thread.sleep(2000);
+    assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
+
+    ADMIN.disableTable(tableName1);
+    ADMIN.restoreSnapshot(snapshot);
+
+    assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
+
+    ADMIN.enableTable(tableName1);
+    ADMIN.deleteSnapshot(snapshot);
+  }
+
+  @Test(timeout = 180000)
+  public void testRestoreSnapshotQuotaExceed() throws Exception {
+    String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
+    NamespaceDescriptor nspDesc =
+        NamespaceDescriptor.create(nsp)
+            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
+    ADMIN.createNamespace(nspDesc);
+    NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
+    assertNotNull("Namespace descriptor found null.", ndesc);
+    TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
+    HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
+    ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
+
+    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
+    assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
+
+    String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
+    ADMIN.snapshot(snapshot, tableName1);
+
+    List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
+    Collections.sort(regions);
+
+    ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
+    Thread.sleep(2000);
+    assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
+
+    ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2");
+    ADMIN.modifyNamespace(ndesc);
+
+    ADMIN.disableTable(tableName1);
+    try {
+      ADMIN.restoreSnapshot(snapshot);
+      fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
+          + " wraps IOException into RestoreSnapshotException");
+    } catch (RestoreSnapshotException ignore) {
+    }
+    ADMIN.enableTable(tableName1);
+    ADMIN.deleteSnapshot(snapshot);
+  }
 }


Mime
View raw message