hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From inigo...@apache.org
Subject [hadoop] 22/45: HDFS-14150. RBF: Quotas of the sub-cluster should be removed when removing the mount point. Contributed by Takanobu Asanuma.
Date Sun, 03 Mar 2019 18:37:01 GMT
This is an automated email from the ASF dual-hosted git repository.

inigoiri pushed a commit to branch HDFS-13891
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit 8493efae2e516d88283d3062d7d283186873c482
Author: Yiqun Lin <yqlin@apache.org>
AuthorDate: Wed Jan 9 17:18:43 2019 +0800

    HDFS-14150. RBF: Quotas of the sub-cluster should be removed when removing the mount point.
Contributed by Takanobu Asanuma.
---
 .../federation/router/RouterAdminServer.java       | 23 +++++++----
 .../src/main/resources/hdfs-rbf-default.xml        |  4 +-
 .../src/site/markdown/HDFSRouterFederation.md      |  4 +-
 .../server/federation/router/TestRouterQuota.java  | 48 +++++++++++++++++++++-
 4 files changed, 67 insertions(+), 12 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
index 5bb7751..18c19e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
@@ -250,23 +250,25 @@ public class RouterAdminServer extends AbstractService
 
     MountTable mountTable = request.getEntry();
     if (mountTable != null && router.isQuotaEnabled()) {
-      synchronizeQuota(mountTable);
+      synchronizeQuota(mountTable.getSourcePath(),
+          mountTable.getQuota().getQuota(),
+          mountTable.getQuota().getSpaceQuota());
     }
     return response;
   }
 
   /**
    * Synchronize the quota value across mount table and subclusters.
-   * @param mountTable Quota set in given mount table.
+   * @param path Source path in given mount table.
+   * @param nsQuota Name quota definition in given mount table.
+   * @param ssQuota Space quota definition in given mount table.
    * @throws IOException
    */
-  private void synchronizeQuota(MountTable mountTable) throws IOException {
-    String path = mountTable.getSourcePath();
-    long nsQuota = mountTable.getQuota().getQuota();
-    long ssQuota = mountTable.getQuota().getSpaceQuota();
-
-    if (nsQuota != HdfsConstants.QUOTA_DONT_SET
-        || ssQuota != HdfsConstants.QUOTA_DONT_SET) {
+  private void synchronizeQuota(String path, long nsQuota, long ssQuota)
+      throws IOException {
+    if (router.isQuotaEnabled() &&
+        (nsQuota != HdfsConstants.QUOTA_DONT_SET
+        || ssQuota != HdfsConstants.QUOTA_DONT_SET)) {
       HdfsFileStatus ret = this.router.getRpcServer().getFileInfo(path);
       if (ret != null) {
         this.router.getRpcServer().getQuotaModule().setQuota(path, nsQuota,
@@ -278,6 +280,9 @@ public class RouterAdminServer extends AbstractService
   @Override
   public RemoveMountTableEntryResponse removeMountTableEntry(
       RemoveMountTableEntryRequest request) throws IOException {
+    // clear sub-cluster's quota definition
+    synchronizeQuota(request.getSrcPath(), HdfsConstants.QUOTA_RESET,
+        HdfsConstants.QUOTA_RESET);
     return getMountTableStore().removeMountTableEntry(request);
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
index 72f6c2f..20ae778 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
@@ -447,7 +447,9 @@
     <name>dfs.federation.router.quota.enable</name>
     <value>false</value>
     <description>
-      Set to true to enable quota system in Router.
+      Set to true to enable quota system in Router. When it's enabled, setting
+      or clearing sub-cluster's quota directly is not recommended since Router
+      Admin server will override sub-cluster's quota with global quota.
     </description>
   </property>
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
index adc4383..959cd63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
@@ -143,6 +143,8 @@ For performance reasons, the Router caches the quota usage and updates
it period
 will be used for quota-verification during each WRITE RPC call invoked in RouterRPCSever.
See [HDFS Quotas Guide](../hadoop-hdfs/HdfsQuotaAdminGuide.html)
 for the quota detail.
 
+Note: When global quota is enabled, setting or clearing sub-cluster's quota directly is not
recommended since Router Admin server will override sub-cluster's quota with global quota.
+
 ### State Store
 The (logically centralized, but physically distributed) State Store maintains:
 
@@ -421,7 +423,7 @@ Global quota supported in federation.
 
 | Property | Default | Description|
 |:---- |:---- |:---- |
-| dfs.federation.router.quota.enable | `false` | If `true`, the quota system enabled in the
Router. |
+| dfs.federation.router.quota.enable | `false` | If `true`, the quota system enabled in the
Router. In that case, setting or clearing sub-cluster's quota directly is not recommended
since Router Admin server will override sub-cluster's quota with global quota.|
 | dfs.federation.router.quota-cache.update.interval | 60s | How often the Router updates
quota cache. This setting supports multiple time unit suffixes. If no suffix is specified
then milliseconds is assumed. |
 
 Metrics
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
index 6a29446..656b401 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
@@ -605,7 +605,7 @@ public class TestRouterQuota {
   @Test
   public void testQuotaRefreshWhenDestinationNotPresent() throws Exception {
     long nsQuota = 5;
-    long ssQuota = 3*BLOCK_SIZE;
+    long ssQuota = 3 * BLOCK_SIZE;
     final FileSystem nnFs = nnContext1.getFileSystem();
 
     // Add three mount tables:
@@ -709,4 +709,50 @@ public class TestRouterQuota {
     assertEquals(updatedSpace, cacheQuota2.getSpaceConsumed());
     assertEquals(updatedSpace, mountQuota2.getSpaceConsumed());
   }
+
+  @Test
+  public void testClearQuotaDefAfterRemovingMountTable() throws Exception {
+    long nsQuota = 5;
+    long ssQuota = 3 * BLOCK_SIZE;
+    final FileSystem nnFs = nnContext1.getFileSystem();
+
+    // Add one mount tables:
+    // /setdir --> ns0---testdir15
+    // Create destination directory
+    nnFs.mkdirs(new Path("/testdir15"));
+
+    MountTable mountTable = MountTable.newInstance("/setdir",
+        Collections.singletonMap("ns0", "/testdir15"));
+    mountTable.setQuota(new RouterQuotaUsage.Builder().quota(nsQuota)
+        .spaceQuota(ssQuota).build());
+    addMountTable(mountTable);
+
+    // Update router quota
+    RouterQuotaUpdateService updateService =
+        routerContext.getRouter().getQuotaCacheUpdateService();
+    updateService.periodicInvoke();
+
+    RouterQuotaManager quotaManager =
+        routerContext.getRouter().getQuotaManager();
+    ClientProtocol client = nnContext1.getClient().getNamenode();
+    QuotaUsage routerQuota = quotaManager.getQuotaUsage("/setdir");
+    QuotaUsage subClusterQuota = client.getQuotaUsage("/testdir15");
+
+    // Verify current quota definitions
+    assertEquals(nsQuota, routerQuota.getQuota());
+    assertEquals(ssQuota, routerQuota.getSpaceQuota());
+    assertEquals(nsQuota, subClusterQuota.getQuota());
+    assertEquals(ssQuota, subClusterQuota.getSpaceQuota());
+
+    // Remove mount table
+    removeMountTable("/setdir");
+    updateService.periodicInvoke();
+    routerQuota = quotaManager.getQuotaUsage("/setdir");
+    subClusterQuota = client.getQuotaUsage("/testdir15");
+
+    // Verify quota definitions are cleared after removing the mount table
+    assertNull(routerQuota);
+    assertEquals(HdfsConstants.QUOTA_RESET, subClusterQuota.getQuota());
+    assertEquals(HdfsConstants.QUOTA_RESET, subClusterQuota.getSpaceQuota());
+  }
 }
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message