hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From x...@apache.org
Subject hadoop git commit: HDFS-9360. Storage type usage isn't updated properly after file deletion. Contributed by Ming Ma.
Date Thu, 05 Nov 2015 09:40:32 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 81f7e8af2 -> 891be3999


HDFS-9360. Storage type usage isn't updated properly after file deletion. Contributed by Ming
Ma.

(cherry picked from commit ea5bb483269b51a349c358b71f84904c76693a66)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/891be399
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/891be399
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/891be399

Branch: refs/heads/branch-2
Commit: 891be39993ea72964d7266cc6c97ebbb759dda9a
Parents: 81f7e8a
Author: Xiaoyu Yao <xyao@apache.org>
Authored: Thu Nov 5 01:30:13 2015 -0800
Committer: Xiaoyu Yao <xyao@apache.org>
Committed: Thu Nov 5 01:40:05 2015 -0800

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 ++
 .../hadoop/hdfs/server/namenode/INodeFile.java  |  4 +-
 .../java/org/apache/hadoop/hdfs/TestQuota.java  | 55 +++++++++++++++++++-
 3 files changed, 60 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/891be399/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6aa9432..a7d742e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1390,6 +1390,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-9357. NN UI renders icons of decommissioned DN incorrectly.
     (Surendra Singh Lilhore via wheat9)
 
+    HDFS-9360. Storage type usage isn't updated properly after file deletion.
+    (Ming Ma via xyao)
+
 Release 2.7.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/891be399/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index ce4ee9c..99faa19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -568,7 +568,9 @@ public class INodeFile extends INodeWithAdditionalFields
       byte blockStoragePolicyId, boolean useCache, int lastSnapshotId) {
     final QuotaCounts counts = new QuotaCounts.Builder().nameSpace(1).build();
 
-    final BlockStoragePolicy bsp = bsps.getPolicy(blockStoragePolicyId);
+    final BlockStoragePolicy bsp = (blockStoragePolicyId ==
+        BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) ? null :
+        bsps.getPolicy(blockStoragePolicyId);
     FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
     if (sf == null) {
       counts.add(storagespaceConsumed(bsp));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/891be399/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
index e68e4d6..d402685 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.tools.DFSAdmin;
 import org.apache.hadoop.hdfs.web.WebHdfsConstants;
@@ -573,7 +574,6 @@ public class TestQuota {
       c = dfs.getContentSummary(quotaDir20);
       assertEquals(c.getSpaceQuota(), 6 * fileSpace);
 
-
       // Create /nqdir0/qdir1/qdir21 and set its space quota to 2 * fileSpace
       final Path quotaDir21 = new Path("/nqdir0/qdir1/qdir21");
       assertTrue(dfs.mkdirs(quotaDir21));
@@ -785,6 +785,59 @@ public class TestQuota {
     }
   }
 
+  /**
+   * Test quota by storage type.
+   */
+  @Test
+  public void testQuotaByStorageType() throws Exception {
+    final Configuration conf = new HdfsConfiguration();
+    // set a smaller block size so that we can test with smaller
+    // diskspace quotas
+    conf.set(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, "512");
+    // Make it relinquish locks. When run serially, the result should
+    // be identical.
+    conf.setInt(DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY, 2);
+    final MiniDFSCluster cluster =
+        new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+    final FileSystem fs = cluster.getFileSystem();
+    assertTrue("Not a HDFS: " + fs.getUri(),
+        fs instanceof DistributedFileSystem);
+    final DistributedFileSystem dfs = (DistributedFileSystem) fs;
+
+    try {
+      int fileLen = 1024;
+      short replication = 3;
+      int fileSpace = fileLen * replication;
+
+      final Path quotaDir20 = new Path("/nqdir0/qdir1/qdir20");
+      assertTrue(dfs.mkdirs(quotaDir20));
+      dfs.setQuota(quotaDir20, HdfsConstants.QUOTA_DONT_SET, 6 * fileSpace);
+
+      // Verify DirectoryWithQuotaFeature's storage type usage
+      // is updated properly after deletion.
+      // File creation followed by deletion shouldn't change storage type
+      // usage regardless whether storage policy is set.
+      Path file = new Path(quotaDir20, "fileDir/file1");
+      DFSTestUtil.createFile(dfs, file, fileLen * 3, replication, 0);
+      dfs.delete(file, false);
+      dfs.setStoragePolicy(quotaDir20, HdfsConstants.HOT_STORAGE_POLICY_NAME);
+      dfs.setQuotaByStorageType(quotaDir20, StorageType.DEFAULT,
+          2 * fileSpace);
+      boolean hasException = false;
+      try {
+        DFSTestUtil.createFile(dfs, file, fileLen * 3, replication, 0);
+      } catch (QuotaByStorageTypeExceededException e) {
+        hasException = true;
+      }
+      assertTrue(hasException);
+      dfs.delete(file, false);
+      dfs.setQuotaByStorageType(quotaDir20, StorageType.DEFAULT,
+          6 * fileSpace);
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
   private static void checkContentSummary(final ContentSummary expected,
       final ContentSummary computed) {
     assertEquals(expected.toString(), computed.toString());


Mime
View raw message