hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kih...@apache.org
Subject [1/2] hadoop git commit: HDFS-8898. Create API and command-line argument to get quota and quota usage without detailed content summary. Contributed by Ming Ma.
Date Fri, 22 Jan 2016 15:21:46 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 493275b27 -> b5db1d441


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5db1d44/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
index d09b8b5..aee3c46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
@@ -21,6 +21,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -33,6 +34,7 @@ import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.QuotaUsage;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
@@ -133,6 +135,7 @@ public class TestQuota {
       
       // 4: count -q /test
       ContentSummary c = dfs.getContentSummary(parent);
+      compareQuotaUsage(c, dfs, parent);
       assertEquals(c.getFileCount()+c.getDirectoryCount(), 3);
       assertEquals(c.getQuota(), 3);
       assertEquals(c.getSpaceConsumed(), fileLen*replication);
@@ -140,10 +143,12 @@ public class TestQuota {
       
       // 5: count -q /test/data0
       c = dfs.getContentSummary(childDir0);
+      compareQuotaUsage(c, dfs, childDir0);
       assertEquals(c.getFileCount()+c.getDirectoryCount(), 1);
       assertEquals(c.getQuota(), -1);
       // check disk space consumed
       c = dfs.getContentSummary(parent);
+      compareQuotaUsage(c, dfs, parent);
       assertEquals(c.getSpaceConsumed(), fileLen*replication);
 
       // 6: create a directory /test/data1
@@ -171,12 +176,14 @@ public class TestQuota {
       // 8: clear quota /test
       runCommand(admin, new String[]{"-clrQuota", parent.toString()}, false);
       c = dfs.getContentSummary(parent);
+      compareQuotaUsage(c, dfs, parent);
       assertEquals(c.getQuota(), -1);
       assertEquals(c.getSpaceQuota(), spaceQuota);
       
       // 9: clear quota /test/data0
       runCommand(admin, new String[]{"-clrQuota", childDir0.toString()}, false);
       c = dfs.getContentSummary(childDir0);
+      compareQuotaUsage(c, dfs, childDir0);
       assertEquals(c.getQuota(), -1);
       
       // 10: create a file /test/datafile1
@@ -197,6 +204,7 @@ public class TestQuota {
       // 9.s: clear diskspace quota
       runCommand(admin, false, "-clrSpaceQuota", parent.toString());
       c = dfs.getContentSummary(parent);
+      compareQuotaUsage(c, dfs, parent);
       assertEquals(c.getQuota(), -1);
       assertEquals(c.getSpaceQuota(), -1);       
       
@@ -223,6 +231,7 @@ public class TestQuota {
       }
       assertTrue(hasException);
       c = dfs.getContentSummary(childDir0);
+      compareQuotaUsage(c, dfs, childDir0);
       assertEquals(c.getDirectoryCount()+c.getFileCount(), 1);
       assertEquals(c.getQuota(), 1);
       
@@ -361,7 +370,7 @@ public class TestQuota {
       }
       assertTrue(hasException);
 
-      assertEquals(4, cluster.getNamesystem().getFSDirectory().getYieldCount());
+      assertEquals(5, cluster.getNamesystem().getFSDirectory().getYieldCount());
     } finally {
       cluster.shutdown();
     }
@@ -386,6 +395,7 @@ public class TestQuota {
       final Path quotaDir1 = new Path("/nqdir0/qdir1");
       dfs.setQuota(quotaDir1, 6, HdfsConstants.QUOTA_DONT_SET);
       ContentSummary c = dfs.getContentSummary(quotaDir1);
+      compareQuotaUsage(c, dfs, quotaDir1);
       assertEquals(c.getDirectoryCount(), 3);
       assertEquals(c.getQuota(), 6);
 
@@ -393,6 +403,7 @@ public class TestQuota {
       final Path quotaDir2 = new Path("/nqdir0/qdir1/qdir20");
       dfs.setQuota(quotaDir2, 7, HdfsConstants.QUOTA_DONT_SET);
       c = dfs.getContentSummary(quotaDir2);
+      compareQuotaUsage(c, dfs, quotaDir2);
       assertEquals(c.getDirectoryCount(), 2);
       assertEquals(c.getQuota(), 7);
 
@@ -401,6 +412,7 @@ public class TestQuota {
       assertTrue(dfs.mkdirs(quotaDir3));
       dfs.setQuota(quotaDir3, 2, HdfsConstants.QUOTA_DONT_SET);
       c = dfs.getContentSummary(quotaDir3);
+      compareQuotaUsage(c, dfs, quotaDir3);
       assertEquals(c.getDirectoryCount(), 1);
       assertEquals(c.getQuota(), 2);
 
@@ -408,6 +420,7 @@ public class TestQuota {
       Path tempPath = new Path(quotaDir3, "nqdir32");
       assertTrue(dfs.mkdirs(tempPath));
       c = dfs.getContentSummary(quotaDir3);
+      compareQuotaUsage(c, dfs, quotaDir3);
       assertEquals(c.getDirectoryCount(), 2);
       assertEquals(c.getQuota(), 2);
 
@@ -421,6 +434,7 @@ public class TestQuota {
       }
       assertTrue(hasException);
       c = dfs.getContentSummary(quotaDir3);
+      compareQuotaUsage(c, dfs, quotaDir3);
       assertEquals(c.getDirectoryCount(), 2);
       assertEquals(c.getQuota(), 2);
 
@@ -428,9 +442,11 @@ public class TestQuota {
       tempPath = new Path(quotaDir2, "nqdir31");
       assertTrue(dfs.mkdirs(tempPath));
       c = dfs.getContentSummary(quotaDir2);
+      compareQuotaUsage(c, dfs, quotaDir2);
       assertEquals(c.getDirectoryCount(), 3);
       assertEquals(c.getQuota(), 7);
       c = dfs.getContentSummary(quotaDir1);
+      compareQuotaUsage(c, dfs, quotaDir1);
       assertEquals(c.getDirectoryCount(), 6);
       assertEquals(c.getQuota(), 6);
 
@@ -448,9 +464,11 @@ public class TestQuota {
       tempPath = new Path(quotaDir2, "nqdir30");
       dfs.rename(new Path(quotaDir3, "nqdir32"), tempPath);
       c = dfs.getContentSummary(quotaDir2);
+      compareQuotaUsage(c, dfs, quotaDir2);
       assertEquals(c.getDirectoryCount(), 4);
       assertEquals(c.getQuota(), 7);
       c = dfs.getContentSummary(quotaDir1);
+      compareQuotaUsage(c, dfs, quotaDir1);
       assertEquals(c.getDirectoryCount(), 6);
       assertEquals(c.getQuota(), 6);
 
@@ -479,9 +497,11 @@ public class TestQuota {
       // 11: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0
       assertTrue(dfs.rename(tempPath, new Path("/nqdir0")));
       c = dfs.getContentSummary(quotaDir2);
+      compareQuotaUsage(c, dfs, quotaDir2);
       assertEquals(c.getDirectoryCount(), 2);
       assertEquals(c.getQuota(), 7);
       c = dfs.getContentSummary(quotaDir1);
+      compareQuotaUsage(c, dfs, quotaDir1);
       assertEquals(c.getDirectoryCount(), 4);
       assertEquals(c.getQuota(), 6);
 
@@ -500,31 +520,38 @@ public class TestQuota {
       // 14: Move /nqdir0/qdir1/qdir21 /nqdir0/qdir1/qdir20
       assertTrue(dfs.rename(quotaDir3, quotaDir2));
       c = dfs.getContentSummary(quotaDir1);
+      compareQuotaUsage(c, dfs, quotaDir1);
       assertEquals(c.getDirectoryCount(), 4);
       assertEquals(c.getQuota(), 6);
       c = dfs.getContentSummary(quotaDir2);
+      compareQuotaUsage(c, dfs, quotaDir2);
       assertEquals(c.getDirectoryCount(), 3);
       assertEquals(c.getQuota(), 7);
       tempPath = new Path(quotaDir2, "qdir21");
       c = dfs.getContentSummary(tempPath);
+      compareQuotaUsage(c, dfs, tempPath);
       assertEquals(c.getDirectoryCount(), 1);
       assertEquals(c.getQuota(), 2);
 
       // 15: Delete /nqdir0/qdir1/qdir20/qdir21
       dfs.delete(tempPath, true);
       c = dfs.getContentSummary(quotaDir2);
+      compareQuotaUsage(c, dfs, quotaDir2);
       assertEquals(c.getDirectoryCount(), 2);
       assertEquals(c.getQuota(), 7);
       c = dfs.getContentSummary(quotaDir1);
+      compareQuotaUsage(c, dfs, quotaDir1);
       assertEquals(c.getDirectoryCount(), 3);
       assertEquals(c.getQuota(), 6);
 
       // 16: Move /nqdir0/qdir30 /nqdir0/qdir1/qdir20
       assertTrue(dfs.rename(new Path("/nqdir0/nqdir30"), quotaDir2));
       c = dfs.getContentSummary(quotaDir2);
+      compareQuotaUsage(c, dfs, quotaDir2);
       assertEquals(c.getDirectoryCount(), 5);
       assertEquals(c.getQuota(), 7);
       c = dfs.getContentSummary(quotaDir1);
+      compareQuotaUsage(c, dfs, quotaDir1);
       assertEquals(c.getDirectoryCount(), 6);
       assertEquals(c.getQuota(), 6);
       assertEquals(14, cluster.getNamesystem().getFSDirectory().getYieldCount());
@@ -566,12 +593,14 @@ public class TestQuota {
       final Path quotaDir1 = new Path("/nqdir0/qdir1");
       dfs.setQuota(quotaDir1, HdfsConstants.QUOTA_DONT_SET, 4 * fileSpace);
       ContentSummary c = dfs.getContentSummary(quotaDir1);
+      compareQuotaUsage(c, dfs, quotaDir1);
       assertEquals(c.getSpaceQuota(), 4 * fileSpace);
       
       // set the quota of /nqdir0/qdir1/qdir20 to 6 * fileSpace 
       final Path quotaDir20 = new Path("/nqdir0/qdir1/qdir20");
       dfs.setQuota(quotaDir20, HdfsConstants.QUOTA_DONT_SET, 6 * fileSpace);
       c = dfs.getContentSummary(quotaDir20);
+      compareQuotaUsage(c, dfs, quotaDir20);
       assertEquals(c.getSpaceQuota(), 6 * fileSpace);
 
       // Create /nqdir0/qdir1/qdir21 and set its space quota to 2 * fileSpace
@@ -579,6 +608,7 @@ public class TestQuota {
       assertTrue(dfs.mkdirs(quotaDir21));
       dfs.setQuota(quotaDir21, HdfsConstants.QUOTA_DONT_SET, 2 * fileSpace);
       c = dfs.getContentSummary(quotaDir21);
+      compareQuotaUsage(c, dfs, quotaDir21);
       assertEquals(c.getSpaceQuota(), 2 * fileSpace);
 
       // 5: Create directory /nqdir0/qdir1/qdir21/nqdir32
@@ -589,6 +619,7 @@ public class TestQuota {
       DFSTestUtil.createFile(dfs, new Path(tempPath, "fileDir/file1"), fileLen, 
                              replication, 0);
       c = dfs.getContentSummary(quotaDir21);
+      compareQuotaUsage(c, dfs, quotaDir21);
       assertEquals(c.getSpaceConsumed(), fileSpace);
       
       // Create a larger file /nqdir0/qdir1/qdir21/nqdir33/
@@ -603,11 +634,13 @@ public class TestQuota {
       // delete nqdir33
       assertTrue(dfs.delete(new Path(quotaDir21, "nqdir33"), true));
       c = dfs.getContentSummary(quotaDir21);
+      compareQuotaUsage(c, dfs, quotaDir21);
       assertEquals(c.getSpaceConsumed(), fileSpace);
       assertEquals(c.getSpaceQuota(), 2*fileSpace);
 
       // Verify space before the move:
       c = dfs.getContentSummary(quotaDir20);
+      compareQuotaUsage(c, dfs, quotaDir20);
       assertEquals(c.getSpaceConsumed(), 0);
       
       // Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30
@@ -620,9 +653,11 @@ public class TestQuota {
       assertEquals(c.getSpaceConsumed(), fileSpace);
       // verify space for its parent
       c = dfs.getContentSummary(quotaDir1);
+      compareQuotaUsage(c, dfs, quotaDir1);
       assertEquals(c.getSpaceConsumed(), fileSpace);
       // verify space for source for the move
       c = dfs.getContentSummary(quotaDir21);
+      compareQuotaUsage(c, dfs, quotaDir21);
       assertEquals(c.getSpaceConsumed(), 0);
       
       final Path file2 = new Path(dstPath, "fileDir/file2");
@@ -633,6 +668,7 @@ public class TestQuota {
       c = dfs.getContentSummary(quotaDir20);
       assertEquals(c.getSpaceConsumed(), 3 * fileSpace);
       c = dfs.getContentSummary(quotaDir21);
+      compareQuotaUsage(c, dfs, quotaDir21);
       assertEquals(c.getSpaceConsumed(), 0);
       
       // Reverse: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/
@@ -653,16 +689,19 @@ public class TestQuota {
       c = dfs.getContentSummary(quotaDir20);
       assertEquals(c.getSpaceConsumed(), 3 * fileSpace);
       c = dfs.getContentSummary(quotaDir21);
+      compareQuotaUsage(c, dfs, quotaDir21);
       assertEquals(c.getSpaceConsumed(), 0);
       
       // Test Append :
       
       // verify space quota
       c = dfs.getContentSummary(quotaDir1);
+      compareQuotaUsage(c, dfs, quotaDir1);
       assertEquals(c.getSpaceQuota(), 4 * fileSpace);
       
       // verify space before append;
       c = dfs.getContentSummary(dstPath);
+      compareQuotaUsage(c, dfs, dstPath);
       assertEquals(c.getSpaceConsumed(), 3 * fileSpace);
       
       OutputStream out = dfs.append(file2);
@@ -674,6 +713,7 @@ public class TestQuota {
       
       // verify space after append;
       c = dfs.getContentSummary(dstPath);
+      compareQuotaUsage(c, dfs, dstPath);
       assertEquals(c.getSpaceConsumed(), 4 * fileSpace);
       
       // now increase the quota for quotaDir1
@@ -695,6 +735,7 @@ public class TestQuota {
       
       // verify space after partial append
       c = dfs.getContentSummary(dstPath);
+      compareQuotaUsage(c, dfs, dstPath);
       assertEquals(c.getSpaceConsumed(), 5 * fileSpace);
       
       // Test set replication :
@@ -704,6 +745,7 @@ public class TestQuota {
       
       // verify that space is reduced by file2Len
       c = dfs.getContentSummary(dstPath);
+      compareQuotaUsage(c, dfs, dstPath);
       assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len);
       
       // now try to increase the replication and and expect an error.
@@ -717,6 +759,7 @@ public class TestQuota {
 
       // verify space consumed remains unchanged.
       c = dfs.getContentSummary(dstPath);
+      compareQuotaUsage(c, dfs, dstPath);
       assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len);
       
       // now increase the quota for quotaDir1 and quotaDir20
@@ -727,6 +770,7 @@ public class TestQuota {
       dfs.setReplication(file2, (short)(replication+1));
       // verify increase in space
       c = dfs.getContentSummary(dstPath);
+      compareQuotaUsage(c, dfs, dstPath);
       assertEquals(c.getSpaceConsumed(), 5 * fileSpace + file2Len);
 
       // Test HDFS-2053 :
@@ -754,32 +798,37 @@ public class TestQuota {
       dfs.setQuota(quotaDir2053_C, HdfsConstants.QUOTA_DONT_SET,
           (sizeFactorC + 1) * fileSpace);
       c = dfs.getContentSummary(quotaDir2053_C);
+      compareQuotaUsage(c, dfs, quotaDir2053_C);
       assertEquals(c.getSpaceQuota(), (sizeFactorC + 1) * fileSpace);
 
       // Create a file under subdirectory A
       DFSTestUtil.createFile(dfs, new Path(quotaDir2053_A, "fileA"),
           sizeFactorA * fileLen, replication, 0);
       c = dfs.getContentSummary(quotaDir2053_A);
+      compareQuotaUsage(c, dfs, quotaDir2053_A);
       assertEquals(c.getSpaceConsumed(), sizeFactorA * fileSpace);
 
       // Create a file under subdirectory B
       DFSTestUtil.createFile(dfs, new Path(quotaDir2053_B, "fileB"),
           sizeFactorB * fileLen, replication, 0);
       c = dfs.getContentSummary(quotaDir2053_B);
+      compareQuotaUsage(c, dfs, quotaDir2053_B);
       assertEquals(c.getSpaceConsumed(), sizeFactorB * fileSpace);
 
       // Create a file under subdirectory C (which has a space quota)
       DFSTestUtil.createFile(dfs, new Path(quotaDir2053_C, "fileC"),
           sizeFactorC * fileLen, replication, 0);
       c = dfs.getContentSummary(quotaDir2053_C);
+      compareQuotaUsage(c, dfs, quotaDir2053_C);
       assertEquals(c.getSpaceConsumed(), sizeFactorC * fileSpace);
 
       // Check space consumed for /hdfs-2053
       c = dfs.getContentSummary(quotaDir2053);
+      compareQuotaUsage(c, dfs, quotaDir2053);
       assertEquals(c.getSpaceConsumed(),
           (sizeFactorA + sizeFactorB + sizeFactorC) * fileSpace);
 
-      assertEquals(20, cluster.getNamesystem().getFSDirectory().getYieldCount());
+      assertEquals(28, cluster.getNamesystem().getFSDirectory().getYieldCount());
     } finally {
       cluster.shutdown();
     }
@@ -885,6 +934,7 @@ public class TestQuota {
       DFSTestUtil.createFile(fs, file1, FILE_SIZE, (short) 3, 1L);
       DFSTestUtil.waitReplication(fs, file1, (short) 3);
       c = fs.getContentSummary(dir);
+      compareQuotaUsage(c, fs, dir);
       checkContentSummary(c, webhdfs.getContentSummary(dir));
       assertEquals("Quota is half consumed", QUOTA_SIZE / 2,
                    c.getSpaceConsumed());
@@ -963,6 +1013,7 @@ public class TestQuota {
 
       // Should account for all 59 files (almost QUOTA_SIZE)
       c = fs.getContentSummary(dir);
+      compareQuotaUsage(c, fs, dir);
       checkContentSummary(c, webhdfs.getContentSummary(dir));
       assertEquals("Invalid space consumed", 59 * FILE_SIZE * 3,
           c.getSpaceConsumed());
@@ -1024,6 +1075,7 @@ public class TestQuota {
       FSDataOutputStream out = dfs.create(new Path("/Folder2/file6"),(short)1);
       out.close();
       ContentSummary contentSummary = dfs.getContentSummary(new Path("/"));
+      compareQuotaUsage(contentSummary, dfs, new Path("/"));
       assertEquals(6, contentSummary.getFileCount());
     } finally {
       if (cluster != null) {
@@ -1033,4 +1085,11 @@ public class TestQuota {
     }
   }
 
+  // check the QuotaUsage got from getContentSummary is the same as
+  // getQuotaUsage
+  private void compareQuotaUsage(final QuotaUsage fromContentSummary,
+      final FileSystem fileSystem, final Path filePath) throws IOException {
+    QuotaUsage quotaUsage = fileSystem.getQuotaUsage(filePath);
+    assertEquals(fromContentSummary, quotaUsage);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5db1d44/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java
index e768306..cbd01b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java
@@ -139,11 +139,23 @@ public class TestQuotasWithHA {
    * exception.
    */
   @Test(expected = StandbyException.class)
-  public void testgetContentSummaryOnStandby() throws Exception {
+  public void testGetContentSummaryOnStandby() throws Exception {
     Configuration nn1conf =cluster.getConfiguration(1);
     // just reset the standby reads to default i.e False on standby.
     HAUtil.setAllowStandbyReads(nn1conf, false);
     cluster.restartNameNode(1);
     cluster.getNameNodeRpc(1).getContentSummary("/");
   }
+
+  /**
+   * Test that getQuotaUsage on Standby should should throw standby exception.
+   */
+  @Test(expected = StandbyException.class)
+  public void testGetQuotaUsageOnStandby() throws Exception {
+    Configuration nn1conf =cluster.getConfiguration(1);
+    // just reset the standby reads to default i.e False on standby.
+    HAUtil.setAllowStandbyReads(nn1conf, false);
+    cluster.restartNameNode(1);
+    cluster.getNameNodeRpc(1).getQuotaUsage("/");
+  }
 }


Mime
View raw message