hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r835169 - in /hadoop/hdfs/branches/branch-0.21: CHANGES.txt src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
Date Thu, 12 Nov 2009 00:29:35 GMT
Author: suresh
Date: Thu Nov 12 00:29:32 2009
New Revision: 835169

URL: http://svn.apache.org/viewvc?rev=835169&view=rev
Log:
HDFS-761. Fix failure to process rename operation from edits log due to quota verification.
Contributed by Suresh Srinivas.

Modified:
    hadoop/hdfs/branches/branch-0.21/CHANGES.txt
    hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
    hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java

Modified: hadoop/hdfs/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/CHANGES.txt?rev=835169&r1=835168&r2=835169&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.21/CHANGES.txt Thu Nov 12 00:29:32 2009
@@ -487,3 +487,6 @@
 
     HDFS-525. The SimpleDateFormat object in ListPathsServlet is not thread
     safe. (Suresh Srinivas and cdouglas)
+
+    HDFS-761. Fix failure to process rename operation from edits log due to 
+    quota verification. (suresh)

Modified: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=835169&r1=835168&r2=835169&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++ hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Thu Nov 12 00:29:32 2009
@@ -1084,6 +1084,10 @@
    */
   private void verifyQuota(INode[] inodes, int pos, long nsDelta, long dsDelta,
       INode commonAncestor) throws QuotaExceededException {
+    if (!ready) {
+      // Do not check quota if edits log is still being processed
+      return;
+    }
     if (pos>inodes.length) {
       pos = inodes.length;
     }
@@ -1117,6 +1121,10 @@
    */
   private void verifyQuotaForRename(INode[] srcInodes, INode[]dstInodes)
       throws QuotaExceededException {
+    if (!ready) {
+      // Do not check quota if edits log is still being processed
+      return;
+    }
     INode srcInode = srcInodes[srcInodes.length - 1];
     INode commonAncestor = null;
     for(int i =0;srcInodes[i] == dstInodes[i]; i++) {

Modified: hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java?rev=835169&r1=835168&r2=835169&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
(original)
+++ hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
Thu Nov 12 00:29:32 2009
@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs;
 
 import java.io.IOException;
+import java.net.URISyntaxException;
 
 import javax.security.auth.login.LoginException;
 
@@ -26,7 +27,6 @@
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.FSConstants;
-import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.security.UnixUserGroupInformation;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -39,17 +39,31 @@
                                   FileContextMainOperationsBaseTest {
   private static MiniDFSCluster cluster;
   private static Path defaultWorkingDirectory;
+  private static HdfsConfiguration CONF = new HdfsConfiguration();
   
   @BeforeClass
-  public static void clusterSetupAtBegining()
-                                    throws IOException, LoginException  {
-    cluster = new MiniDFSCluster(new HdfsConfiguration(), 2, true, null);
+  public static void clusterSetupAtBegining() throws IOException,
+      LoginException, URISyntaxException {
+    cluster = new MiniDFSCluster(CONF, 2, true, null);
+    cluster.waitClusterUp();
     fc = FileContext.getFileContext(cluster.getFileSystem());
     defaultWorkingDirectory = fc.makeQualified( new Path("/user/" + 
         UnixUserGroupInformation.login().getUserName()));
     fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true);
   }
 
+  private static void restartCluster() throws IOException, LoginException {
+    if (cluster != null) {
+      cluster.shutdown();
+      cluster = null;
+    }
+    cluster = new MiniDFSCluster(CONF, 1, false, null);
+    cluster.waitClusterUp();
+    fc = FileContext.getFileContext(cluster.getFileSystem());
+    defaultWorkingDirectory = fc.makeQualified( new Path("/user/" + 
+        UnixUserGroupInformation.login().getUserName()));
+    fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true);
+  }
       
   @AfterClass
   public static void ClusterShutdownAtEnd() throws Exception {
@@ -79,45 +93,79 @@
   }
   
   @Test
-  public void testRenameWithQuota() throws Exception {
+  public void testOldRenameWithQuota() throws Exception {
     DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();
-    Path src1 = getTestRootPath("testRenameWithQuota/srcdir/src1");
-    Path src2 = getTestRootPath("testRenameWithQuota/srcdir/src2");
-    Path dst1 = getTestRootPath("testRenameWithQuota/dstdir/dst1");
-    Path dst2 = getTestRootPath("testRenameWithQuota/dstdir/dst2");
+    Path src1 = getTestRootPath("test/testOldRenameWithQuota/srcdir/src1");
+    Path src2 = getTestRootPath("test/testOldRenameWithQuota/srcdir/src2");
+    Path dst1 = getTestRootPath("test/testOldRenameWithQuota/dstdir/dst1");
+    Path dst2 = getTestRootPath("test/testOldRenameWithQuota/dstdir/dst2");
     createFile(src1);
     createFile(src2);
     fs.setQuota(src1.getParent(), FSConstants.QUOTA_DONT_SET,
         FSConstants.QUOTA_DONT_SET);
     fc.mkdir(dst1.getParent(), FileContext.DEFAULT_PERM, true);
-    fs.setQuota(dst1.getParent(), FSConstants.QUOTA_DONT_SET,
-        FSConstants.QUOTA_DONT_SET);
-
-    // Test1: src does not exceed quota and dst has no quota check and hence 
-    // accommodates rename
-    rename(src1, dst1, true, false);
-
-    // Test2: src does not exceed quota and dst has *no* quota to accommodate
-    // rename. 
-
-    // testRenameWithQuota/dstDir now has quota = 1 and dst1 already uses it
-    fs.setQuota(dst1.getParent(), 1, FSConstants.QUOTA_DONT_SET);
-    rename(src2, dst2, false, true);
 
-    // Test3: src exceeds quota and dst has *no* quota to accommodate rename
-    
+    fs.setQuota(dst1.getParent(), 2, FSConstants.QUOTA_DONT_SET);
+    /* 
+     * Test1: src does not exceed quota and dst has no quota check and hence 
+     * accommodates rename
+     */
+    oldRename(src1, dst1, true, false);
+
+    /*
+     * Test2: src does not exceed quota and dst has *no* quota to accommodate 
+     * rename. 
+     */
+    // dstDir quota = 1 and dst1 already uses it
+    oldRename(src2, dst2, false, true);
+
+    /*
+     * Test3: src exceeds quota and dst has *no* quota to accommodate rename
+     */
     // src1 has no quota to accommodate new rename node
     fs.setQuota(src1.getParent(), 1, FSConstants.QUOTA_DONT_SET);
-    rename(dst1, src1, false, true);
+    oldRename(dst1, src1, false, true);
+
   }
   
-  private void rename(Path src, Path dst, boolean renameSucceeds,
-      boolean quotaException) throws Exception {
+    
+  /**
+   * Perform operations such as setting quota, deletion of files, rename and
+   * ensure system can apply edits log during startup.
+   */
+  @Test
+  public void testEditsLogOldRename() throws Exception {
+    DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();
+    Path src1 = getTestRootPath("testEditsLogOldRename/srcdir/src1");
+    Path dst1 = getTestRootPath("testEditsLogOldRename/dstdir/dst1");
+    createFile(src1);
+    fs.mkdirs(dst1.getParent());
+    createFile(dst1);
+    
+    // Set quota so that dst1 parent cannot allow under it new files/directories 
+    fs.setQuota(dst1.getParent(), 2, FSConstants.QUOTA_DONT_SET);
+    // Free up quota for a subsequent rename
+    fs.delete(dst1, true);
+    oldRename(src1, dst1, true, false);
+    
+    // Restart the cluster and ensure the above operations can be
+    // loaded from the edits log
+    restartCluster();
+    fs = (DistributedFileSystem)cluster.getFileSystem();
+    src1 = getTestRootPath("testEditsLogOldRename/srcdir/src1");
+    dst1 = getTestRootPath("testEditsLogOldRename/dstdir/dst1");
+    Assert.assertFalse(fs.exists(src1));   // ensure src1 is already renamed
+    Assert.assertTrue(fs.exists(dst1));    // ensure rename dst exists
+  }
+  
+
+  private void oldRename(Path src, Path dst, boolean renameSucceeds,
+      boolean exception) throws Exception {
     DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();
     try {
       Assert.assertEquals(renameSucceeds, fs.rename(src, dst));
-    } catch (QuotaExceededException ex) {
-      Assert.assertTrue(quotaException);
+    } catch (Exception ex) {
+      Assert.assertTrue(exception);
     }
     Assert.assertEquals(renameSucceeds, !fc.exists(src));
     Assert.assertEquals(renameSucceeds, fc.exists(dst));



Mime
View raw message