Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: hdfs-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1235017 - in
 /hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs:
 CHANGES.HDFS-1623.txt
 src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java
Date: Mon, 23 Jan 2012 21:57:00 -0000
To: hdfs-commits@hadoop.apache.org
From: todd@apache.org
Message-Id: <20120123215700.B73632388A40@eris.apache.org>

Author: todd
Date: Mon Jan 23 21:57:00 2012
New Revision: 1235017

URL: http://svn.apache.org/viewvc?rev=1235017&view=rev
Log:
HDFS-2688. Add tests for quota tracking in an HA cluster. Contributed by Todd Lipcon.

Added:
    hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java
Modified:
    hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt

Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt?rev=1235017&r1=1235016&r2=1235017&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt Mon Jan 23 21:57:00 2012
@@ -123,3 +123,5 @@ HDFS-2812. When becoming active, the NN 
 HDFS-2737. Automatically trigger log rolls periodically on the active NN. (todd and atm)
 
 HDFS-2820. Add a simple sanity check for HA config (todd)
+
+HDFS-2688. Add tests for quota tracking in an HA cluster. (todd)

Added: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java?rev=1235017&view=auto
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java (added)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java Mon Jan 23 21:57:00 2012
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode.ha;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HAUtil;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestQuotasWithHA {
+  private static final Path TEST_DIR = new Path("/test");
+  private static final Path TEST_FILE = new Path(TEST_DIR, "file");
+  private static final String TEST_DIR_STR = TEST_DIR.toUri().getPath();
+  
+  private static final long NS_QUOTA = 10000;
+  private static final long DS_QUOTA = 10000;
+  private static final long BLOCK_SIZE = 1024; // 1KB blocks
+  
+  private MiniDFSCluster cluster;
+  private NameNode nn0;
+  private NameNode nn1;
+  private FileSystem fs;
+
+  @Before
+  public void setupCluster() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+    conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+    HAUtil.setAllowStandbyReads(conf, true);
+    
+    cluster = new MiniDFSCluster.Builder(conf)
+      .nnTopology(MiniDFSNNTopology.simpleHATopology())
+      .numDataNodes(1)
+      .waitSafeMode(false)
+      .build();
+    cluster.waitActive();
+    
+    nn0 = cluster.getNameNode(0);
+    nn1 = cluster.getNameNode(1);
+    fs = HATestUtil.configureFailoverFs(cluster, conf);
+    
+    cluster.transitionToActive(0);
+  }
+  
+  @After
+  public void shutdownCluster() throws IOException {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+  }
+
+  /**
+   * Test that quotas are properly tracked by the standby through
+   * create, append, delete.
+   */
+  @Test(timeout=60000)
+  public void testQuotasTrackedOnStandby() throws Exception {
+    fs.mkdirs(TEST_DIR);
+    DistributedFileSystem dfs = (DistributedFileSystem)fs;
+    dfs.setQuota(TEST_DIR, NS_QUOTA, DS_QUOTA);
+    long expectedSize = 3 * BLOCK_SIZE + BLOCK_SIZE/2;
+    DFSTestUtil.createFile(fs, TEST_FILE, expectedSize, (short)1, 1L);
+
+    HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
+    ContentSummary cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
+    assertEquals(NS_QUOTA, cs.getQuota());
+    assertEquals(DS_QUOTA, cs.getSpaceQuota());
+    assertEquals(expectedSize, cs.getSpaceConsumed());
+    assertEquals(1, cs.getDirectoryCount());
+    assertEquals(1, cs.getFileCount());
+
+    // Append to the file and make sure quota is updated correctly.
+    FSDataOutputStream stm = fs.append(TEST_FILE);
+    try {
+      byte[] data = new byte[(int) (BLOCK_SIZE * 3 / 2)];
+      stm.write(data);
+      expectedSize += data.length;
+    } finally {
+      IOUtils.closeStream(stm);
+    }
+    
+    HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
+    cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
+    assertEquals(NS_QUOTA, cs.getQuota());
+    assertEquals(DS_QUOTA, cs.getSpaceQuota());
+    assertEquals(expectedSize, cs.getSpaceConsumed());
+    assertEquals(1, cs.getDirectoryCount());
+    assertEquals(1, cs.getFileCount());
+
+    
+    fs.delete(TEST_FILE, true);
+    expectedSize = 0;
+    HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
+    cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
+    assertEquals(NS_QUOTA, cs.getQuota());
+    assertEquals(DS_QUOTA, cs.getSpaceQuota());
+    assertEquals(expectedSize, cs.getSpaceConsumed());
+    assertEquals(1, cs.getDirectoryCount());
+    assertEquals(0, cs.getFileCount());
+  }
+}