Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D584BD866 for ; Fri, 3 Aug 2012 19:02:38 +0000 (UTC) Received: (qmail 21106 invoked by uid 500); 3 Aug 2012 19:02:30 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 20871 invoked by uid 500); 3 Aug 2012 19:02:30 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 20445 invoked by uid 99); 3 Aug 2012 19:02:29 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 03 Aug 2012 19:02:29 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 03 Aug 2012 19:02:21 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 2B16D2388C63; Fri, 3 Aug 2012 19:01:28 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1369164 [12/16] - in /hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project: ./ hadoop-hdfs-httpfs/ hadoop-hdfs-httpfs/dev-support/ hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/ hadoop-hdfs-httpfs/src/main/java/o... Date: Fri, 03 Aug 2012 19:00:59 -0000 To: hdfs-commits@hadoop.apache.org From: gkesavan@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120803190128.2B16D2388C63@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java Fri Aug 3 19:00:15 2012 @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hdfs.server.balancer; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + import java.io.IOException; import java.net.URI; import java.util.ArrayList; @@ -41,12 +44,11 @@ import org.apache.hadoop.hdfs.protocol.B import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; +import org.apache.hadoop.util.Time; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; /** * This class tests if a balancer schedules tasks correctly. @@ -221,7 +223,7 @@ public class TestBalancer { throws IOException, TimeoutException { long timeout = TIMEOUT; long failtime = (timeout <= 0L) ? Long.MAX_VALUE - : System.currentTimeMillis() + timeout; + : Time.now() + timeout; while (true) { long[] status = client.getStats(); @@ -233,7 +235,7 @@ public class TestBalancer { && usedSpaceVariance < CAPACITY_ALLOWED_VARIANCE) break; //done - if (System.currentTimeMillis() > failtime) { + if (Time.now() > failtime) { throw new TimeoutException("Cluster failed to reached expected values of " + "totalSpace (current: " + status[0] + ", expected: " + expectedTotalSpace @@ -259,7 +261,7 @@ public class TestBalancer { throws IOException, TimeoutException { long timeout = TIMEOUT; long failtime = (timeout <= 0L) ? Long.MAX_VALUE - : System.currentTimeMillis() + timeout; + : Time.now() + timeout; final double avgUtilization = ((double)totalUsedSpace) / totalCapacity; boolean balanced; do { @@ -272,7 +274,7 @@ public class TestBalancer { / datanode.getCapacity(); if (Math.abs(avgUtilization - nodeUtilization) > BALANCE_ALLOWED_VARIANCE) { balanced = false; - if (System.currentTimeMillis() > failtime) { + if (Time.now() > failtime) { throw new TimeoutException( "Rebalancing expected avg utilization to become " + avgUtilization + ", but on datanode " + datanode @@ -454,6 +456,39 @@ public class TestBalancer { } /** + * Test parse method in Balancer#Cli class with wrong number of params + */ + + @Test + public void testBalancerCliParseWithWrongParams() { + String parameters[] = new String[] { "-threshold" }; + String reason = + "IllegalArgumentException is expected when value is not specified"; + try { + Balancer.Cli.parse(parameters); + fail(reason); + } catch (IllegalArgumentException e) { + + } + parameters = new String[] { "-policy" }; + try { + Balancer.Cli.parse(parameters); + fail(reason); + } catch (IllegalArgumentException e) { + + } + parameters = new String[] { "-threshold 1 -policy" }; + try { + Balancer.Cli.parse(parameters); + fail(reason); + } catch (IllegalArgumentException e) { + + } + + } + + + /** * @param args */ public static void main(String[] args) throws Exception { Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java Fri Aug 3 19:00:15 2012 @@ -29,8 +29,8 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; -import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf; +import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java Fri Aug 3 19:00:15 2012 @@ -40,8 +40,8 @@ import org.apache.hadoop.hdfs.protocol.B import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.LeaseManager; import org.apache.hadoop.hdfs.server.namenode.NameNode; Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java Fri Aug 3 19:00:15 2012 @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import java.util.ArrayList; import java.util.Iterator; @@ -29,7 +29,7 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.GenerationStamp; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.junit.Test; /** * This class provides tests for BlockInfo class, which is used in BlocksMap. Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java Fri Aug 3 19:00:15 2012 @@ -17,25 +17,25 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import junit.framework.TestCase; +import static org.junit.Assert.assertEquals; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.junit.Test; /** * Test if FSNamesystem handles heartbeat right */ -public class TestComputeInvalidateWork extends TestCase { +public class TestComputeInvalidateWork { /** * Test if {@link FSNamesystem#computeInvalidateWork(int)} * can schedule invalidate work correctly */ + @Test public void testCompInvalidate() throws Exception { final Configuration conf = new HdfsConfiguration(); final int NUM_OF_DATANODES = 3; Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java Fri Aug 3 19:00:15 2012 @@ -17,6 +17,11 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + import java.io.IOException; import java.util.Arrays; import java.util.HashMap; @@ -24,12 +29,11 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.protocol.Block; +import org.junit.Test; /** @@ -38,7 +42,7 @@ import org.apache.hadoop.hdfs.protocol.B * CorruptReplicasMap::getCorruptReplicaBlockIds * return the correct values */ -public class TestCorruptReplicaInfo extends TestCase { +public class TestCorruptReplicaInfo { private static final Log LOG = LogFactory.getLog(TestCorruptReplicaInfo.class); @@ -60,6 +64,7 @@ public class TestCorruptReplicaInfo exte return getBlock((long)block_id); } + @Test public void testCorruptReplicaInfo() throws IOException, InterruptedException { Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java Fri Aug 3 19:00:15 2012 @@ -17,21 +17,25 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + import java.util.ArrayList; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.GenerationStamp; - -import junit.framework.TestCase; +import org.junit.Test; /** * This class tests that methods in DatanodeDescriptor */ -public class TestDatanodeDescriptor extends TestCase { +public class TestDatanodeDescriptor { /** * Test that getInvalidateBlocks observes the maxlimit. */ + @Test public void testGetInvalidateBlocks() throws Exception { final int MAX_BLOCKS = 10; final int REMAINING_BLOCKS = 2; @@ -49,6 +53,7 @@ public class TestDatanodeDescriptor exte assertEquals(bc.length, REMAINING_BLOCKS); } + @Test public void testBlocksCounter() throws Exception { DatanodeDescriptor dd = DFSTestUtil.getLocalDatanodeDescriptor(); assertEquals(0, dd.numBlocks()); Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java Fri Aug 3 19:00:15 2012 @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import java.util.ArrayList; +import static org.junit.Assert.assertEquals; -import junit.framework.TestCase; +import java.util.ArrayList; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -34,17 +34,19 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.junit.Test; /** * Test if FSNamesystem handles heartbeat right */ -public class TestHeartbeatHandling extends TestCase { +public class TestHeartbeatHandling { /** * Test if * {@link FSNamesystem#handleHeartbeat} * can pick up replication and/or invalidate requests and observes the max * limit */ + @Test public void testHeartbeat() throws Exception { final Configuration conf = new HdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHost2NodesMap.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHost2NodesMap.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHost2NodesMap.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHost2NodesMap.java Fri Aug 3 19:00:15 2012 @@ -18,13 +18,15 @@ package org.apache.hadoop.hdfs.server.blockmanagement; -import org.apache.hadoop.hdfs.DFSTestUtil; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.*; - public class TestHost2NodesMap { private Host2NodesMap map = new Host2NodesMap(); private DatanodeDescriptor dataNodes[]; Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java Fri Aug 3 19:00:15 2012 @@ -17,12 +17,12 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import static org.junit.Assert.assertTrue; + import java.util.Collection; import java.util.Iterator; import java.util.concurrent.TimeoutException; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -32,10 +32,9 @@ import org.apache.hadoop.hdfs.MiniDFSClu import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; -import org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager; -import org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.util.Time; +import org.junit.Test; /** * Test if live nodes count per node is correct @@ -44,7 +43,7 @@ import org.apache.hadoop.hdfs.server.nam * Two of the "while" loops below use "busy wait" * because they are detecting transient states. */ -public class TestNodeCount extends TestCase { +public class TestNodeCount { final short REPLICATION_FACTOR = (short)2; final long TIMEOUT = 20000L; long timeout = 0; @@ -52,6 +51,7 @@ public class TestNodeCount extends TestC Block lastBlock = null; NumberReplicas lastNum = null; + @Test public void testNodeCount() throws Exception { // start a mini dfs cluster of 2 nodes final Configuration conf = new HdfsConfiguration(); @@ -140,7 +140,7 @@ public class TestNodeCount extends TestC void initializeTimeout(long timeout) { this.timeout = timeout; - this.failtime = System.currentTimeMillis() + this.failtime = Time.now() + ((timeout <= 0) ? Long.MAX_VALUE : timeout); } @@ -151,7 +151,7 @@ public class TestNodeCount extends TestC /* check for timeout, then wait for cycleTime msec */ void checkTimeout(String testLabel, long cycleTime) throws TimeoutException { - if (System.currentTimeMillis() > failtime) { + if (Time.now() > failtime) { throw new TimeoutException("Timeout: " + testLabel + " for block " + lastBlock + " after " + timeout + " msec. Last counts: live = " + lastNum.liveReplicas() Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java Fri Aug 3 19:00:15 2012 @@ -17,13 +17,14 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import static org.apache.hadoop.hdfs.server.common.Util.now; -import static org.junit.Assert.*; +import static org.apache.hadoop.util.Time.now; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import java.io.File; import java.io.IOException; import java.util.Collection; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataOutputStream; Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java Fri Aug 3 19:00:15 2012 @@ -17,13 +17,13 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import java.util.Queue; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.server.blockmanagement.PendingDataNodeMessages.ReportedBlockInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.junit.Test; Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java Fri Aug 3 19:00:15 2012 @@ -17,17 +17,19 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import junit.framework.TestCase; -import java.lang.System; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import org.apache.hadoop.hdfs.protocol.Block; +import org.junit.Test; /** * This class tests the internals of PendingReplicationBlocks.java */ -public class TestPendingReplication extends TestCase { +public class TestPendingReplication { final static int TIMEOUT = 3; // 3 seconds + @Test public void testPendingReplication() { PendingReplicationBlocks pendingReplications; pendingReplications = new PendingReplicationBlocks(TIMEOUT * 1000); Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java Fri Aug 3 19:00:15 2012 @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import java.io.File; import java.io.IOException; @@ -34,8 +37,6 @@ import org.apache.hadoop.hdfs.server.dat import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * Test when RBW block is removed. Invalidation of the corrupted block happens Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java Fri Aug 3 19:00:15 2012 @@ -17,7 +17,9 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.io.File; import java.util.ArrayList; Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java Fri Aug 3 19:00:15 2012 @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -26,7 +30,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; @@ -39,7 +42,7 @@ import org.apache.hadoop.net.NetworkTopo import org.apache.hadoop.net.Node; import org.junit.Test; -public class TestReplicationPolicyWithNodeGroup extends TestCase { +public class TestReplicationPolicyWithNodeGroup { private static final int BLOCK_SIZE = 1024; private static final int NUM_OF_DATANODES = 8; private static final Configuration CONF = new HdfsConfiguration(); @@ -104,6 +107,7 @@ public class TestReplicationPolicyWithNo * the 1st is on dataNodes[0] and the 2nd is on a different rack. * @throws Exception */ + @Test public void testChooseTarget1() throws Exception { dataNodes[0].updateHeartbeat( 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, @@ -164,6 +168,7 @@ public class TestReplicationPolicyWithNo * node group, and the rest should be placed on a third rack. * @throws Exception */ + @Test public void testChooseTarget2() throws Exception { HashMap excludedNodes; DatanodeDescriptor[] targets; @@ -207,6 +212,7 @@ public class TestReplicationPolicyWithNo * and the rest should be placed on the third rack. * @throws Exception */ + @Test public void testChooseTarget3() throws Exception { // make data node 0 to be not qualified to choose dataNodes[0].updateHeartbeat( @@ -259,6 +265,7 @@ public class TestReplicationPolicyWithNo * in different node group. * @throws Exception */ + @Test public void testChooseTarget4() throws Exception { // make data node 0-2 to be not qualified to choose: not enough disk space for(int i=0; i<3; i++) { @@ -302,6 +309,7 @@ public class TestReplicationPolicyWithNo * the 3rd replica should be placed on the same rack as the 2nd replica, * @throws Exception */ + @Test public void testChooseTarget5() throws Exception { setupDataNodeCapacity(); DatanodeDescriptor[] targets; @@ -333,6 +341,7 @@ public class TestReplicationPolicyWithNo * the 1st replica. The 3rd replica can be placed randomly. * @throws Exception */ + @Test public void testRereplicate1() throws Exception { setupDataNodeCapacity(); List chosenNodes = new ArrayList(); @@ -369,6 +378,7 @@ public class TestReplicationPolicyWithNo * the rest replicas can be placed randomly, * @throws Exception */ + @Test public void testRereplicate2() throws Exception { setupDataNodeCapacity(); List chosenNodes = new ArrayList(); @@ -399,6 +409,7 @@ public class TestReplicationPolicyWithNo * the rest replicas can be placed randomly, * @throws Exception */ + @Test public void testRereplicate3() throws Exception { setupDataNodeCapacity(); List chosenNodes = new ArrayList(); Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java Fri Aug 3 19:00:15 2012 @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import junit.framework.TestCase; +import static org.junit.Assert.assertEquals; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -27,8 +27,10 @@ import org.apache.hadoop.hdfs.DFSTestUti import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.junit.Test; -public class TestUnderReplicatedBlocks extends TestCase { +public class TestUnderReplicatedBlocks { + @Test public void testSetrepIncWithUnderReplicatedBlocks() throws Exception { Configuration conf = new HdfsConfiguration(); final short REPLICATION_FACTOR = 2; Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestGetUriFromString.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestGetUriFromString.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestGetUriFromString.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestGetUriFromString.java Fri Aug 3 19:00:15 2012 @@ -17,19 +17,21 @@ */ package org.apache.hadoop.hdfs.server.common; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import java.io.IOException; import java.net.URI; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.junit.Test; /** * This is a unit test, which tests {@link Util#stringAsURI(String)} * for Windows and Unix style file paths. */ -public class TestGetUriFromString extends TestCase { +public class TestGetUriFromString { private static final Log LOG = LogFactory.getLog(TestGetUriFromString.class); private static final String RELATIVE_FILE_PATH = "relativeFilePath"; @@ -49,6 +51,7 @@ public class TestGetUriFromString extend * Test for a relative path, os independent * @throws IOException */ + @Test public void testRelativePathAsURI() throws IOException { URI u = Util.stringAsURI(RELATIVE_FILE_PATH); LOG.info("Uri: " + u); @@ -59,6 +62,7 @@ public class TestGetUriFromString extend * Test for an OS dependent absolute paths. * @throws IOException */ + @Test public void testAbsolutePathAsURI() throws IOException { URI u = null; u = Util.stringAsURI(ABSOLUTE_PATH_WINDOWS); @@ -74,6 +78,7 @@ public class TestGetUriFromString extend * Test for a URI * @throws IOException */ + @Test public void testURI() throws IOException { LOG.info("Testing correct Unix URI: " + URI_UNIX); URI u = Util.stringAsURI(URI_UNIX); Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java Fri Aug 3 19:00:15 2012 @@ -31,8 +31,13 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer; +import org.apache.hadoop.hdfs.web.resources.DoAsParam; +import org.apache.hadoop.hdfs.web.resources.UserParam; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; @@ -136,4 +141,262 @@ public class TestJspHelper { Assert.assertEquals("", delegationTokenParam); } + @Test + public void testGetUgiFromToken() throws IOException { + conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://localhost:4321/"); + ServletContext context = mock(ServletContext.class); + String realUser = "TheDoctor"; + String user = "TheNurse"; + conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(conf); + UserGroupInformation ugi; + HttpServletRequest request; + + Text ownerText = new Text(user); + DelegationTokenIdentifier dtId = new DelegationTokenIdentifier( + ownerText, ownerText, new Text(realUser)); + Token token = new Token( + dtId, new DummySecretManager(0, 0, 0, 0)); + String tokenString = token.encodeToUrlString(); + + // token with no auth-ed user + request = getMockRequest(null, null, null); + when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn( + tokenString); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNotNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser); + Assert.assertEquals(ugi.getShortUserName(), user); + checkUgiFromToken(ugi); + + // token with auth-ed user + request = getMockRequest(realUser, null, null); + when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn( + tokenString); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNotNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser); + Assert.assertEquals(ugi.getShortUserName(), user); + checkUgiFromToken(ugi); + + // completely different user, token trumps auth + request = getMockRequest("rogue", null, null); + when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn( + tokenString); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNotNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser); + Assert.assertEquals(ugi.getShortUserName(), user); + checkUgiFromToken(ugi); + + // expected case + request = getMockRequest(null, user, null); + when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn( + tokenString); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNotNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser); + Assert.assertEquals(ugi.getShortUserName(), user); + checkUgiFromToken(ugi); + + // can't proxy with a token! + request = getMockRequest(null, null, "rogue"); + when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn( + tokenString); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Usernames not matched: name=rogue != expected="+user, + ioe.getMessage()); + } + + // can't proxy with a token! + request = getMockRequest(null, user, "rogue"); + when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn( + tokenString); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Usernames not matched: name=rogue != expected="+user, + ioe.getMessage()); + } + } + + @Test + public void testGetNonProxyUgi() throws IOException { + conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://localhost:4321/"); + ServletContext context = mock(ServletContext.class); + String realUser = "TheDoctor"; + String user = "TheNurse"; + conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(conf); + UserGroupInformation ugi; + HttpServletRequest request; + + // have to be auth-ed with remote user + request = getMockRequest(null, null, null); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Security enabled but user not authenticated by filter", + ioe.getMessage()); + } + request = getMockRequest(null, realUser, null); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Security enabled but user not authenticated by filter", + ioe.getMessage()); + } + + // ugi for remote user + request = getMockRequest(realUser, null, null); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getShortUserName(), realUser); + checkUgiFromAuth(ugi); + + // ugi for remote user = real user + request = getMockRequest(realUser, realUser, null); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getShortUserName(), realUser); + checkUgiFromAuth(ugi); + + // ugi for remote user != real user + request = getMockRequest(realUser, user, null); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Usernames not matched: name="+user+" != expected="+realUser, + ioe.getMessage()); + } + } + + @Test + public void testGetProxyUgi() throws IOException { + conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://localhost:4321/"); + ServletContext context = mock(ServletContext.class); + String realUser = "TheDoctor"; + String user = "TheNurse"; + conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + + conf.set(ProxyUsers.CONF_HADOOP_PROXYUSER+realUser+".groups", "*"); + conf.set(ProxyUsers.CONF_HADOOP_PROXYUSER+realUser+".hosts", "*"); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf); + UserGroupInformation.setConfiguration(conf); + UserGroupInformation ugi; + HttpServletRequest request; + + // have to be auth-ed with remote user + request = getMockRequest(null, null, user); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Security enabled but user not authenticated by filter", + ioe.getMessage()); + } + request = getMockRequest(null, realUser, user); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Security enabled but user not authenticated by filter", + ioe.getMessage()); + } + + // proxy ugi for user via remote user + request = getMockRequest(realUser, null, user); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNotNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser); + Assert.assertEquals(ugi.getShortUserName(), user); + checkUgiFromAuth(ugi); + + // proxy ugi for user vi a remote user = real user + request = getMockRequest(realUser, realUser, user); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNotNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser); + Assert.assertEquals(ugi.getShortUserName(), user); + checkUgiFromAuth(ugi); + + // proxy ugi for user via remote user != real user + request = getMockRequest(realUser, user, user); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Usernames not matched: name="+user+" != expected="+realUser, + ioe.getMessage()); + } + + // try to get get a proxy user with unauthorized user + try { + request = getMockRequest(user, null, realUser); + JspHelper.getUGI(context, request, conf); + Assert.fail("bad proxy request allowed"); + } catch (AuthorizationException ae) { + Assert.assertEquals( + "User: " + user + " is not allowed to impersonate " + realUser, + ae.getMessage()); + } + try { + request = getMockRequest(user, user, realUser); + JspHelper.getUGI(context, request, conf); + Assert.fail("bad proxy request allowed"); + } catch (AuthorizationException ae) { + Assert.assertEquals( + "User: " + user + " is not allowed to impersonate " + realUser, + ae.getMessage()); + } + } + + private HttpServletRequest getMockRequest(String remoteUser, String user, String doAs) { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getParameter(UserParam.NAME)).thenReturn(user); + if (doAs != null) { + when(request.getParameter(DoAsParam.NAME)).thenReturn(doAs); + } + when(request.getRemoteUser()).thenReturn(remoteUser); + return request; + } + + private void checkUgiFromAuth(UserGroupInformation ugi) { + if (ugi.getRealUser() != null) { + Assert.assertEquals(AuthenticationMethod.PROXY, + ugi.getAuthenticationMethod()); + Assert.assertEquals(AuthenticationMethod.KERBEROS_SSL, + ugi.getRealUser().getAuthenticationMethod()); + } else { + Assert.assertEquals(AuthenticationMethod.KERBEROS_SSL, + ugi.getAuthenticationMethod()); + } + } + + private void checkUgiFromToken(UserGroupInformation ugi) { + if (ugi.getRealUser() != null) { + Assert.assertEquals(AuthenticationMethod.PROXY, + ugi.getAuthenticationMethod()); + Assert.assertEquals(AuthenticationMethod.TOKEN, + ugi.getRealUser().getAuthenticationMethod()); + } else { + Assert.assertEquals(AuthenticationMethod.TOKEN, + ugi.getAuthenticationMethod()); + } + } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Fri Aug 3 19:00:15 2012 @@ -47,6 +47,7 @@ import org.apache.hadoop.hdfs.server.dat import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.DiskChecker.DiskErrorException; @@ -132,10 +133,12 @@ public class SimulatedFSDataset implemen } } + @Override synchronized public long getGenerationStamp() { return theBlock.getGenerationStamp(); } + @Override synchronized public long getNumBytes() { if (!finalized) { return bytesRcvd; @@ -144,6 +147,7 @@ public class SimulatedFSDataset implemen } } + @Override synchronized public void setNumBytes(long length) { if (!finalized) { bytesRcvd = length; @@ -686,7 +690,7 @@ public class SimulatedFSDataset implemen public synchronized InputStream getBlockInputStream(ExtendedBlock b, long seekOffset) throws IOException { InputStream result = getBlockInputStream(b); - result.skip(seekOffset); + IOUtils.skipFully(result, seekOffset); return result; } @@ -876,14 +880,17 @@ public class SimulatedFSDataset implemen DataNode.LOG.info("Registered FSDatasetState MBean"); } + @Override public void shutdown() { if (mbeanName != null) MBeans.unregister(mbeanName); } + @Override public String getStorageInfo() { return "Simulated FSDataset-" + storageId; } + @Override public boolean hasEnoughResource() { return true; } Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java Fri Aug 3 19:00:15 2012 @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; import java.io.IOException; import java.net.InetSocketAddress; Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java Fri Aug 3 19:00:15 2012 @@ -38,21 +38,27 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; @@ -561,4 +567,69 @@ public class TestBlockRecovery { streams.close(); } } + + /** + * Test to verify the race between finalizeBlock and Lease recovery + * + * @throws Exception + */ + @Test(timeout = 20000) + public void testRaceBetweenReplicaRecoveryAndFinalizeBlock() throws Exception { + tearDown();// Stop the Mocked DN started in startup() + + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(MiniDFSNNTopology.simpleSingleNN(8020, 50070)) + .numDataNodes(1).build(); + try { + cluster.waitClusterUp(); + DistributedFileSystem fs = cluster.getFileSystem(); + Path path = new Path("/test"); + FSDataOutputStream out = fs.create(path); + out.writeBytes("data"); + out.hsync(); + + List blocks = DFSTestUtil.getAllBlocks(fs.open(path)); + final LocatedBlock block = blocks.get(0); + final DataNode dataNode = cluster.getDataNodes().get(0); + + final AtomicBoolean recoveryInitResult = new AtomicBoolean(true); + Thread recoveryThread = new Thread() { + @Override + public void run() { + try { + DatanodeInfo[] locations = block.getLocations(); + final RecoveringBlock recoveringBlock = new RecoveringBlock( + block.getBlock(), locations, block.getBlock() + .getGenerationStamp() + 1); + synchronized (dataNode.data) { + Thread.sleep(2000); + dataNode.initReplicaRecovery(recoveringBlock); + } + } catch (Exception e) { + recoveryInitResult.set(false); + } + } + }; + recoveryThread.start(); + try { + out.close(); + } catch (IOException e) { + Assert.assertTrue("Writing should fail", + e.getMessage().contains("are bad. Aborting...")); + } finally { + recoveryThread.join(); + } + Assert.assertTrue("Recovery should be initiated successfully", + recoveryInitResult.get()); + + dataNode.updateReplicaUnderRecovery(block.getBlock(), block.getBlock() + .getGenerationStamp() + 1, block.getBlockSize()); + } finally { + if (null != cluster) { + cluster.shutdown(); + cluster = null; + } + } + } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java Fri Aug 3 19:00:15 2012 @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hdfs.server.datanode; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; @@ -47,14 +51,10 @@ import org.apache.hadoop.hdfs.protocol.p import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.net.NetUtils; - +import org.apache.hadoop.util.Time; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; /** * This class tests if block replacement request to data nodes work correctly. @@ -71,7 +71,7 @@ public class TestBlockReplacement { long bandwidthPerSec = 1024*1024L; final long TOTAL_BYTES =6*bandwidthPerSec; long bytesToSend = TOTAL_BYTES; - long start = Util.now(); + long start = Time.now(); DataTransferThrottler throttler = new DataTransferThrottler(bandwidthPerSec); long totalBytes = 0L; long bytesSent = 1024*512L; // 0.5MB @@ -84,7 +84,7 @@ public class TestBlockReplacement { Thread.sleep(1000); } catch (InterruptedException ignored) {} throttler.throttle(bytesToSend); - long end = Util.now(); + long end = Time.now(); assertTrue(totalBytes*1000/(end-start)<=bandwidthPerSec); } @@ -208,7 +208,7 @@ public class TestBlockReplacement { throws IOException, TimeoutException { boolean notDone; final long TIMEOUT = 20000L; - long starttime = System.currentTimeMillis(); + long starttime = Time.now(); long failtime = starttime + TIMEOUT; do { try { @@ -233,7 +233,7 @@ public class TestBlockReplacement { } } } - if (System.currentTimeMillis() > failtime) { + if (Time.now() > failtime) { String expectedNodesList = ""; String currentNodesList = ""; for (DatanodeInfo dn : includeNodes) @@ -248,7 +248,7 @@ public class TestBlockReplacement { } } while(notDone); LOG.info("Achieved expected replication values in " - + (System.currentTimeMillis() - starttime) + " msec."); + + (Time.now() - starttime) + " msec."); } /* Copy a block from sourceProxy to destination. If the block becomes Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java Fri Aug 3 19:00:15 2012 @@ -55,6 +55,7 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils.DelayAnswer; +import org.apache.hadoop.util.Time; import org.apache.log4j.Level; import org.junit.After; import org.junit.Before; @@ -615,12 +616,12 @@ public class TestBlockReport { final DataNode dn1 = cluster.getDataNodes().get(DN_N1); String bpid = cluster.getNamesystem().getBlockPoolId(); Replica r = DataNodeTestUtils.fetchReplicaInfo(dn1, bpid, bl.getBlockId()); - long start = System.currentTimeMillis(); + long start = Time.now(); int count = 0; while (r == null) { waitTil(5); r = DataNodeTestUtils.fetchReplicaInfo(dn1, bpid, bl.getBlockId()); - long waiting_period = System.currentTimeMillis() - start; + long waiting_period = Time.now() - start; if (count++ % 100 == 0) if(LOG.isDebugEnabled()) { LOG.debug("Has been waiting for " + waiting_period + " ms."); @@ -634,7 +635,7 @@ public class TestBlockReport { if(LOG.isDebugEnabled()) { LOG.debug("Replica state before the loop " + state.getValue()); } - start = System.currentTimeMillis(); + start = Time.now(); while (state != HdfsServerConstants.ReplicaState.TEMPORARY) { waitTil(5); state = r.getState(); @@ -642,7 +643,7 @@ public class TestBlockReport { LOG.debug("Keep waiting for " + bl.getBlockName() + " is in state " + state.getValue()); } - if (System.currentTimeMillis() - start > TIMEOUT) + if (Time.now() - start > TIMEOUT) assertTrue("Was waiting too long for a replica to become TEMPORARY", tooLongWait); } @@ -761,6 +762,7 @@ public class TestBlockReport { this.all = all; } + @Override public boolean accept(File file, String s) { if (all) return s != null && s.startsWith(nameToAccept); @@ -830,6 +832,7 @@ public class TestBlockReport { this.filePath = filePath; } + @Override public void run() { try { startDNandWait(filePath, true); Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java Fri Aug 3 19:00:15 2012 @@ -23,10 +23,10 @@ import java.util.List; import javax.management.MBeanServer; import javax.management.ObjectName; -import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.conf.Configuration; -import org.junit.Test; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.Assert; +import org.junit.Test; /** * Class for testing {@link DataNodeMXBean} implementation Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java Fri Aug 3 19:00:15 2012 @@ -18,20 +18,26 @@ package org.apache.hadoop.hdfs.server.datanode; import static org.apache.hadoop.test.MetricsAsserts.assertCounter; +import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges; import static org.apache.hadoop.test.MetricsAsserts.getLongCounter; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import java.util.List; -import java.util.Random; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.junit.Test; @@ -58,8 +64,10 @@ public class TestDataNodeMetrics { } @Test - public void testSendDataPacket() throws Exception { + public void testSendDataPacketMetrics() throws Exception { Configuration conf = new HdfsConfiguration(); + final int interval = 1; + conf.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); try { FileSystem fs = cluster.getFileSystem(); @@ -72,64 +80,110 @@ public class TestDataNodeMetrics { assertEquals(datanodes.size(), 1); DataNode datanode = datanodes.get(0); MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name()); - // Expect 2 packets, 1 for the 1 byte read, 1 for the empty packet // signaling the end of the block assertCounter("SendDataPacketTransferNanosNumOps", (long)2, rb); assertCounter("SendDataPacketBlockedOnNetworkNanosNumOps", (long)2, rb); + // Wait for at least 1 rollover + Thread.sleep((interval + 1) * 1000); + // Check that the sendPacket percentiles rolled to non-zero values + String sec = interval + "s"; + assertQuantileGauges("SendDataPacketBlockedOnNetworkNanos" + sec, rb); + assertQuantileGauges("SendDataPacketTransferNanos" + sec, rb); } finally { if (cluster != null) {cluster.shutdown();} } } @Test - public void testFlushMetric() throws Exception { + public void testReceivePacketMetrics() throws Exception { Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + final int interval = 1; + conf.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); try { cluster.waitActive(); DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem(); Path testFile = new Path("/testFlushNanosMetric.txt"); - DFSTestUtil.createFile(fs, testFile, 1, (short)1, new Random().nextLong()); - + FSDataOutputStream fout = fs.create(testFile); + fout.write(new byte[1]); + fout.hsync(); + fout.close(); List datanodes = cluster.getDataNodes(); DataNode datanode = datanodes.get(0); MetricsRecordBuilder dnMetrics = getMetrics(datanode.getMetrics().name()); - // Expect 2 flushes, 1 for the flush that occurs after writing, 1 that occurs - // on closing the data and metadata files. + // Expect two flushes, 1 for the flush that occurs after writing, + // 1 that occurs on closing the data and metadata files. assertCounter("FlushNanosNumOps", 2L, dnMetrics); + // Expect two syncs, one from the hsync, one on close. + assertCounter("FsyncNanosNumOps", 2L, dnMetrics); + // Wait for at least 1 rollover + Thread.sleep((interval + 1) * 1000); + // Check the receivePacket percentiles that should be non-zero + String sec = interval + "s"; + assertQuantileGauges("FlushNanos" + sec, dnMetrics); + assertQuantileGauges("FsyncNanos" + sec, dnMetrics); } finally { if (cluster != null) {cluster.shutdown();} } } + /** + * Tests that round-trip acks in a datanode write pipeline are correctly + * measured. + */ @Test public void testRoundTripAckMetric() throws Exception { - final int DATANODE_COUNT = 2; - + final int datanodeCount = 2; + final int interval = 1; Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_COUNT).build(); + conf.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes( + datanodeCount).build(); try { cluster.waitActive(); - DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem(); - + FileSystem fs = cluster.getFileSystem(); + // Open a file and get the head of the pipeline Path testFile = new Path("/testRoundTripAckMetric.txt"); - DFSTestUtil.createFile(fs, testFile, 1, (short)DATANODE_COUNT, - new Random().nextLong()); - - boolean foundNonzeroPacketAckNumOps = false; + FSDataOutputStream fsout = fs.create(testFile, (short) datanodeCount); + DFSOutputStream dout = (DFSOutputStream) fsout.getWrappedStream(); + // Slow down the writes to catch the write pipeline + dout.setChunksPerPacket(5); + dout.setArtificialSlowdown(3000); + fsout.write(new byte[10000]); + DatanodeInfo[] pipeline = null; + int count = 0; + while (pipeline == null && count < 5) { + pipeline = dout.getPipeline(); + System.out.println("Waiting for pipeline to be created."); + Thread.sleep(1000); + count++; + } + // Get the head node that should be receiving downstream acks + DatanodeInfo headInfo = pipeline[0]; + DataNode headNode = null; for (DataNode datanode : cluster.getDataNodes()) { - MetricsRecordBuilder dnMetrics = getMetrics(datanode.getMetrics().name()); - if (getLongCounter("PacketAckRoundTripTimeNanosNumOps", dnMetrics) > 0) { - foundNonzeroPacketAckNumOps = true; + if (datanode.getDatanodeId().equals(headInfo)) { + headNode = datanode; + break; } } - assertTrue( - "Expected at least one datanode to have reported PacketAckRoundTripTimeNanos metric", - foundNonzeroPacketAckNumOps); + assertNotNull("Could not find the head of the datanode write pipeline", + headNode); + // Close the file and wait for the metrics to rollover + Thread.sleep((interval + 1) * 1000); + // Check the ack was received + MetricsRecordBuilder dnMetrics = getMetrics(headNode.getMetrics() + .name()); + assertTrue("Expected non-zero number of acks", + getLongCounter("PacketAckRoundTripTimeNanosNumOps", dnMetrics) > 0); + assertQuantileGauges("PacketAckRoundTripTimeNanos" + interval + + "s", dnMetrics); } finally { - if (cluster != null) {cluster.shutdown();} + if (cluster != null) { + cluster.shutdown(); + } } } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java Fri Aug 3 19:00:15 2012 @@ -375,6 +375,7 @@ public class TestDataNodeVolumeFailure { private String [] metaFilesInDir(File dir) { String [] res = dir.list( new FilenameFilter() { + @Override public boolean accept(File dir, String name) { return name.startsWith("blk_") && name.endsWith(Block.METADATA_EXTENSION); Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java Fri Aug 3 19:00:15 2012 @@ -19,12 +19,13 @@ package org.apache.hadoop.hdfs.server.datanode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; -import junit.framework.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -74,7 +75,7 @@ public class TestDeleteBlockPool { // Although namenode is shutdown, the bp offerservice is still running try { dn1.deleteBlockPool(bpid1, true); - Assert.fail("Must not delete a running block pool"); + fail("Must not delete a running block pool"); } catch (IOException expected) { } @@ -85,7 +86,7 @@ public class TestDeleteBlockPool { try { dn1.deleteBlockPool(bpid1, false); - Assert.fail("Must not delete if any block files exist unless " + fail("Must not delete if any block files exist unless " + "force is true"); } catch (IOException expected) { } @@ -115,7 +116,7 @@ public class TestDeleteBlockPool { // on dn2 try { dn2.deleteBlockPool(bpid1, true); - Assert.fail("Must not delete a running block pool"); + fail("Must not delete a running block pool"); } catch (IOException expected) { } @@ -180,21 +181,21 @@ public class TestDeleteBlockPool { Configuration nn1Conf = cluster.getConfiguration(0); nn1Conf.set(DFSConfigKeys.DFS_NAMESERVICES, "namesServerId1"); dn1.refreshNamenodes(nn1Conf); - Assert.assertEquals(1, dn1.getAllBpOs().length); + assertEquals(1, dn1.getAllBpOs().length); DFSAdmin admin = new DFSAdmin(nn1Conf); String dn1Address = dn1.getDatanodeId().getIpAddr() + ":" + dn1.getIpcPort(); String[] args = { "-deleteBlockPool", dn1Address, bpid2 }; int ret = admin.run(args); - Assert.assertFalse(0 == ret); + assertFalse(0 == ret); verifyBlockPoolDirectories(true, dn1StorageDir1, bpid2); verifyBlockPoolDirectories(true, dn1StorageDir2, bpid2); String[] forceArgs = { "-deleteBlockPool", dn1Address, bpid2, "force" }; ret = admin.run(forceArgs); - Assert.assertEquals(0, ret); + assertEquals(0, ret); verifyBlockPoolDirectories(false, dn1StorageDir1, bpid2); verifyBlockPoolDirectories(false, dn1StorageDir2, bpid2); @@ -216,7 +217,7 @@ public class TestDeleteBlockPool { + bpid); if (shouldExist == false) { - Assert.assertFalse(bpDir.exists()); + assertFalse(bpDir.exists()); } else { File bpCurrentDir = new File(bpDir, DataStorage.STORAGE_DIR_CURRENT); File finalizedDir = new File(bpCurrentDir, @@ -224,9 +225,9 @@ public class TestDeleteBlockPool { File rbwDir = new File(bpCurrentDir, DataStorage.STORAGE_DIR_RBW); File versionFile = new File(bpCurrentDir, "VERSION"); - Assert.assertTrue(finalizedDir.isDirectory()); - Assert.assertTrue(rbwDir.isDirectory()); - Assert.assertTrue(versionFile.exists()); + assertTrue(finalizedDir.isDirectory()); + assertTrue(rbwDir.isDirectory()); + assertTrue(versionFile.exists()); } } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java Fri Aug 3 19:00:15 2012 @@ -17,6 +17,12 @@ */ package org.apache.hadoop.hdfs.server.datanode; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -25,8 +31,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Random; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -38,15 +42,16 @@ import org.apache.hadoop.hdfs.HdfsConfig import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.GenerationStamp; -import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil; +import org.junit.Test; /** * Tests {@link DirectoryScanner} handling of differences * between blocks on the disk and block in memory. */ -public class TestDirectoryScanner extends TestCase { +public class TestDirectoryScanner { private static final Log LOG = LogFactory.getLog(TestDirectoryScanner.class); private static final Configuration CONF = new HdfsConfiguration(); private static final int DEFAULT_GEN_STAMP = 9999; @@ -218,6 +223,7 @@ public class TestDirectoryScanner extend assertEquals(mismatchBlocks, stats.mismatchBlocks); } + @Test public void testDirectoryScanner() throws Exception { // Run the test with and without parallel scanning for (int parallelism = 1; parallelism < 3; parallelism++) { Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java Fri Aug 3 19:00:15 2012 @@ -17,7 +17,8 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import static org.apache.hadoop.test.MetricsAsserts.*; +import static org.apache.hadoop.test.MetricsAsserts.assertCounter; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import java.util.EnumSet; import java.util.Random; Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestRefreshNamenodes.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestRefreshNamenodes.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestRefreshNamenodes.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestRefreshNamenodes.java Fri Aug 3 19:00:15 2012 @@ -18,7 +18,8 @@ package org.apache.hadoop.hdfs.server.datanode; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.net.InetSocketAddress; Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java Fri Aug 3 19:00:15 2012 @@ -17,13 +17,16 @@ */ package org.apache.hadoop.hdfs.server.datanode; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.protocol.Block; @@ -33,25 +36,23 @@ import org.apache.hadoop.hdfs.server.dat import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory; import org.apache.hadoop.util.DataChecksum; +import org.junit.Before; +import org.junit.Test; /** * this class tests the methods of the SimulatedFSDataset. */ -public class TestSimulatedFSDataset extends TestCase { +public class TestSimulatedFSDataset { Configuration conf = null; static final String bpid = "BP-TEST"; static final int NUMBLOCKS = 20; static final int BLOCK_LENGTH_MULTIPLIER = 79; - protected void setUp() throws Exception { - super.setUp(); + @Before + public void setUp() throws Exception { conf = new HdfsConfiguration(); SimulatedFSDataset.setFactory(conf); } - - protected void tearDown() throws Exception { - super.tearDown(); - } long blockIdToLen(long blkid) { return blkid*BLOCK_LENGTH_MULTIPLIER; @@ -88,6 +89,7 @@ public class TestSimulatedFSDataset exte return addSomeBlocks(fsdataset, 1); } + @Test public void testFSDatasetFactory() { final Configuration conf = new Configuration(); FsDatasetSpi.Factory f = FsDatasetSpi.Factory.getFactory(conf); @@ -100,6 +102,7 @@ public class TestSimulatedFSDataset exte assertTrue(s.isSimulated()); } + @Test public void testGetMetaData() throws IOException { final SimulatedFSDataset fsdataset = getSimulatedFSDataset(); ExtendedBlock b = new ExtendedBlock(bpid, 1, 5, 0); @@ -121,6 +124,7 @@ public class TestSimulatedFSDataset exte } + @Test public void testStorageUsage() throws IOException { final SimulatedFSDataset fsdataset = getSimulatedFSDataset(); assertEquals(fsdataset.getDfsUsed(), 0); @@ -144,6 +148,7 @@ public class TestSimulatedFSDataset exte assertEquals(expectedLen, lengthRead); } + @Test public void testWriteRead() throws IOException { final SimulatedFSDataset fsdataset = getSimulatedFSDataset(); addSomeBlocks(fsdataset); @@ -155,6 +160,7 @@ public class TestSimulatedFSDataset exte } } + @Test public void testGetBlockReport() throws IOException { SimulatedFSDataset fsdataset = getSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.getBlockReport(bpid); @@ -168,6 +174,7 @@ public class TestSimulatedFSDataset exte } } + @Test public void testInjectionEmpty() throws IOException { SimulatedFSDataset fsdataset = getSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.getBlockReport(bpid); @@ -196,6 +203,7 @@ public class TestSimulatedFSDataset exte assertEquals(sfsdataset.getCapacity()-bytesAdded, sfsdataset.getRemaining()); } + @Test public void testInjectionNonEmpty() throws IOException { SimulatedFSDataset fsdataset = getSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.getBlockReport(bpid); @@ -269,6 +277,7 @@ public class TestSimulatedFSDataset exte } } + @Test public void testInValidBlocks() throws IOException { final SimulatedFSDataset fsdataset = getSimulatedFSDataset(); ExtendedBlock b = new ExtendedBlock(bpid, 1, 5, 0); @@ -280,6 +289,7 @@ public class TestSimulatedFSDataset exte checkInvalidBlock(b); } + @Test public void testInvalidate() throws IOException { final SimulatedFSDataset fsdataset = getSimulatedFSDataset(); int bytesAdded = addSomeBlocks(fsdataset); Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java?rev=1369164&r1=1369163&r2=1369164&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java Fri Aug 3 19:00:15 2012 @@ -17,7 +17,9 @@ */ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; -import static org.junit.Assert.*; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.IOException; import java.net.InetSocketAddress;