hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject [43/50] git commit: HDFS-7072. Fix TestBlockManager and TestStorageMover. Contributed by Jing Zhao
Date Thu, 18 Sep 2014 23:15:20 GMT
HDFS-7072. Fix TestBlockManager and TestStorageMover.  Contributed by Jing Zhao


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd2e409d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd2e409d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd2e409d

Branch: refs/heads/trunk
Commit: bd2e409de128dcf4143f69c1d572e03a51511517
Parents: 432b863
Author: Tsz-Wo Nicholas Sze <szetszwo@hortonworks.com>
Authored: Wed Sep 17 13:48:32 2014 +0800
Committer: Tsz-Wo Nicholas Sze <szetszwo@hortonworks.com>
Committed: Wed Sep 17 13:48:32 2014 +0800

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt       |  3 +++
 .../blockmanagement/DatanodeStorageInfo.java      |  2 +-
 .../server/blockmanagement/TestBlockManager.java  |  1 +
 .../hdfs/server/mover/TestStorageMover.java       | 18 ++++++++++++++++++
 4 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd2e409d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7584582..59a130e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -23,6 +23,9 @@ HDFS-6584: Archival Storage
     HDFS-6847. Support storage policy on directories and include storage policy 
     in HdfsFileStatus.  (Jing Zhao via szetszwo)
 
+    HDFS-7072. Fix TestBlockManager and TestStorageMover.  (Jing Zhao
+    via szetszwo)
+
 Trunk (Unreleased)
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd2e409d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index c363f4b..58ca2ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -109,7 +109,7 @@ public class DatanodeStorageInfo {
 
   private long capacity;
   private long dfsUsed;
-  private long remaining;
+  private volatile long remaining;
   private long blockPoolUsed;
 
   private volatile BlockInfo blockList = null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd2e409d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 9b980dc..7c0623c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -666,6 +666,7 @@ public class TestBlockManager {
     excessTypes.add(StorageType.DEFAULT);
     Assert.assertTrue(BlockManager.useDelHint(true, delHint, null,
         moreThan1Racks, excessTypes));
+    excessTypes.remove(0);
     excessTypes.add(StorageType.SSD);
     Assert.assertFalse(BlockManager.useDelHint(true, delHint, null,
         moreThan1Racks, excessTypes));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd2e409d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
index ceedfc2..eeea62b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
@@ -588,6 +588,20 @@ public class TestStorageMover {
     }
   }
 
+  private void waitForAllReplicas(int expectedReplicaNum, Path file,
+      DistributedFileSystem dfs) throws Exception {
+    for (int i = 0; i < 5; i++) {
+      LocatedBlocks lbs = dfs.getClient().getLocatedBlocks(file.toString(), 0,
+          BLOCK_SIZE);
+      LocatedBlock lb = lbs.get(0);
+      if (lb.getLocations().length >= expectedReplicaNum) {
+        return;
+      } else {
+        Thread.sleep(1000);
+      }
+    }
+  }
+
   /**
    * Test DISK is running out of spaces.
    */
@@ -618,6 +632,7 @@ public class TestStorageMover {
           for (; ; hotFileCount++) {
             final Path p = new Path(pathPolicyMap.hot, "file" + hotFileCount);
             DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L);
+            waitForAllReplicas(replication, p, test.dfs);
           }
         } catch (IOException e) {
           LOG.info("Expected: hotFileCount=" + hotFileCount, e);
@@ -632,6 +647,7 @@ public class TestStorageMover {
           for (; ; hotFileCount_r1++) {
             final Path p = new Path(pathPolicyMap.hot, "file_r1_" + hotFileCount_r1);
             DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, (short) 1, 0L);
+            waitForAllReplicas(1, p, test.dfs);
           }
         } catch (IOException e) {
           LOG.info("Expected: hotFileCount_r1=" + hotFileCount_r1, e);
@@ -699,6 +715,7 @@ public class TestStorageMover {
           for (; ; coldFileCount++) {
             final Path p = new Path(pathPolicyMap.cold, "file" + coldFileCount);
             DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L);
+            waitForAllReplicas(replication, p, test.dfs);
           }
         } catch (IOException e) {
           LOG.info("Expected: coldFileCount=" + coldFileCount, e);
@@ -713,6 +730,7 @@ public class TestStorageMover {
           for (; ; coldFileCount_r1++) {
             final Path p = new Path(pathPolicyMap.cold, "file_r1_" + coldFileCount_r1);
             DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, (short) 1, 0L);
+            waitForAllReplicas(1, p, test.dfs);
           }
         } catch (IOException e) {
           LOG.info("Expected: coldFileCount_r1=" + coldFileCount_r1, e);


Mime
View raw message