hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vinayakum...@apache.org
Subject [1/3] hadoop git commit: HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the correct namenode (Contributed by Wei-Chiu Chuang)
Date Thu, 07 Jan 2016 08:51:55 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 aa710bd46 -> 08f39bd5d
  refs/heads/branch-2.8 a14baa05b -> 3303326dd
  refs/heads/trunk 34cd7cd76 -> 6702e7d66


HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the correct namenode (Contributed
by Wei-Chiu Chuang)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6702e7d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6702e7d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6702e7d6

Branch: refs/heads/trunk
Commit: 6702e7d66a4949a40bcdbd9f51ca1e746c8991e1
Parents: 34cd7cd
Author: Vinayakumar B <vinayakumarb@apache.org>
Authored: Thu Jan 7 14:20:18 2016 +0530
Committer: Vinayakumar B <vinayakumarb@apache.org>
Committed: Thu Jan 7 14:20:18 2016 +0530

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 ++
 .../server/datanode/SimulatedFSDataset.java     |  3 +-
 .../server/datanode/TestSimulatedFSDataset.java | 47 ++++++++++++++++++++
 3 files changed, 52 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6702e7d6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index dfa9701..c451ea6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2567,6 +2567,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-9605. Add links to failed volumes to explorer.html in HDFS Web UI.
     (Archana T via wheat9)
 
+    HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the
+    correct namenode (Wei-Chiu Chuang via vinayakumarb)
+
 Release 2.7.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6702e7d6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 4316854..7a10379 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -29,6 +29,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 
 import javax.management.NotCompliantMBeanException;
 import javax.management.ObjectName;
@@ -521,7 +522,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi>
{
   }
 
   private final Map<String, Map<Block, BInfo>> blockMap
-      = new HashMap<String, Map<Block,BInfo>>();
+      = new ConcurrentHashMap<String, Map<Block,BInfo>>();
   private final SimulatedStorage storage;
   private final SimulatedVolume volume;
   private final String datanodeUuid;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6702e7d6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
index 8dc80d5..dd7d239 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
@@ -27,6 +27,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 
+import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -38,6 +39,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory;
 import org.apache.hadoop.util.DataChecksum;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -338,4 +340,49 @@ public class TestSimulatedFSDataset {
     fsdataset.addBlockPool(bpid, conf);
     return fsdataset;
   }
+
+  @Test
+  public void testConcurrentAddBlockPool() throws InterruptedException,
+      IOException {
+    final String[] bpids = {"BP-TEST1-", "BP-TEST2-"};
+    final SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, conf);
+    class AddBlockPoolThread extends Thread {
+      private int id;
+      private IOException ioe;
+      public AddBlockPoolThread(int id) {
+        super();
+        this.id = id;
+      }
+      public void test() throws InterruptedException, IOException {
+        this.join();
+        if (ioe != null) {
+          throw ioe;
+        }
+      }
+      public void run() {
+        for (int i=0; i < 10000; i++) {
+          // add different block pools concurrently
+          String newbpid = bpids[id] + i;
+          fsdataset.addBlockPool(newbpid, conf);
+          // and then add a block into the pool
+          ExtendedBlock block = new ExtendedBlock(newbpid,1);
+          try {
+            // it will throw an exception if the block pool is not found
+            fsdataset.createTemporary(StorageType.DEFAULT, block);
+          } catch (IOException ioe) {
+            // JUnit does not capture exception in non-main thread,
+            // so cache it and then let main thread throw later.
+            this.ioe = ioe;
+          }
+          assert(fsdataset.getReplicaString(newbpid,1) != "null");
+        }
+      }
+    };
+    AddBlockPoolThread t1 = new AddBlockPoolThread(0);
+    AddBlockPoolThread t2 = new AddBlockPoolThread(1);
+    t1.start();
+    t2.start();
+    t1.test();
+    t2.test();
+  }
 }


Mime
View raw message