hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vvasu...@apache.org
Subject [33/50] [abbrv] hadoop git commit: HDFS-10225. DataNode hot swap drives should disallow storage type changes. Contributed by Lei (Eddy) Xu.
Date Mon, 25 Jul 2016 17:26:19 GMT
HDFS-10225. DataNode hot swap drives should disallow storage type changes. Contributed by Lei
(Eddy) Xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/132deb4c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/132deb4c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/132deb4c

Branch: refs/heads/YARN-3926
Commit: 132deb4cacc413a85a6af2f390ec79a76c91961c
Parents: 12aa184
Author: Xiao Chen <xiao@apache.org>
Authored: Thu Jul 21 16:41:02 2016 -0700
Committer: Xiao Chen <xiao@apache.org>
Committed: Thu Jul 21 16:41:02 2016 -0700

----------------------------------------------------------------------
 .../hadoop/hdfs/server/datanode/DataNode.java   | 14 +++++++++++-
 .../datanode/TestDataNodeHotSwapVolumes.java    | 23 +++++++++++++++++++-
 2 files changed, 35 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/132deb4c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 1cd2dee..ac55397 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -634,7 +634,7 @@ public class DataNode extends ReconfigurableBase
    * @param newVolumes a comma separated string that specifies the data volumes.
    * @return changed volumes.
    * @throws IOException if none of the directories are specified in the
-   * configuration.
+   * configuration, or the storage type of a directory is changed.
    */
   @VisibleForTesting
   ChangedVolumes parseChangedVolumes(String newVolumes) throws IOException {
@@ -646,6 +646,12 @@ public class DataNode extends ReconfigurableBase
       throw new IOException("No directory is specified.");
     }
 
+    // Use the existing StorageLocation to detect storage type changes.
+    Map<String, StorageLocation> existingLocations = new HashMap<>();
+    for (StorageLocation loc : getStorageLocations(this.conf)) {
+      existingLocations.put(loc.getFile().getCanonicalPath(), loc);
+    }
+
     ChangedVolumes results = new ChangedVolumes();
     results.newLocations.addAll(locations);
 
@@ -659,6 +665,12 @@ public class DataNode extends ReconfigurableBase
         if (location.getFile().getCanonicalPath().equals(
             dir.getRoot().getCanonicalPath())) {
           sl.remove();
+          StorageLocation old = existingLocations.get(
+              location.getFile().getCanonicalPath());
+          if (old != null &&
+              old.getStorageType() != location.getStorageType()) {
+            throw new IOException("Changing storage type is not allowed.");
+          }
           results.unchangedLocations.add(location);
           found = true;
           break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/132deb4c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index 659806b..c03b02b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.BlockMissingException;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -82,7 +83,6 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.junit.Assume.assumeTrue;
 import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.timeout;
@@ -256,6 +256,27 @@ public class TestDataNodeHotSwapVolumes {
     }
   }
 
+  @Test
+  public void testParseStorageTypeChanges() throws IOException {
+    startDFSCluster(1, 1);
+    DataNode dn = cluster.getDataNodes().get(0);
+    Configuration conf = dn.getConf();
+    List<StorageLocation> oldLocations = DataNode.getStorageLocations(conf);
+
+    // Change storage type of an existing StorageLocation
+    String newLoc = String.format("[%s]%s", StorageType.SSD,
+        oldLocations.get(1).getUri());
+    String newDataDirs = oldLocations.get(0).toString() + "," + newLoc;
+
+    try {
+      dn.parseChangedVolumes(newDataDirs);
+      fail("should throw IOE because storage type changes.");
+    } catch (IOException e) {
+      GenericTestUtils.assertExceptionContains(
+          "Changing storage type is not allowed", e);
+    }
+  }
+
   /** Add volumes to the first DataNode. */
   private void addVolumes(int numNewVolumes)
       throws ReconfigurationException, IOException {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message