Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 28C50200B57 for ; Sat, 23 Jul 2016 00:12:20 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 27247160A8E; Fri, 22 Jul 2016 22:12:20 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 6D867160A6D for ; Sat, 23 Jul 2016 00:12:19 +0200 (CEST) Received: (qmail 18099 invoked by uid 500); 22 Jul 2016 22:12:18 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 18090 invoked by uid 99); 22 Jul 2016 22:12:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 22 Jul 2016 22:12:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 276A2E058E; Fri, 22 Jul 2016 22:12:18 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xiao@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-10225. DataNode hot swap drives should disallow storage type changes. Contributed by Lei (Eddy) Xu. Date: Fri, 22 Jul 2016 22:12:18 +0000 (UTC) archived-at: Fri, 22 Jul 2016 22:12:20 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2.8 c96cb3fd4 -> 8559442ed HDFS-10225. DataNode hot swap drives should disallow storage type changes. Contributed by Lei (Eddy) Xu. (cherry picked from commit 132deb4cacc413a85a6af2f390ec79a76c91961c) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8559442e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8559442e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8559442e Branch: refs/heads/branch-2.8 Commit: 8559442ed15c2e8a9aa6934ec35a1f4d1157e1a1 Parents: c96cb3f Author: Xiao Chen Authored: Thu Jul 21 16:41:02 2016 -0700 Committer: Xiao Chen Committed: Fri Jul 22 15:04:21 2016 -0700 ---------------------------------------------------------------------- .../hadoop/hdfs/server/datanode/DataNode.java | 14 ++++++++++++- .../datanode/TestDataNodeHotSwapVolumes.java | 22 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8559442e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 5d6514e..cd8212f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -625,7 +625,7 @@ public class DataNode extends ReconfigurableBase * @param newVolumes a comma separated string that specifies the data volumes. * @return changed volumes. * @throws IOException if none of the directories are specified in the - * configuration. + * configuration, or the storage type of a directory is changed. */ @VisibleForTesting ChangedVolumes parseChangedVolumes(String newVolumes) throws IOException { @@ -637,6 +637,12 @@ public class DataNode extends ReconfigurableBase throw new IOException("No directory is specified."); } + // Use the existing StorageLocation to detect storage type changes. + Map existingLocations = new HashMap<>(); + for (StorageLocation loc : getStorageLocations(this.conf)) { + existingLocations.put(loc.getFile().getCanonicalPath(), loc); + } + ChangedVolumes results = new ChangedVolumes(); results.newLocations.addAll(locations); @@ -650,6 +656,12 @@ public class DataNode extends ReconfigurableBase if (location.getFile().getCanonicalPath().equals( dir.getRoot().getCanonicalPath())) { sl.remove(); + StorageLocation old = existingLocations.get( + location.getFile().getCanonicalPath()); + if (old != null && + old.getStorageType() != location.getStorageType()) { + throw new IOException("Changing storage type is not allowed."); + } results.unchangedLocations.add(location); found = true; break; http://git-wip-us.apache.org/repos/asf/hadoop/blob/8559442e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java index 0328e10..c03b02b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.BlockMissingException; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; @@ -255,6 +256,27 @@ public class TestDataNodeHotSwapVolumes { } } + @Test + public void testParseStorageTypeChanges() throws IOException { + startDFSCluster(1, 1); + DataNode dn = cluster.getDataNodes().get(0); + Configuration conf = dn.getConf(); + List oldLocations = DataNode.getStorageLocations(conf); + + // Change storage type of an existing StorageLocation + String newLoc = String.format("[%s]%s", StorageType.SSD, + oldLocations.get(1).getUri()); + String newDataDirs = oldLocations.get(0).toString() + "," + newLoc; + + try { + dn.parseChangedVolumes(newDataDirs); + fail("should throw IOE because storage type changes."); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains( + "Changing storage type is not allowed", e); + } + } + /** Add volumes to the first DataNode. */ private void addVolumes(int numNewVolumes) throws ReconfigurationException, IOException { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org