Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id EDAD018CCC for ; Thu, 4 Feb 2016 19:22:21 +0000 (UTC) Received: (qmail 31063 invoked by uid 500); 4 Feb 2016 19:22:21 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 30992 invoked by uid 500); 4 Feb 2016 19:22:21 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 30982 invoked by uid 99); 4 Feb 2016 19:22:21 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 04 Feb 2016 19:22:21 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 93057DFCE4; Thu, 4 Feb 2016 19:22:21 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: kihwal@apache.org To: common-commits@hadoop.apache.org Message-Id: <86017635701d49f6806cc03c6bf76841@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-9730. Storage ID update does not happen when there is a layout change. Contributed by Tsz Wo Nicholas Sze. Date: Thu, 4 Feb 2016 19:22:21 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.8 adadc220f -> 609021187 HDFS-9730. Storage ID update does not happen when there is a layout change. Contributed by Tsz Wo Nicholas Sze. (cherry picked from commit 496f33de0ce80dc455cfd51f19612da6f9b914f9) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60902118 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60902118 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60902118 Branch: refs/heads/branch-2.8 Commit: 60902118723dac6d81dfa54287807c21d1d8774e Parents: adadc22 Author: Kihwal Lee Authored: Thu Feb 4 13:22:05 2016 -0600 Committer: Kihwal Lee Committed: Thu Feb 4 13:22:05 2016 -0600 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hdfs/server/datanode/DataStorage.java | 47 +++++++++----------- 2 files changed, 23 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/60902118/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7e7b32e..27de25f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1791,6 +1791,9 @@ Release 2.7.3 - UNRELEASED HDFS-9740. Use a reasonable limit in DFSTestUtil.waitForMetric() (Chang Li via vinayakumarb) + HDFS-9730. Storage ID update does not happen when there is a layout change + (Tsz Wo Nicholas Sze via kihwal) + Release 2.7.2 - 2016-01-25 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/60902118/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index 57bb8b2..7903194 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -518,11 +518,7 @@ public class DataStorage extends Storage { this.cTime = 0; setDatanodeUuid(datanodeUuid); - if (sd.getStorageUuid() == null) { - // Assign a new Storage UUID. - sd.setStorageUuid(DatanodeStorage.generateUuid()); - } - + createStorageID(sd, false); writeProperties(sd); } @@ -696,7 +692,13 @@ public class DataStorage extends Storage { // do upgrade if (this.layoutVersion > HdfsServerConstants.DATANODE_LAYOUT_VERSION) { - doUpgrade(sd, nsInfo, conf); // upgrade + if (federationSupported) { + // If the existing on-disk layout version supports federation, + // simply update the properties. + upgradeProperties(sd); + } else { + doUpgradePreFederation(sd, nsInfo, conf); + } return true; // doUgrade already has written properties } @@ -710,7 +712,8 @@ public class DataStorage extends Storage { } /** - * Upgrade -- Move current storage into a backup directory, + * Upgrade from a pre-federation layout. + * Move current storage into a backup directory, * and hardlink all its blocks into the new current directory. * * Upgrade from pre-0.22 to 0.22 or later release e.g. 0.19/0.20/ => 0.22/0.23 @@ -729,25 +732,9 @@ public class DataStorage extends Storage { * There should be only ONE namenode in the cluster for first * time upgrade to 0.22 * @param sd storage directory - * @throws IOException on error */ - void doUpgrade(final StorageDirectory sd, final NamespaceInfo nsInfo, - final Configuration conf) throws IOException { - // If the existing on-disk layout version supportes federation, simply - // update its layout version. - if (DataNodeLayoutVersion.supports( - LayoutVersion.Feature.FEDERATION, layoutVersion)) { - // The VERSION file is already read in. Override the layoutVersion - // field and overwrite the file. The upgrade work is handled by - // {@link BlockPoolSliceStorage#doUpgrade} - LOG.info("Updating layout version from " + layoutVersion + " to " - + HdfsServerConstants.DATANODE_LAYOUT_VERSION + " for storage " - + sd.getRoot()); - layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; - writeProperties(sd); - return; - } - + void doUpgradePreFederation(final StorageDirectory sd, + final NamespaceInfo nsInfo, final Configuration conf) throws IOException { final int oldLV = getLayoutVersion(); LOG.info("Upgrading storage directory " + sd.getRoot() + ".\n old LV = " + oldLV @@ -791,15 +778,21 @@ public class DataStorage extends Storage { linkAllBlocks(tmpDir, bbwDir, toDir, oldLV, conf); // 4. Write version file under /current - layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; clusterID = nsInfo.getClusterID(); - writeProperties(sd); + upgradeProperties(sd); // 5. Rename /previous.tmp to /previous rename(tmpDir, prevDir); LOG.info("Upgrade of " + sd.getRoot()+ " is complete"); + } + void upgradeProperties(StorageDirectory sd) throws IOException { createStorageID(sd, layoutVersion); + LOG.info("Updating layout version from " + layoutVersion + + " to " + HdfsServerConstants.DATANODE_LAYOUT_VERSION + + " for storage " + sd.getRoot()); + layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; + writeProperties(sd); } /**