hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kih...@apache.org
Subject hadoop git commit: HDFS-9730. Storage ID update does not happen when there is a layout change. Contributed by Tsz Wo Nicholas Sze.
Date Thu, 04 Feb 2016 19:22:21 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 adadc220f -> 609021187


HDFS-9730. Storage ID update does not happen when there is a layout change. Contributed by
 Tsz Wo Nicholas Sze.

(cherry picked from commit 496f33de0ce80dc455cfd51f19612da6f9b914f9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60902118
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60902118
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60902118

Branch: refs/heads/branch-2.8
Commit: 60902118723dac6d81dfa54287807c21d1d8774e
Parents: adadc22
Author: Kihwal Lee <kihwal@apache.org>
Authored: Thu Feb 4 13:22:05 2016 -0600
Committer: Kihwal Lee <kihwal@apache.org>
Committed: Thu Feb 4 13:22:05 2016 -0600

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 ++
 .../hdfs/server/datanode/DataStorage.java       | 47 +++++++++-----------
 2 files changed, 23 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/60902118/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7e7b32e..27de25f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1791,6 +1791,9 @@ Release 2.7.3 - UNRELEASED
     HDFS-9740. Use a reasonable limit in DFSTestUtil.waitForMetric()
     (Chang Li via vinayakumarb)
 
+    HDFS-9730. Storage ID update does not happen when there is a layout change
+    (Tsz Wo Nicholas Sze via kihwal)
+
 Release 2.7.2 - 2016-01-25
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/60902118/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 57bb8b2..7903194 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -518,11 +518,7 @@ public class DataStorage extends Storage {
     this.cTime = 0;
     setDatanodeUuid(datanodeUuid);
 
-    if (sd.getStorageUuid() == null) {
-      // Assign a new Storage UUID.
-      sd.setStorageUuid(DatanodeStorage.generateUuid());
-    }
-
+    createStorageID(sd, false);
     writeProperties(sd);
   }
 
@@ -696,7 +692,13 @@ public class DataStorage extends Storage {
 
     // do upgrade
     if (this.layoutVersion > HdfsServerConstants.DATANODE_LAYOUT_VERSION) {
-      doUpgrade(sd, nsInfo, conf);  // upgrade
+      if (federationSupported) {
+        // If the existing on-disk layout version supports federation,
+        // simply update the properties.
+        upgradeProperties(sd);
+      } else {
+        doUpgradePreFederation(sd, nsInfo, conf);
+      }
       return true; // doUgrade already has written properties
     }
     
@@ -710,7 +712,8 @@ public class DataStorage extends Storage {
   }
 
   /**
-   * Upgrade -- Move current storage into a backup directory,
+   * Upgrade from a pre-federation layout.
+   * Move current storage into a backup directory,
    * and hardlink all its blocks into the new current directory.
    * 
    * Upgrade from pre-0.22 to 0.22 or later release e.g. 0.19/0.20/ => 0.22/0.23
@@ -729,25 +732,9 @@ public class DataStorage extends Storage {
    * There should be only ONE namenode in the cluster for first 
    * time upgrade to 0.22
    * @param sd  storage directory
-   * @throws IOException on error
    */
-  void doUpgrade(final StorageDirectory sd, final NamespaceInfo nsInfo,
-      final Configuration conf) throws IOException {
-    // If the existing on-disk layout version supportes federation, simply
-    // update its layout version.
-    if (DataNodeLayoutVersion.supports(
-        LayoutVersion.Feature.FEDERATION, layoutVersion)) {
-      // The VERSION file is already read in. Override the layoutVersion 
-      // field and overwrite the file. The upgrade work is handled by
-      // {@link BlockPoolSliceStorage#doUpgrade}
-      LOG.info("Updating layout version from " + layoutVersion + " to "
-          + HdfsServerConstants.DATANODE_LAYOUT_VERSION + " for storage "
-          + sd.getRoot());
-      layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
-      writeProperties(sd);
-      return;
-    }
-    
+  void doUpgradePreFederation(final StorageDirectory sd,
+      final NamespaceInfo nsInfo, final Configuration conf) throws IOException {
     final int oldLV = getLayoutVersion();
     LOG.info("Upgrading storage directory " + sd.getRoot()
              + ".\n   old LV = " + oldLV
@@ -791,15 +778,21 @@ public class DataStorage extends Storage {
     linkAllBlocks(tmpDir, bbwDir, toDir, oldLV, conf);
 
     // 4. Write version file under <SD>/current
-    layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
     clusterID = nsInfo.getClusterID();
-    writeProperties(sd);
+    upgradeProperties(sd);
     
     // 5. Rename <SD>/previous.tmp to <SD>/previous
     rename(tmpDir, prevDir);
     LOG.info("Upgrade of " + sd.getRoot()+ " is complete");
+  }
 
+  void upgradeProperties(StorageDirectory sd) throws IOException {
     createStorageID(sd, layoutVersion);
+    LOG.info("Updating layout version from " + layoutVersion
+        + " to " + HdfsServerConstants.DATANODE_LAYOUT_VERSION
+        + " for storage " + sd.getRoot());
+    layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
+    writeProperties(sd);
   }
 
   /**


Mime
View raw message