hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1134138 - in /hadoop/hdfs/trunk: ./ src/java/org/apache/hadoop/hdfs/protocol/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
Date Fri, 10 Jun 2011 00:17:11 GMT
Author: suresh
Date: Fri Jun 10 00:17:10 2011
New Revision: 1134138

URL: http://svn.apache.org/viewvc?rev=1134138&view=rev
Log:
HDFS-2030. Improve usability of namenode -upgrade command. Contributed by Bharath Mundlapudi.


Added:
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStartupOptionUpgrade.java
Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1134138&r1=1134137&r2=1134138&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Fri Jun 10 00:17:10 2011
@@ -510,6 +510,9 @@ Trunk (unreleased changes)
     HDFS-941. The DFS client should cache and reuse open sockets to datanodes
     while performing reads. (bc Wong and Todd Lipcon via todd)
 
+    HDFS-2030. Improve usability of namenode -upgrade command.
+    (Bharath Mundlapudi via suresh)
+
   BUG FIXES
 
     HDFS-1449. Fix test failures - ExtendedBlock must return 

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java?rev=1134138&r1=1134137&r2=1134138&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java Fri Jun
10 00:17:10 2011
@@ -106,6 +106,30 @@ public class LayoutVersion {
       this.ancestorLV = ancestorLV;
       this.description = description;
     }
+    
+    /** 
+     * Accessor method for feature layout version 
+     * @return int lv value
+     */
+    public int getLayoutVersion() {
+      return lv;
+    }
+
+    /** 
+     * Accessor method for feature ancestor layout version 
+     * @return int ancestor LV value
+     */
+    public int getAncestorLayoutVersion() {
+      return ancestorLV;
+    }
+
+    /** 
+     * Accessor method for feature description 
+     * @return String feature description 
+     */
+    public String getDescription() {
+      return description;
+    }
   }
   
   // Build layout version and corresponding feature matrix

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1134138&r1=1134137&r2=1134138&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Fri Jun
10 00:17:10 2011
@@ -267,24 +267,9 @@ public class FSImage implements NNStorag
           + "Please restart NameNode with -upgrade option.");
     }
     
-    // Upgrade to federation requires -upgrade -clusterid <clusterID> option
-    if (startOpt == StartupOption.UPGRADE && 
-        !LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) {
-      if (startOpt.getClusterId() == null) {
-        throw new IOException(
-            "\nFile system image contains an old layout version "
-                + layoutVersion + ".\nAn upgrade to version "
-                + FSConstants.LAYOUT_VERSION
-                + " is required.\nPlease restart NameNode with "
-                + "-upgrade -clusterid <clusterID> option.");
-      }
-      storage.setClusterID(startOpt.getClusterId());
-      
-      // Create new block pool Id
-      storage.setBlockPoolID(storage.newBlockPoolID());
-    }
-    
-    // check whether distributed upgrade is reguired and/or should be continued
+    storage.processStartupOptionsForUpgrade(startOpt, layoutVersion);
+
+    // check whether distributed upgrade is required and/or should be continued
     storage.verifyDistributedUpgradeProgress(startOpt);
 
     // 2. Format unformatted dirs.

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1134138&r1=1134137&r2=1134138&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Fri Jun
10 00:17:10 2011
@@ -932,6 +932,40 @@ public class NNStorage extends Storage i
     LOG.debug("at the end current list of storage dirs:" + lsd);
   }
   
+  /** 
+   * Processes the startup options for the clusterid and blockpoolid 
+   * for the upgrade. 
+   * @param startOpt Startup options 
+   * @param layoutVersion Layout version for the upgrade 
+   * @throws IOException
+   */
+  void processStartupOptionsForUpgrade(StartupOption startOpt, int layoutVersion)
+      throws IOException {
+    if (startOpt == StartupOption.UPGRADE) {
+      // If upgrade from a release that does not support federation,
+      // if clusterId is provided in the startupOptions use it.
+      // Else generate a new cluster ID      
+      if (!LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) {
+        if (startOpt.getClusterId() == null) {
+          startOpt.setClusterId(newClusterID());
+        }
+        setClusterID(startOpt.getClusterId());
+        setBlockPoolID(newBlockPoolID());
+      } else {
+        // Upgrade from one version of federation to another supported
+        // version of federation doesn't require clusterID.
+        // Warn the user if the current clusterid didn't match with the input
+        // clusterid.
+        if (startOpt.getClusterId() != null
+            && !startOpt.getClusterId().equals(getClusterID())) {
+          LOG.warn("Clusterid mismatch - current clusterid: " + getClusterID()
+              + ", Ignoring given clusterid: " + startOpt.getClusterId());
+        }
+      }
+      LOG.info("Using clusterid: " + getClusterID());
+    }
+  }
+  
   /**
    * Generate new clusterID.
    * 

Added: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStartupOptionUpgrade.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStartupOptionUpgrade.java?rev=1134138&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStartupOptionUpgrade.java
(added)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStartupOptionUpgrade.java
Fri Jun 10 00:17:10 2011
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
+import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * This class tests various upgrade cases from earlier versions to current
+ * version with and without clusterid.
+ */
+public class TestStartupOptionUpgrade {
+
+  private Configuration conf;
+  private StartupOption startOpt;
+  private int layoutVersion;
+  NNStorage storage;
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new HdfsConfiguration();
+    startOpt = StartupOption.UPGRADE;
+    startOpt.setClusterId(null);
+    storage = new NNStorage(conf);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    conf = null;
+    startOpt = null;
+  }
+
+  /**
+   * Tests the upgrade from version 0.20.204 to Federation version Test without
+   * clusterid the case: -upgrade 
+   * Expected to generate clusterid
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testStartupOptUpgradeFrom204() throws Exception {
+    layoutVersion = Feature.RESERVED_REL20_204.getLayoutVersion();
+    storage.processStartupOptionsForUpgrade(startOpt, layoutVersion);
+    Assert.assertTrue("Clusterid should start with CID", storage.getClusterID()
+        .startsWith("CID"));
+  }
+
+  /**
+   * Tests the upgrade from version 0.22 to Federation version Test with
+   * clusterid case: -upgrade -clusterid <cid> 
+   * Expected to reuse user given clusterid
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testStartupOptUpgradeFrom22WithCID() throws Exception {
+    startOpt.setClusterId("cid");
+    layoutVersion = Feature.RESERVED_REL22.getLayoutVersion();
+    storage.processStartupOptionsForUpgrade(startOpt, layoutVersion);
+    Assert.assertEquals("Clusterid should match with the given clusterid",
+        "cid", storage.getClusterID());
+  }
+
+  /**
+   * Tests the upgrade from one version of Federation to another Federation
+   * version Test without clusterid case: -upgrade
+   * Expected to reuse existing clusterid
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testStartupOptUpgradeFromFederation()
+      throws Exception {
+    // Test assumes clusterid already exists, set the clusterid
+    storage.setClusterID("currentcid");
+    layoutVersion = Feature.FEDERATION.getLayoutVersion();
+    storage.processStartupOptionsForUpgrade(startOpt, layoutVersion);
+    Assert.assertEquals("Clusterid should match with the existing one",
+        "currentcid", storage.getClusterID());
+  }
+
+  /**
+   * Tests the upgrade from one version of Federation to another Federation
+   * version Test with wrong clusterid case: -upgrade -clusterid <cid> 
+   * Expected to reuse existing clusterid and ignore user given clusterid
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testStartupOptUpgradeFromFederationWithWrongCID()
+      throws Exception {
+    startOpt.setClusterId("wrong-cid");
+    storage.setClusterID("currentcid");
+    layoutVersion = Feature.FEDERATION.getLayoutVersion();
+    storage.processStartupOptionsForUpgrade(startOpt, layoutVersion);
+    Assert.assertEquals("Clusterid should match with the existing one",
+        "currentcid", storage.getClusterID());
+  }
+
+  /**
+   * Tests the upgrade from one version of Federation to another Federation
+   * version Test with correct clusterid case: -upgrade -clusterid <cid>
+   * Expected to reuse existing clusterid and ignore user given clusterid
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testStartupOptUpgradeFromFederationWithCID()
+      throws Exception {
+    startOpt.setClusterId("currentcid");
+    storage.setClusterID("currentcid");
+    layoutVersion = Feature.FEDERATION.getLayoutVersion();
+    storage.processStartupOptionsForUpgrade(startOpt, layoutVersion);
+    Assert.assertEquals("Clusterid should match with the existing one",
+        "currentcid", storage.getClusterID());
+  }
+}
\ No newline at end of file



Mime
View raw message