hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kih...@apache.org
Subject svn commit: r1593471 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/test/java/org/apache/hadoop/hdfs/ src/test/java/org/apache/hadoop/hdfs/web/
Date Fri, 09 May 2014 01:48:28 GMT
Author: kihwal
Date: Fri May  9 01:48:28 2014
New Revision: 1593471

URL: http://svn.apache.org/r1593471
Log:
svn merge -c 1593470 merging from trunk to branch-2 to fix:HDFS-6329. WebHdfs does not work
if HA is enabled on NN but logical URI is not configured. Contributed by Kihwal Lee.

Modified:
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1593471&r1=1593470&r2=1593471&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri May  9
01:48:28 2014
@@ -186,6 +186,8 @@ Release 2.5.0 - UNRELEASED
     HDFS-5381. ExtendedBlock#hashCode should use both blockId and block pool ID    
     (Benoy Antony via Colin Patrick McCabe)
 
+    HDFS-6329. WebHdfs does not work if HA is enabled on NN but logical URI is
+    not configured. (kihwal)
 
 Release 2.4.1 - UNRELEASED
 

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1593471&r1=1593470&r2=1593471&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Fri May  9 01:48:28 2014
@@ -272,10 +272,11 @@ public class NameNode implements NameNod
   private JvmPauseMonitor pauseMonitor;
   private ObjectName nameNodeStatusBeanName;
   /**
-   * The service name of the delegation token issued by the namenode. It is
-   * the name service id in HA mode, or the rpc address in non-HA mode.
+   * The namenode address that clients will use to access this namenode
+   * or the name service. For HA configurations using logical URI, it
+   * will be the logical address.
    */
-  private String tokenServiceName;
+  private String clientNamenodeAddress;
   
   /** Format a new filesystem.  Destroys any filesystem that may already
    * exist at this location.  **/
@@ -318,7 +319,54 @@ public class NameNode implements NameNod
    *
    * @return The name service id in HA-mode, or the rpc address in non-HA mode
    */
-  public String getTokenServiceName() { return tokenServiceName; }
+  public String getTokenServiceName() {
+    return getClientNamenodeAddress();
+  }
+
+  /**
+   * Set the namenode address that will be used by clients to access this
+   * namenode or name service. This needs to be called before the config
+   * is overriden.
+   */
+  public void setClientNamenodeAddress(Configuration conf) {
+    String nnAddr = conf.get(FS_DEFAULT_NAME_KEY);
+    if (nnAddr == null) {
+      // default fs is not set.
+      clientNamenodeAddress = null;
+      return;
+    }
+
+    LOG.info(FS_DEFAULT_NAME_KEY + " is " + nnAddr);
+    URI nnUri = URI.create(nnAddr);
+
+    String nnHost = nnUri.getHost();
+    if (nnHost == null) {
+      clientNamenodeAddress = null;
+      return;
+    }
+
+    if (DFSUtil.getNameServiceIds(conf).contains(nnHost)) {
+      // host name is logical
+      clientNamenodeAddress = nnHost;
+    } else if (nnUri.getPort() > 0) {
+      // physical address with a valid port
+      clientNamenodeAddress = nnUri.getAuthority();
+    } else {
+      // the port is missing or 0. Figure out real bind address later.
+      clientNamenodeAddress = null;
+      return;
+    }
+    LOG.info("Clients are to use " + clientNamenodeAddress + " to access"
+        + " this namenode/service.");
+  }
+
+  /**
+   * Get the namenode address to be used by clients.
+   * @return nn address
+   */
+  public String getClientNamenodeAddress() {
+    return clientNamenodeAddress;
+  }
 
   public static InetSocketAddress getAddress(String address) {
     return NetUtils.createSocketAddr(address, DEFAULT_PORT);
@@ -534,9 +582,14 @@ public class NameNode implements NameNod
     loadNamesystem(conf);
 
     rpcServer = createRpcServer(conf);
-    final String nsId = getNameServiceId(conf);
-    tokenServiceName = HAUtil.isHAEnabled(conf, nsId) ? nsId : NetUtils
-            .getHostPortString(rpcServer.getRpcAddress());
+    if (clientNamenodeAddress == null) {
+      // This is expected for MiniDFSCluster. Set it now using 
+      // the RPC server's bind address.
+      clientNamenodeAddress = 
+          NetUtils.getHostPortString(rpcServer.getRpcAddress());
+      LOG.info("Clients are to use " + clientNamenodeAddress + " to access"
+          + " this namenode/service.");
+    }
     if (NamenodeRole.NAMENODE == role) {
       httpServer.setNameNodeAddress(getNameNodeAddress());
       httpServer.setFSImage(getFSImage());
@@ -682,6 +735,7 @@ public class NameNode implements NameNod
       throws IOException { 
     this.conf = conf;
     this.role = role;
+    setClientNamenodeAddress(conf);
     String nsId = getNameServiceId(conf);
     String namenodeId = HAUtil.getNameNodeId(conf, nsId);
     this.haEnabled = HAUtil.isHAEnabled(conf, nsId);

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1593471&r1=1593470&r2=1593471&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
Fri May  9 01:48:28 2014
@@ -139,8 +139,8 @@ public class DFSTestUtil {
     String clusterId = StartupOption.FORMAT.getClusterId();
     if(clusterId == null || clusterId.isEmpty())
       StartupOption.FORMAT.setClusterId("testClusterID");
-
-    NameNode.format(conf);
+    // Use a copy of conf as it can be altered by namenode during format.
+    NameNode.format(new Configuration(conf));
   }
 
   /**

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1593471&r1=1593470&r2=1593471&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
Fri May  9 01:48:28 2014
@@ -762,8 +762,11 @@ public class MiniDFSCluster {
 
     if (!federation && nnTopology.countNameNodes() == 1) {
       NNConf onlyNN = nnTopology.getOnlyNameNode();
-      // we only had one NN, set DEFAULT_NAME for it
-      conf.set(FS_DEFAULT_NAME_KEY, "127.0.0.1:" + onlyNN.getIpcPort());
+      // we only had one NN, set DEFAULT_NAME for it. If not explicitly
+      // specified initially, the port will be 0 to make NN bind to any
+      // available port. It will be set to the right address after
+      // NN is started.
+      conf.set(FS_DEFAULT_NAME_KEY, "hdfs://127.0.0.1:" + onlyNN.getIpcPort());
     }
     
     List<String> allNsIds = Lists.newArrayList();
@@ -779,6 +782,7 @@ public class MiniDFSCluster {
     int nnCounter = 0;
     for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) {
       String nsId = nameservice.getId();
+      String lastDefaultFileSystem = null;
       
       Preconditions.checkArgument(
           !federation || nsId != null,
@@ -862,10 +866,19 @@ public class MiniDFSCluster {
       for (NNConf nn : nameservice.getNNs()) {
         initNameNodeConf(conf, nsId, nn.getNnId(), manageNameDfsDirs,
             enableManagedDfsDirsRedundancy, nnCounter);
-        createNameNode(nnCounter++, conf, numDataNodes, false, operation,
+        createNameNode(nnCounter, conf, numDataNodes, false, operation,
             clusterId, nsId, nn.getNnId());
+        // Record the last namenode uri
+        if (nameNodes[nnCounter] != null && nameNodes[nnCounter].conf != null) {
+          lastDefaultFileSystem =
+              nameNodes[nnCounter].conf.get(FS_DEFAULT_NAME_KEY);
+        }
+        nnCounter++;
+      }
+      if (!federation && lastDefaultFileSystem != null) {
+        // Set the default file system to the actual bind address of NN.
+        conf.set(FS_DEFAULT_NAME_KEY, lastDefaultFileSystem);
       }
-      
     }
 
   }
@@ -979,7 +992,8 @@ public class MiniDFSCluster {
       operation.setClusterId(clusterId);
     }
     
-    // Start the NameNode
+    // Start the NameNode after saving the default file system.
+    String originalDefaultFs = conf.get(FS_DEFAULT_NAME_KEY);
     String[] args = createArgs(operation);
     NameNode nn =  NameNode.createNameNode(args, conf);
     if (operation == StartupOption.RECOVER) {
@@ -1003,6 +1017,12 @@ public class MiniDFSCluster {
         DFS_NAMENODE_HTTP_ADDRESS_KEY);
     nameNodes[nnIndex] = new NameNodeInfo(nn, nameserviceId, nnId,
         operation, new Configuration(conf));
+    // Restore the default fs name
+    if (originalDefaultFs == null) {
+      conf.set(FS_DEFAULT_NAME_KEY, "");
+    } else {
+      conf.set(FS_DEFAULT_NAME_KEY, originalDefaultFs);
+    }
   }
 
   /**

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java?rev=1593471&r1=1593470&r2=1593471&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java
Fri May  9 01:48:28 2014
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.token.Token;
@@ -37,6 +38,7 @@ import org.junit.Test;
 import java.io.IOException;
 import java.net.URI;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 
@@ -119,6 +121,8 @@ public class TestWebHDFSForHA {
   @Test
   public void testFailoverAfterOpen() throws IOException {
     Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME);
+    conf.set(FS_DEFAULT_NAME_KEY, HdfsConstants.HDFS_URI_SCHEME +
+        "://" + LOGICAL_NAME);
     MiniDFSCluster cluster = null;
     FileSystem fs = null;
     final Path p = new Path("/test");
@@ -152,4 +156,4 @@ public class TestWebHDFSForHA {
       }
     }
   }
-}
\ No newline at end of file
+}



Mime
View raw message