hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1104395 - in /hadoop/hdfs/trunk: CHANGES.txt src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java
Date Tue, 17 May 2011 16:46:06 GMT
Author: suresh
Date: Tue May 17 16:46:05 2011
New Revision: 1104395

URL: http://svn.apache.org/viewvc?rev=1104395&view=rev
Log:
HDFS-1914. Federation: namenode storage directories must be configurable specific to name
service. Contributed by Suresh Srinivas.


Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1104395&r1=1104394&r2=1104395&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Tue May 17 16:46:05 2011
@@ -278,6 +278,9 @@ Trunk (unreleased changes)
     HDFS-1814. Add "hdfs groups" command to query the server-side groups
     resolved for a user. (Aaron T. Myers via todd)
 
+    HDFS-1914. Federation: namenode storage directories must be configurable
+    specific to name service. (suresh)
+
 
   IMPROVEMENTS
 

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1104395&r1=1104394&r2=1104395&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Tue May
17 16:46:05 2011
@@ -146,6 +146,34 @@ public class NameNode implements Namenod
     Configuration.addDefaultResource("hdfs-site.xml");
   }
   
+  /**
+   * HDFS federation configuration can have two types of parameters:
+   * <ol>
+   * <li>Parameter that is common for all the name services in the cluster.</li>
+   * <li>Parameters that are specific to a name service. This keys are suffixed
+   * with nameserviceId in the configuration. For example,
+   * "dfs.namenode.rpc-address.nameservice1".</li>
+   * </ol>
+   * 
+   * Following are nameservice specific keys.
+   */
+  public static final String[] NAMESERVICE_SPECIFIC_KEYS = {
+    DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY,
+    DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+    DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
+    DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY,
+    DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_EDITS_DIR_KEY,
+    DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
+    DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
+    DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY,
+    DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY,
+    DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
+    DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY,
+    DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY,
+    DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY,
+    DFSConfigKeys.DFS_NAMENODE_BACKUP_SERVICE_RPC_ADDRESS_KEY
+  };
+  
   public long getProtocolVersion(String protocol, 
                                  long clientVersion) throws IOException {
     if (protocol.equals(ClientProtocol.class.getName())) {
@@ -1662,25 +1690,15 @@ public class NameNode implements Namenod
    * @param conf
    *          Configuration object to lookup specific key and to set the value
    *          to the key passed. Note the conf object is modified
-   * @see DFSUtil#setGenericConf()
+   * @see DFSUtil#setGenericConf(Configuration, String, String...)
    */
-  static void initializeGenericKeys(Configuration conf) {
+  public static void initializeGenericKeys(Configuration conf) {
     final String nameserviceId = DFSUtil.getNameServiceId(conf);
     if ((nameserviceId == null) || nameserviceId.isEmpty()) {
       return;
     }
     
-    DFSUtil.setGenericConf(conf, nameserviceId,
-        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY,
-        DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
-        DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
-        DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY,
-        DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY,
-        DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
-        DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY,
-        DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY,
-        DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY,
-        DFSConfigKeys.DFS_NAMENODE_BACKUP_SERVICE_RPC_ADDRESS_KEY);
+    DFSUtil.setGenericConf(conf, nameserviceId, NAMESERVICE_SPECIFIC_KEYS);
     
     if (conf.get(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY) != null) {
       URI defaultUri = URI.create(FSConstants.HDFS_URI_SCHEME + "://"

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1104395&r1=1104394&r2=1104395&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java Tue May 17 16:46:05
2011
@@ -20,16 +20,22 @@ package org.apache.hadoop.hdfs;
 
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
 
+import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
 import java.util.List;
 
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
 
 public class TestDFSUtil {
@@ -72,4 +78,159 @@ public class TestDFSUtil {
     bs = DFSUtil.locatedBlocks2Locations(new LocatedBlocks());
     assertEquals(0, bs.length);
   }
+
+  /** 
+   * Test for
+   * {@link DFSUtil#getNameServiceIds(Configuration)}
+   * {@link DFSUtil#getNameServiceId(Configuration)}
+   * {@link DFSUtil#getNNServiceRpcAddresses(Configuration)}
+   */
+  @Test
+  public void testMultipleNamenodes() throws IOException {
+    HdfsConfiguration conf = new HdfsConfiguration();
+    conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, "nn1,nn2");
+    
+    // Test - The configured nameserviceIds are returned
+    Collection<String> nameserviceIds = DFSUtil.getNameServiceIds(conf);
+    Iterator<String> it = nameserviceIds.iterator();
+    assertEquals(2, nameserviceIds.size());
+    assertEquals("nn1", it.next().toString());
+    assertEquals("nn2", it.next().toString());
+    
+    // Tests default nameserviceId is returned
+    conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICE_ID, "nn1");
+    assertEquals("nn1", DFSUtil.getNameServiceId(conf));
+    
+    // Test - configured list of namenodes are returned
+    final String NN1_ADDRESS = "localhost:9000";
+    final String NN2_ADDRESS = "localhost:9001";
+    final String NN3_ADDRESS = "localhost:9002";
+    conf.set(DFSUtil.getNameServiceIdKey(
+        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, "nn1"), NN1_ADDRESS);
+    conf.set(DFSUtil.getNameServiceIdKey(
+        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, "nn2"), NN2_ADDRESS);
+    
+    Collection<InetSocketAddress> nnAddresses = 
+      DFSUtil.getNNServiceRpcAddresses(conf);
+    assertEquals(2, nnAddresses.size());
+    Iterator<InetSocketAddress> iterator = nnAddresses.iterator();
+    assertEquals(2, nameserviceIds.size());
+    InetSocketAddress addr = iterator.next();
+    assertEquals("localhost", addr.getHostName());
+    assertEquals(9000, addr.getPort());
+    addr = iterator.next();
+    assertEquals("localhost", addr.getHostName());
+    assertEquals(9001, addr.getPort());
+    
+    // Test - can look up nameservice ID from service address
+    InetSocketAddress testAddress1 = NetUtils.createSocketAddr(NN1_ADDRESS);
+    String nameserviceId = DFSUtil.getNameServiceIdFromAddress(
+        conf, testAddress1,
+        DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
+        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
+    assertEquals("nn1", nameserviceId);
+    InetSocketAddress testAddress2 = NetUtils.createSocketAddr(NN2_ADDRESS);
+    nameserviceId = DFSUtil.getNameServiceIdFromAddress(
+        conf, testAddress2,
+        DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
+        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
+    assertEquals("nn2", nameserviceId);
+    InetSocketAddress testAddress3 = NetUtils.createSocketAddr(NN3_ADDRESS);
+    nameserviceId = DFSUtil.getNameServiceIdFromAddress(
+        conf, testAddress3,
+        DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
+        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
+    assertNull(nameserviceId);
+  }
+  
+  /** 
+   * Test for
+   * {@link DFSUtil#isDefaultNamenodeAddress(Configuration, InetSocketAddress, String...)}
+   */
+  @Test
+  public void testSingleNamenode() {
+    HdfsConfiguration conf = new HdfsConfiguration();
+    final String DEFAULT_ADDRESS = "localhost:9000";
+    final String NN2_ADDRESS = "localhost:9001";
+    conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, DEFAULT_ADDRESS);
+    
+    InetSocketAddress testAddress1 = NetUtils.createSocketAddr(DEFAULT_ADDRESS);
+    boolean isDefault = DFSUtil.isDefaultNamenodeAddress(conf, testAddress1,
+        DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
+        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
+    assertTrue(isDefault);
+    InetSocketAddress testAddress2 = NetUtils.createSocketAddr(NN2_ADDRESS);
+    isDefault = DFSUtil.isDefaultNamenodeAddress(conf, testAddress2,
+        DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
+        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
+    assertFalse(isDefault);
+  }
+  
+  /** Tests to ensure default namenode is used as fallback */
+  @Test
+  public void testDefaultNamenode() throws IOException {
+    HdfsConfiguration conf = new HdfsConfiguration();
+    final String hdfs_default = "hdfs://localhost:9999/";
+    conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, hdfs_default);
+    // If DFSConfigKeys.DFS_FEDERATION_NAMESERVICES is not set, verify that 
+    // default namenode address is returned.
+    List<InetSocketAddress> addrList = DFSUtil.getNNServiceRpcAddresses(conf);
+    assertEquals(1, addrList.size());
+    assertEquals(9999, addrList.get(0).getPort());
+  }
+  
+  /**
+   * Test to ensure nameservice specific keys in the configuration are
+   * copied to generic keys when the namenode starts.
+   */
+  @Test
+  public void testConfModification() throws IOException {
+    final HdfsConfiguration conf = new HdfsConfiguration();
+    conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, "nn1");
+    conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICE_ID, "nn1");
+    final String nameserviceId = DFSUtil.getNameServiceId(conf);
+    
+    // Set the nameservice specific keys with nameserviceId in the config key
+    for (String key : NameNode.NAMESERVICE_SPECIFIC_KEYS) {
+      // Note: value is same as the key
+      conf.set(DFSUtil.getNameServiceIdKey(key, nameserviceId), key);
+    }
+    
+    // Initialize generic keys from specific keys
+    NameNode.initializeGenericKeys(conf);
+    
+    // Retrieve the keys without nameserviceId and Ensure generic keys are set
+    // to the correct value
+    for (String key : NameNode.NAMESERVICE_SPECIFIC_KEYS) {
+      assertEquals(key, conf.get(key));
+    }
+  }
+  
+  /**
+   * Tests for empty configuration, an exception is thrown from
+   * {@link DFSUtil#getNNServiceRpcAddresses(Configuration)}
+   * {@link DFSUtil#getBackupNodeAddresses(Configuration)}
+   * {@link DFSUtil#getSecondaryNameNodeAddresses(Configuration)}
+   */
+  @Test
+  public void testEmptyConf() {
+    HdfsConfiguration conf = new HdfsConfiguration(false);
+    try {
+      DFSUtil.getNNServiceRpcAddresses(conf);
+      fail("Expected IOException is not thrown");
+    } catch (IOException expected) {
+    }
+
+    try {
+      DFSUtil.getBackupNodeAddresses(conf);
+      fail("Expected IOException is not thrown");
+    } catch (IOException expected) {
+    }
+
+    try {
+      DFSUtil.getSecondaryNameNodeAddresses(conf);
+      fail("Expected IOException is not thrown");
+    } catch (IOException expected) {
+    }
+  }
 }
\ No newline at end of file



Mime
View raw message