hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jiten...@apache.org
Subject svn commit: r1074724 - in /hadoop/hdfs/branches/HDFS-1052: ./ src/java/org/apache/hadoop/hdfs/ src/java/org/apache/hadoop/hdfs/server/datanode/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/hdfs/
Date Fri, 25 Feb 2011 22:30:39 GMT
Author: jitendra
Date: Fri Feb 25 22:30:38 2011
New Revision: 1074724

URL: http://svn.apache.org/viewvc?rev=1074724&view=rev
Log:
Federation: Multiple namenode configuration.

Added:
    hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java
Modified:
    hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java

Modified: hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/CHANGES.txt?rev=1074724&r1=1074723&r2=1074724&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/CHANGES.txt (original)
+++ hadoop/hdfs/branches/HDFS-1052/CHANGES.txt Fri Feb 25 22:30:38 2011
@@ -37,6 +37,8 @@ Trunk (unreleased changes)
     HDFS-1638.DataNode.handleDiskError needs to inform ALL namenodes if a disk 
     failed (boryas)
 
+    HDFS-1647. Federation: Multiple namenode configuration. (jitendra)
+
   IMPROVEMENTS
 
     HDFS-1510. Added test-patch.properties required by test-patch.sh (nigel)

Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1074724&r1=1074723&r2=1074724&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java Fri Feb 25
22:30:38 2011
@@ -18,19 +18,32 @@
 
 package org.apache.hadoop.hdfs;
 
+import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
 import java.util.StringTokenizer;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.net.NodeBase;
 
 @InterfaceAudience.Private
 public class DFSUtil {
+  
+  final private static String PRIMARY_NAMENODE_SUFFIX = "-primary.namenode";
+  
   /**
    * Whether the pathname is valid.  Currently prohibits relative paths, 
    * and names which contain a ":" or "/" 
@@ -226,6 +239,72 @@ public class DFSUtil {
     return blkLocations;
   }
 
+  private static URI getDefaultNamenode(Configuration conf) {
+    return FileSystem.getDefaultUri(conf);
+  }
+  
+  /**
+   * Returns the list of namenode URIs
+   * 
+   * @param conf
+   * @return list of namenode URIs
+   */
+  public static List<URI> getNamenodeList(Configuration conf) {
+    Collection<String> namenodes = conf
+        .getStringCollection(DFSConfigKeys.DFS_FEDERATION_NAMENODES);
+
+    List<URI> namenodeUris = new ArrayList<URI>();
+    if (namenodes.isEmpty()) {
+      namenodeUris.add(getDefaultNamenode(conf));
+    } else {
+      Iterator<String> nIter = namenodes.iterator();
+      while (nIter.hasNext()) {
+        namenodeUris.add(URI.create(nIter.next()));
+      }
+    }
+    return namenodeUris;
+  }
 
+  /**
+   * Returns the hostname of the primary namenode for a given secondary
+   * namenode.
+   * 
+   * @param conf
+   * @param secondaryHostName
+   *          hostname of the secondary namenode.
+   * @return primary namenode hostname
+   */
+  public static URI getPrimaryNamenode(Configuration conf,
+      String secondaryHostName) {
+    String key = secondaryHostName + PRIMARY_NAMENODE_SUFFIX;
+    String nn = conf.get(key);
+    if (nn == null) {
+      return getDefaultNamenode(conf);
+    } else {
+      return URI.create(nn);
+    }
+  }
+  
+  /**
+   * Returns the InetSocketAddresses for each configured namenode
+   * @param conf
+   * @return Array of InetSocketAddresses
+   * @throws IOException
+   */
+  public static InetSocketAddress[] getNNAddresses(Configuration conf)
+      throws IOException {
+    List<URI> nns = getNamenodeList(conf);
+    if (nns == null) {
+      throw new IOException("Federation namnodes are not configured correctly");
+    }
+
+    InetSocketAddress[] isas = new InetSocketAddress[nns.size()];
+    int i = 0;
+    for (URI u : nns) {
+      isas[i++] = NameNode.getAddress(u);
+    }
+    return isas;
+  }
+  
 }
 

Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1074724&r1=1074723&r2=1074724&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
Fri Feb 25 22:30:38 2011
@@ -64,6 +64,7 @@ import org.apache.hadoop.fs.LocalFileSys
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.HDFSPolicyProvider;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -1143,7 +1144,7 @@ public class DataNode extends Configured
       return nameNodeThreads; // already initialized
     
     // get NNs addresses from the configuration
-    InetSocketAddress[] isas = NameNode.getNNAddresses(conf);
+    InetSocketAddress[] isas = DFSUtil.getNNAddresses(conf);
 
     AbstractList<BPOfferService> al = new ArrayList<BPOfferService> (isas.length);
     for(InetSocketAddress isa : isas) {

Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1074724&r1=1074723&r2=1074724&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Fri Feb 25 22:30:38 2011
@@ -221,51 +221,6 @@ public class NameNode implements Namenod
   }
   
   /**
-   * TODO:FEDERATION
-   * at this moment only support fs.default style enteries.
-   * @param conf
-   * @return array of namenodes' addresses
-   */
-  public static InetSocketAddress [] getNNAddresses(Configuration conf) 
-  throws IOException {
-    URI[] nns=getNameNodesURIs(conf);
-    if(nns == null) {
-      throw new IOException("Federation namnodes are not configured correctly");
-    }
-
-    InetSocketAddress [] isas = new InetSocketAddress[nns.length];
-    int i=0;
-    for(URI u : nns) {
-      isas[i++] = getAddress(u); 
-    }
-    return isas;
-  }
-
-  /**
-   * TODO:FEDERATION
-   * get the list of namenodes from the configuration
-   * create URI for each one of them
-   * @param conf
-   * @return list of URIs of all configured NameNodes
-   */
-  public static URI [] getNameNodesURIs(Configuration conf) {
-    String [] nnURIs = conf.getStrings(DFSConfigKeys.DFS_FEDERATION_NAMENODES);
-    if(nnURIs == null) {
-      nnURIs = new String[] {conf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY)};
-    }
-
-    AbstractList<URI> nns = new ArrayList<URI>(nnURIs.length);
-    for(String uri : nnURIs) {
-      // name should be prepened with FileSystem.fixName(uri)  
-      // TBD
-      nns.add(URI.create(uri));
-    }
-
-    URI[] r = new URI[nns.size()];
-    return nns.toArray(r);
-  }
-
-  /**
    * Set the configuration property for the service rpc address
    * to address
    */

Added: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1074724&view=auto
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java (added)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java Fri
Feb 25 22:30:38 2011
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import static org.junit.Assert.assertEquals;
+
+import java.net.URI;
+import java.util.Iterator;
+import java.util.List;
+
+import org.junit.Test;
+
+
+public class TestDFSUtil {
+  
+  @Test
+  public void testMultipleNamenodes() {
+    HdfsConfiguration conf = new HdfsConfiguration();
+    conf.set(DFSConfigKeys.DFS_FEDERATION_NAMENODES, "nn1,nn2");
+    conf.set("sn1-primary.namenode", "nn1");
+    conf.set("sn2-primary.namenode", "nn2");
+    List<URI> namenodes = DFSUtil.getNamenodeList(conf);
+    Iterator<URI> it = namenodes.iterator();
+    assertEquals("nn1", it.next().toString());
+    assertEquals("nn2", it.next().toString());
+    assertEquals("nn1", DFSUtil.getPrimaryNamenode(conf, "sn1").toString());
+    assertEquals("nn2", DFSUtil.getPrimaryNamenode(conf, "sn2").toString());
+  }
+  
+  @Test
+  public void testDefaultNamenode() {
+    HdfsConfiguration conf = new HdfsConfiguration();
+    final String hdfs_default = "hdfs://x.y.z:9999/";
+    conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, hdfs_default);
+    List<URI> namenodes = DFSUtil.getNamenodeList(conf);
+    assertEquals(1, namenodes.size());
+    Iterator<URI> it = namenodes.iterator();
+    assertEquals(hdfs_default, it.next().toString());
+    assertEquals(hdfs_default, DFSUtil.getPrimaryNamenode(conf, "sn1")
+        .toString());
+  }
+}



Mime
View raw message