hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1098777 - in /hadoop/hdfs/branches/yahoo-merge: ./ src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/ src/java/org/apache/hadoop/hdfs/ src/java/org/apache/hadoop/hdfs/server/datanode/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/jav...
Date Mon, 02 May 2011 20:30:06 GMT
Author: suresh
Date: Mon May  2 20:30:05 2011
New Revision: 1098777

URL: http://svn.apache.org/viewvc?rev=1098777&view=rev
Log:
Merging change r1081603 for HDFS-1754 from federation

Modified:
    hadoop/hdfs/branches/yahoo-merge/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
    hadoop/hdfs/branches/yahoo-merge/build.xml   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DFSUtil.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
  (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/tools/DFSck.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/datanode/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/secondary/   (props changed)

Propchange: hadoop/hdfs/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:30:05 2011
@@ -1,5 +1,5 @@
 /hadoop/core/branches/branch-0.19/hdfs:713112
-/hadoop/hdfs/branches/HDFS-1052:1078943,1080331,1080391,1080402
+/hadoop/hdfs/branches/HDFS-1052:1078943,1080331,1080391,1080402,1081603
 /hadoop/hdfs/branches/HDFS-265:796829-820463
 /hadoop/hdfs/branches/branch-0.21:820487
 /hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1036738,1052823,1060619,1061067,1062020,1062045,1062052

Modified: hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/CHANGES.txt?rev=1098777&r1=1098776&r2=1098777&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/hdfs/branches/yahoo-merge/CHANGES.txt Mon May  2 20:30:05 2011
@@ -221,6 +221,8 @@ Trunk (unreleased changes)
     HDFS-1703. Improve start/stop scripts and add decommission tool for
     federation. (Tanping Wang, Erik Steffl via suresh)
 
+   HDFS-1754. Federation: testFsck fails. (boryas)
+
   IMPROVEMENTS
 
     HDFS-1510. Added test-patch.properties required by test-patch.sh (nigel)

Propchange: hadoop/hdfs/branches/yahoo-merge/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:30:05 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/build.xml:713112
 /hadoop/core/trunk/build.xml:779102
-/hadoop/hdfs/branches/HDFS-1052/build.xml:1078943,1080331,1080391,1080402
+/hadoop/hdfs/branches/HDFS-1052/build.xml:1078943,1080331,1080391,1080402,1081603
 /hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
 /hadoop/hdfs/branches/branch-0.21/build.xml:820487
 /hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Propchange: hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:30:05 2011
@@ -1,4 +1,4 @@
 /hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
 /hadoop/core/trunk/src/c++/libhdfs:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs:1078943,1080331,1080391,1080402
+/hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs:1078943,1080331,1080391,1080402,1081603
 /hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Propchange: hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:30:05 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/contrib/hdfsproxy:713112
 /hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy:1078943,1080331,1080391,1080402
+/hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy:1078943,1080331,1080391,1080402,1081603
 /hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
 /hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:30:05 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/java:713112
 /hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
-/hadoop/hdfs/branches/HDFS-1052/src/java:1078943,1080331,1080391,1080402
+/hadoop/hdfs/branches/HDFS-1052/src/java:1078943,1080331,1080391,1080402,1081603
 /hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java:820487
 /hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1098777&r1=1098776&r2=1098777&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DFSUtil.java Mon May
 2 20:30:05 2011
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.protocol.L
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.net.NodeBase;
+import org.apache.hadoop.security.UserGroupInformation;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
 
 @InterfaceAudience.Private
@@ -437,6 +438,44 @@ public class DFSUtil {
     // client should try {@link isDefaultNamenodeAddress} instead
     return null;
   }
+
+  /**
+   * return HTTP server info from the configuration
+   * @param conf
+   * @param namenode - namenode address
+   * @return http server info
+   */
+  public static String getInfoServer(
+      InetSocketAddress namenode, Configuration conf) {
+    String httpAddress = null;
+    
+    String httpAddressKey = UserGroupInformation.isSecurityEnabled() ?
+        DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY
+        : DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
+    String httpAddressDefault = UserGroupInformation.isSecurityEnabled() ?
+        DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT 
+        :DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT;
+    
+    if(namenode != null) {
+      // if non-default namenode, try reverse look up 
+      // the nameServiceID if it is available
+      String nameServiceId = DFSUtil.getNameServiceIdFromAddress(
+          conf, namenode,
+          DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
+          DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
+
+      if (nameServiceId != null) {
+        httpAddress = conf.get(DFSUtil.getNameServiceIdKey(
+            httpAddressKey, nameServiceId));
+      }
+    }
+    // else - Use non-federation style configuration
+    if (httpAddress == null) {
+      httpAddress = conf.get(httpAddressKey, httpAddressDefault);
+    }
+
+    return httpAddress;
+  }
   
   /**
    * Given the InetSocketAddress for any configured communication with a 

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:30:05 2011
@@ -2,7 +2,7 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:713112
 /hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DatanodeBlockInfo.java:776175-785643,785929-786278
 /hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
-/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1078943,1080331,1080391,1080402
+/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1078943,1080331,1080391,1080402,1081603
 /hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487
 /hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1098777&r1=1098776&r2=1098777&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Mon May  2 20:30:05 2011
@@ -435,14 +435,6 @@ public class NameNode implements Namenod
     this.emptier.setDaemon(true);
     this.emptier.start();
   }
-
-  public static String getInfoServer(Configuration conf) {
-    return UserGroupInformation.isSecurityEnabled() ? conf.get(
-        DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY,
-        DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT) : conf.get(
-        DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
-        DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT);
-  }
   
   private void startHttpServer(final Configuration conf) throws IOException {
     final InetSocketAddress infoSocAddr = getHttpServerAddress(conf);

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1098777&r1=1098776&r2=1098777&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
Mon May  2 20:30:05 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.DFSUtil.ErrorSimulator;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.protocol.FSConstants;
@@ -393,7 +394,7 @@ public class SecondaryNameNode implement
       throw new IOException("This is not a DFS");
     }
 
-    String configuredAddress = NameNode.getInfoServer(conf);
+    String configuredAddress = DFSUtil.getInfoServer(null, conf);
     InetSocketAddress sockAddr = NetUtils.createSocketAddr(configuredAddress);
     if (sockAddr.getAddress().isAnyLocalAddress()) {
       if(UserGroupInformation.isSecurityEnabled()) {

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/tools/DFSck.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/tools/DFSck.java?rev=1098777&r1=1098776&r2=1098777&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/tools/DFSck.java (original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/tools/DFSck.java Mon
May  2 20:30:05 2011
@@ -199,16 +199,16 @@ public class DFSck extends Configured im
     return errCode;
   }
   
-  /*
+  /**
    * Derive the namenode http address from the current file system,
    * either default or as set by "-fs" in the generic options.
-   * Returns null if failure.
+   * @return Returns http address or null if failure.
    */
   private String getCurrentNamenodeAddress() {
-    String nnAddress = null;
+    //String nnAddress = null;
     Configuration conf = getConf();
 
-    //get the filesystem object
+    //get the filesystem object to verify it is an HDFS system
     FileSystem fs;
     try {
       fs = FileSystem.get(conf);
@@ -227,41 +227,10 @@ public class DFSck extends Configured im
     // The URI may have been provided by a human, and the server name may be
     // aliased, so compare InetSocketAddresses instead of URI strings, and
     // test against both possible variants of RPC address.
-    InetSocketAddress namenode = NameNode.getAddress(dfs.getUri().getAuthority());
-    String nameServiceId = DFSUtil.getNameServiceIdFromAddress(
-        conf, namenode,
-        DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
-        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
+    InetSocketAddress namenode = 
+      NameNode.getAddress(dfs.getUri().getAuthority());
     
-    //Look up nameservice-specific http address
-    String httpAddressKey = UserGroupInformation.isSecurityEnabled() ?
-        DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY
-        : DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
-    if (nameServiceId != null) {
-      nnAddress = conf.get(DFSUtil.getNameServiceIdKey(
-          httpAddressKey, nameServiceId));
-      if (nnAddress != null) 
-        return nnAddress;
-      else {
-        System.err.println("Nameservice value for "
-            + DFSUtil.getNameServiceIdKey(httpAddressKey, nameServiceId)
-            + " expected but not available.  Trying generic value.");
-        //and fall through to next block
-      }
-    }
-    // If that didn't work, check to see whether the filesystem URI addresses 
-    // the default namenode.  If so, look up generic http address.
-    boolean isDefaultNamenode = DFSUtil.isDefaultNamenodeAddress(
-        conf, namenode,
-        DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
-        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
-    if (isDefaultNamenode) {
-      return NameNode.getInfoServer(conf);
-    } else {
-      System.err.println("Cannot derive an unambiguous value for "
-          + httpAddressKey + " from FileSystem " + fs.getUri());
-      return null;
-    }
+    return DFSUtil.getInfoServer(namenode, conf);
   }
 
   private int doWork(final String[] args) throws IOException {

Propchange: hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:30:05 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs:713112
 /hadoop/core/trunk/src/test/hdfs:776175-785643
-/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:1078943,1080331,1080391,1080402
+/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:1078943,1080331,1080391,1080402,1081603
 /hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
 /hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:30:05 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112
 /hadoop/core/trunk/src/webapps/datanode:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode:1078943,1080331,1080391,1080402
+/hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode:1078943,1080331,1080391,1080402,1081603
 /hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
 /hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:30:05 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112
 /hadoop/core/trunk/src/webapps/hdfs:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs:1078943,1080331,1080391,1080402
+/hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs:1078943,1080331,1080391,1080402,1081603
 /hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
 /hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:30:05 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112
 /hadoop/core/trunk/src/webapps/secondary:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary:1078943,1080331,1080391,1080402
+/hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary:1078943,1080331,1080391,1080402,1081603
 /hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
 /hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052



Mime
View raw message