hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1074776 - in /hadoop/hdfs/branches/HDFS-1052: ./ src/c++/libhdfs/ src/contrib/hdfsproxy/ src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/ src/java/ src/java/org/apache/...
Date Sat, 26 Feb 2011 01:49:27 GMT
Author: suresh
Date: Sat Feb 26 01:49:26 2011
New Revision: 1074776

URL: http://svn.apache.org/viewvc?rev=1074776&view=rev
Log:
Merging chagnes -r1034081:1034181 from trunk to federation

Modified:
    hadoop/hdfs/branches/HDFS-1052/   (props changed)
    hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
    hadoop/hdfs/branches/HDFS-1052/build.xml   (props changed)
    hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs/   (props changed)
    hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/   (props changed)
    hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFilter.java
    hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java
    hadoop/hdfs/branches/HDFS-1052/src/java/   (props changed)
    hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
  (props changed)
    hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/   (props changed)
    hadoop/hdfs/branches/HDFS-1052/src/test/system/java/org/apache/hadoop/hdfs/test/system/DNClient.java
    hadoop/hdfs/branches/HDFS-1052/src/test/system/java/org/apache/hadoop/hdfs/test/system/NNClient.java
    hadoop/hdfs/branches/HDFS-1052/src/test/system/test/org/apache/hadoop/hdfs/TestHL040.java
    hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode/   (props changed)
    hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs/   (props changed)
    hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary/   (props changed)

Propchange: hadoop/hdfs/branches/HDFS-1052/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Feb 26 01:49:26 2011
@@ -1,4 +1,4 @@
 /hadoop/core/branches/branch-0.19/hdfs:713112
 /hadoop/hdfs/branches/HDFS-265:796829-820463
 /hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1036738
+/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034082-1034181,1036738

Modified: hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/CHANGES.txt?rev=1074776&r1=1074775&r2=1074776&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/CHANGES.txt (original)
+++ hadoop/hdfs/branches/HDFS-1052/CHANGES.txt Sat Feb 26 01:49:26 2011
@@ -117,6 +117,8 @@ Release 0.22.0 - Unreleased
     HDFS-1457. Provide an option to throttle image transmission between
     pimary and secondary NameNodes. (Yifei Lu and hairong via hairong)
 
+    HDFS-1164. TestHdfsProxy is failing. (Todd Lipcon via cos)
+
   IMPROVEMENTS
 
     HDFS-1304. Add a new unit test for HftpFileSystem.open(..).  (szetszwo)

Propchange: hadoop/hdfs/branches/HDFS-1052/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Feb 26 01:49:26 2011
@@ -2,4 +2,4 @@
 /hadoop/core/trunk/build.xml:779102
 /hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
 /hadoop/hdfs/branches/branch-0.21/build.xml:820487
-/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639
+/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034082-1034181

Propchange: hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Feb 26 01:49:26 2011
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
 /hadoop/core/trunk/src/c++/libhdfs:776175-784663
-/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639
+/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034082-1034181

Propchange: hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Feb 26 01:49:26 2011
@@ -2,4 +2,4 @@
 /hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
 /hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034082-1034181

Modified: hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFilter.java?rev=1074776&r1=1074775&r2=1074776&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFilter.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFilter.java
Sat Feb 26 01:49:26 2011
@@ -315,7 +315,7 @@ public class ProxyFilter implements Filt
       String ugi = rqst.getParameter("ugi");
       if (ugi != null) {
         rqst.setAttribute("authorized.ugi", UserGroupInformation.createRemoteUser(ugi));
-        rqst.setAttribute("org.apache.hadoop.hdfsproxy.authorized.userID", ugi);
+        rqst.setAttribute("org.apache.hadoop.hdfsproxy.authorized.userID", ugi.split(",")[0]);
       } 
     }
     chain.doFilter(request, response);

Modified: hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java?rev=1074776&r1=1074775&r2=1074776&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java
Sat Feb 26 01:49:26 2011
@@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.server.dat
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
 
 /**
  * A JUnit test for HdfsProxy
@@ -203,13 +204,17 @@ public class TestHdfsProxy extends TestC
     MiniDFSCluster cluster = null;
     HdfsProxy proxy = null;
     try {
+      final UserGroupInformation CLIENT_UGI = UserGroupInformation.getCurrentUser();
+      final String testUser = CLIENT_UGI.getShortUserName();
+      final String testGroup = CLIENT_UGI.getGroupNames()[0];
 
       final Configuration dfsConf = new HdfsConfiguration();
-      dfsConf.set("hadoop.proxyuser." + System.getProperty("user.name") +
-          ".users", "users");
-      dfsConf.set("hadoop.proxyuser.users.ip-addresses", "localhost");
-      dfsConf.set("hadoop.proxyuser." + System.getProperty("user.name") +
-          ".ip-addresses", "localhost");
+      dfsConf.set("hadoop.proxyuser." + testUser +
+          ".groups", testGroup);
+      dfsConf.set("hadoop.proxyuser." + testGroup + ".hosts",
+          "localhost,127.0.0.1");
+      dfsConf.set("hadoop.proxyuser." + testUser +
+          ".hosts", "localhost,127.0.0.1");
       cluster = new MiniDFSCluster.Builder(dfsConf).numDataNodes(2).build();
       cluster.waitActive();
 

Propchange: hadoop/hdfs/branches/HDFS-1052/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Feb 26 01:49:26 2011
@@ -2,4 +2,4 @@
 /hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
 /hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639
+/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034082-1034181

Propchange: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Feb 26 01:49:26 2011
@@ -4,4 +4,4 @@
 /hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
 /hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034082-1034181

Propchange: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Feb 26 01:49:26 2011
@@ -2,4 +2,4 @@
 /hadoop/core/trunk/src/test/hdfs:776175-785643
 /hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
-/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639
+/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034082-1034181

Modified: hadoop/hdfs/branches/HDFS-1052/src/test/system/java/org/apache/hadoop/hdfs/test/system/DNClient.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/system/java/org/apache/hadoop/hdfs/test/system/DNClient.java?rev=1074776&r1=1074775&r2=1074776&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/system/java/org/apache/hadoop/hdfs/test/system/DNClient.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/system/java/org/apache/hadoop/hdfs/test/system/DNClient.java
Sat Feb 26 01:49:26 2011
@@ -36,6 +36,7 @@ import org.apache.hadoop.test.system.pro
 public class DNClient extends HDFSDaemonClient<DNProtocol> {
 
   DNProtocol proxy;
+  private static final String HADOOP_DATANODE_OPTS_ENV = "HADOOP_DATANODE_OPTS";
 
   public DNClient(Configuration conf, RemoteProcess process) throws IOException {
     super(conf, process);
@@ -79,4 +80,20 @@ public class DNClient extends HDFSDaemon
   public Configuration getDatanodeConfig() throws IOException {
     return getProxy().getDaemonConf();
   }
+
+  @Override
+  public String getHadoopOptsEnvName() {
+    return HADOOP_DATANODE_OPTS_ENV;
+  }
+
+  /**
+   * Concrete implementation of abstract super class method
+   * @param attributeName name of the attribute to be retrieved
+   * @return Object value of the given attribute
+   * @throws IOException is thrown in case of communication errors
+   */
+  @Override
+  public Object getDaemonAttribute (String attributeName) throws IOException {
+    return getJmxAttribute("DataNode", "DataNodeInfo", attributeName);
+  }
 }

Modified: hadoop/hdfs/branches/HDFS-1052/src/test/system/java/org/apache/hadoop/hdfs/test/system/NNClient.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/system/java/org/apache/hadoop/hdfs/test/system/NNClient.java?rev=1074776&r1=1074775&r2=1074776&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/system/java/org/apache/hadoop/hdfs/test/system/NNClient.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/system/java/org/apache/hadoop/hdfs/test/system/NNClient.java
Sat Feb 26 01:49:26 2011
@@ -31,6 +31,7 @@ import org.apache.hadoop.test.system.pro
 public class NNClient extends HDFSDaemonClient<NNProtocol> {
   
   NNProtocol proxy;
+  private static final String HADOOP_NAMENODE_OPTS_ENV = "HADOOP_NAMENODE_OPTS";
 
   public NNClient(Configuration conf, RemoteProcess process) throws IOException {
     super(conf, process);
@@ -68,4 +69,20 @@ public class NNClient extends HDFSDaemon
   protected NNProtocol getProxy() {
     return proxy;
   }
+
+  @Override
+  public String getHadoopOptsEnvName() {
+    return HADOOP_NAMENODE_OPTS_ENV;
+  }
+
+  /**
+   * Concrete implementation of abstract super class method
+   * @param attributeName name of the attribute to be retrieved
+   * @return Object value of the given attribute
+   * @throws IOException is thrown in case of communication errors
+   */
+  @Override
+  public Object getDaemonAttribute (String attributeName) throws IOException {
+    return getJmxAttribute("NameNode", "NameNodeInfo", attributeName);
+  }
 }

Modified: hadoop/hdfs/branches/HDFS-1052/src/test/system/test/org/apache/hadoop/hdfs/TestHL040.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/system/test/org/apache/hadoop/hdfs/TestHL040.java?rev=1074776&r1=1074775&r2=1074776&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/system/test/org/apache/hadoop/hdfs/TestHL040.java
(original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/system/test/org/apache/hadoop/hdfs/TestHL040.java
Sat Feb 26 01:49:26 2011
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -31,6 +32,7 @@ import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.mortbay.util.ajax.JSON;
 
 public class TestHL040 {
   private HDFSCluster cluster = null;
@@ -64,6 +66,21 @@ public class TestHL040 {
           dnC.getProcessInfo());
       Assert.assertNotNull("Datanode process info isn't suppose to be null",
           dnC.getProcessInfo());
+      LOG.info("Free space " + getFreeSpace(dnC));
     }
   }
+
+  private long getFreeSpace(DNClient dnC) throws IOException {
+    Object volObj = dnC.getDaemonAttribute("VolumeInfo");
+    Assert.assertNotNull("Attribute value is expected to be not null", volObj);
+    LOG.debug("Got object: " + volObj);
+    Map volInfoMap = (Map) JSON.parse(volObj.toString());
+    long totalFreeSpace = 0L;
+    for (Object key : volInfoMap.keySet()) {
+      Map attrMap = (Map) volInfoMap.get(key);
+      long freeSpace = (Long) attrMap.get("freeSpace");
+      totalFreeSpace += freeSpace;
+    }
+    return totalFreeSpace;
+  }
 }

Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Feb 26 01:49:26 2011
@@ -2,4 +2,4 @@
 /hadoop/core/trunk/src/webapps/datanode:776175-784663
 /hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
-/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639
+/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034082-1034181

Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Feb 26 01:49:26 2011
@@ -2,4 +2,4 @@
 /hadoop/core/trunk/src/webapps/hdfs:776175-784663
 /hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
-/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639
+/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034082-1034181

Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Feb 26 01:49:26 2011
@@ -2,4 +2,4 @@
 /hadoop/core/trunk/src/webapps/secondary:776175-784663
 /hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
-/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639
+/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034082-1034181



Mime
View raw message