hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1208147 - in /hadoop/common/branches/branch-1.0: ./ src/docs/src/documentation/content/xdocs/ src/hdfs/org/apache/hadoop/hdfs/server/datanode/ src/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/hdfs/org/apache/hadoop/hdfs/web/ src/test/o...
Date Tue, 29 Nov 2011 23:45:29 GMT
Author: szetszwo
Date: Tue Nov 29 23:45:27 2011
New Revision: 1208147

URL: http://svn.apache.org/viewvc?rev=1208147&view=rev
Log:
svn merge -c 1208143 from branch-1 for HDFS-2604.

Modified:
    hadoop/common/branches/branch-1.0/   (props changed)
    hadoop/common/branches/branch-1.0/CHANGES.txt   (contents, props changed)
    hadoop/common/branches/branch-1.0/src/docs/src/documentation/content/xdocs/webhdfs.xml
    hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
    hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
    hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
    hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java

Propchange: hadoop/common/branches/branch-1.0/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Nov 29 23:45:27 2011
@@ -3,6 +3,6 @@
 /hadoop/common/branches/branch-0.20-security:1170042,1170087,1170997,1171137,1171380,1171613,1171891,1171905,1172184,1172188,1172190,1172192,1173470,1174471,1175114,1176179,1176720,1177907,1179036,1179171,1179519,1179857,1183086,1183101,1183176,1183556,1186509,1187141,1189029,1189361,1190079,1190089,1190100,1195737,1197330,1198906,1199401,1199571,1200733,1205148,1205655,1206180
 /hadoop/common/branches/branch-0.20-security-203:1096071,1097011,1097249,1097269,1097281,1097966,1098816,1098819,1098823,1098827,1098832,1098839,1098854,1098863,1099088,1099191,1099324,1099330,1099333,1102071,1128115
 /hadoop/common/branches/branch-0.20-security-204:1128390,1147228,1148069,1149316,1154413
-/hadoop/common/branches/branch-1:1207579
+/hadoop/common/branches/branch-1:1207579,1208143
 /hadoop/core/branches/branch-0.19:713112
 /hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772884,772920,773889,776638,778962,778966,779893,781720,784661,785046,785569

Modified: hadoop/common/branches/branch-1.0/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/CHANGES.txt?rev=1208147&r1=1208146&r2=1208147&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1.0/CHANGES.txt Tue Nov 29 23:45:27 2011
@@ -55,6 +55,9 @@ Release 1.0.0 - 2011.11.27
     HADOOP-7804. Enable hadoop config generator to set configurations to enable 
     short circuit read. (Arpit Gupta via jitendra)
 
+    HDFS-2604. Add a log message to show if WebHDFS is enabled and a
+    configuration section in the forrest doc.  (szetszwo)
+
   BUG FIXES
 
     HADOOP-7827. jsp pages missing DOCTYPE. (Dave Vronay via mattf)

Propchange: hadoop/common/branches/branch-1.0/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Nov 29 23:45:27 2011
@@ -3,7 +3,7 @@
 /hadoop/common/branches/branch-0.20-security/CHANGES.txt:1170042,1170087,1170997,1171137,1171181,1171380,1171613,1171891,1171905,1172184,1172188,1172190,1172192,1173470,1173843,1174326,1174471,1174476,1174482,1175114,1176179,1176182,1176270,1176276,1176675,1176720,1177031,1177036,1177098,1177101,1177907,1178074,1179036,1179171,1179471,1179519,1179713,1179722,1179857,1179919,1183086,1183101,1183176,1183556,1186509,1187141,1189029,1189361,1190079,1190089,1190100,1195737,1197330,1198906,1199401,1199571,1200733,1205148,1205655,1206180
 /hadoop/common/branches/branch-0.20-security-203/CHANGES.txt:1096071,1097011,1097249,1097269,1097281,1097966,1098816,1098819,1098823,1098827,1098832,1098839,1098854,1098863,1099088,1099191,1099324,1099330,1099333,1102071,1128115
 /hadoop/common/branches/branch-0.20-security-204/CHANGES.txt:1128390,1147228,1148069,1149316,1154413,1159730,1161741
-/hadoop/common/branches/branch-1/CHANGES.txt:1207579
+/hadoop/common/branches/branch-1/CHANGES.txt:1207579,1208143
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
 /hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,772844,772876,772884,772920,773889,776638,778962,778966,779893,781720,784661,785046,785569

Modified: hadoop/common/branches/branch-1.0/src/docs/src/documentation/content/xdocs/webhdfs.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/docs/src/documentation/content/xdocs/webhdfs.xml?rev=1208147&r1=1208146&r2=1208147&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/src/docs/src/documentation/content/xdocs/webhdfs.xml
(original)
+++ hadoop/common/branches/branch-1.0/src/docs/src/documentation/content/xdocs/webhdfs.xml
Tue Nov 29 23:45:27 2011
@@ -138,6 +138,28 @@
   http://<HOST>:<HTTP_PORT>/webhdfs/v1/<PATH>?op=...
 </source>
       </section>
+<!-- ***************************************************************************** -->
+      <section>
+        <title>HDFS Configuration Options</title>
+<p>
+  Below are the HDFS configuration options for WebHDFS.
+</p>
+<table>
+<tr><th>Property Name</th><th>Description</th></tr>
+<tr><td><code>dfs.webhdfs.enabled</code></td>
+<td>Enable/disable WebHDFS in Namenodes and Datanodes
+</td></tr>
+<tr><td><code>dfs.web.authentication.kerberos.principal</code></td>
+<td>The HTTP Kerberos principal used by Hadoop-Auth in the HTTP endpoint.
+    The HTTP Kerberos principal MUST start with 'HTTP/' per Kerberos
+    HTTP SPENGO specification.
+</td></tr>
+<tr><td><code>dfs.web.authentication.kerberos.keytab</code></td>
+<td>The Kerberos keytab file with the credentials for the
+    HTTP Kerberos principal used by Hadoop-Auth in the HTTP endpoint.
+</td></tr>
+</table>
+      </section>
     </section>
 <!-- ***************************************************************************** -->
     <section id="Authentication">

Modified: hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1208147&r1=1208146&r2=1208147&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
(original)
+++ hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
Tue Nov 29 23:45:27 2011
@@ -464,8 +464,7 @@ public class DataNode extends Configured
     this.infoServer.addServlet(null, "/blockScannerReport", 
                                DataBlockScanner.Servlet.class);
 
-    if (conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY,
-        DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT)) {
+    if (WebHdfsFileSystem.isEnabled(conf, LOG)) {
       infoServer.addJerseyResourcePackage(DatanodeWebHdfsMethods.class
           .getPackage().getName() + ";" + Param.class.getPackage().getName(),
           WebHdfsFileSystem.PATH_PREFIX + "/*");

Modified: hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1208147&r1=1208146&r2=1208147&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++ hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Tue Nov 29 23:45:27 2011
@@ -359,8 +359,7 @@ public class NameNode implements ClientP
               infoPort == 0, conf, 
               SecurityUtil.getAdminAcls(conf, DFSConfigKeys.DFS_ADMIN)) {
             {
-              if (conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY,
-                  DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT)) {
+              if (WebHdfsFileSystem.isEnabled(conf, LOG)) {
                 //add SPNEGO authentication filter for webhdfs
                 final String name = "SPNEGO";
                 final String classname =  AuthFilter.class.getName();

Modified: hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1208147&r1=1208146&r2=1208147&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
(original)
+++ hadoop/common/branches/branch-1.0/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
Tue Nov 29 23:45:27 2011
@@ -125,6 +125,14 @@ public class WebHdfsFileSystem extends F
     DT_RENEWER.addRenewAction(webhdfs);
   }
 
+  /** Is WebHDFS enabled in conf? */
+  public static boolean isEnabled(final Configuration conf, final Log log) {
+    final boolean b = conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY,
+        DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT);
+    log.info(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY + " = " + b);
+    return b;
+  }
+
   private final UserGroupInformation ugi;
   private InetSocketAddress nnAddr;
   private Token<?> delegationToken;

Modified: hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java?rev=1208147&r1=1208146&r2=1208147&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java
(original)
+++ hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java
Tue Nov 29 23:45:27 2011
@@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.web.WebHdf
 import org.apache.hadoop.hdfs.web.resources.DoAsParam;
 import org.apache.hadoop.hdfs.web.resources.ExceptionHandler;
 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
+import org.apache.hadoop.hdfs.web.resources.PostOpParam;
 import org.apache.hadoop.hdfs.web.resources.PutOpParam;
 import org.apache.hadoop.security.TestDoAsEffectiveUser;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -105,6 +106,7 @@ public class TestDelegationTokenForProxy
   public void setUp() throws Exception {
     config = new Configuration();
     config.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
+    config.setBoolean(DFSConfigKeys.DFS_SUPPORT_APPEND_KEY, true);
     config.setLong(
         DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
     config.setLong(
@@ -197,9 +199,9 @@ public class TestDelegationTokenForProxy
       Assert.assertEquals("/user/" + PROXY_USER, responsePath);
     }
 
+    final Path f = new Path("/testWebHdfsDoAs/a.txt");
     {
       //test create file with doAs
-      final Path f = new Path("/testWebHdfsDoAs/a.txt");
       final PutOpParam.Op op = PutOpParam.Op.CREATE;
       final URL url = WebHdfsTestUtil.toUrl(webhdfs, op,  f, new DoAsParam(PROXY_USER));
       WebHdfsTestUtil.LOG.info("url=" + url);
@@ -213,5 +215,21 @@ public class TestDelegationTokenForProxy
       WebHdfsTestUtil.LOG.info("status.getOwner()=" + status.getOwner());
       Assert.assertEquals(PROXY_USER, status.getOwner());
     }
+
+    {
+      //test append file with doAs
+      final PostOpParam.Op op = PostOpParam.Op.APPEND;
+      final URL url = WebHdfsTestUtil.toUrl(webhdfs, op,  f, new DoAsParam(PROXY_USER));
+      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+      conn = WebHdfsTestUtil.twoStepWrite(conn, op);
+      final FSDataOutputStream out = WebHdfsTestUtil.write(webhdfs, op, conn, 4096);
+      out.write("\nHello again!".getBytes());
+      out.close();
+  
+      final FileStatus status = webhdfs.getFileStatus(f);
+      WebHdfsTestUtil.LOG.info("status.getOwner()=" + status.getOwner());
+      WebHdfsTestUtil.LOG.info("status.getLen()  =" + status.getLen());
+      Assert.assertEquals(PROXY_USER, status.getOwner());
+    }
   }
 }



Mime
View raw message