hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1199402 - in /hadoop/common/branches/branch-0.20-security-205: ./ src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/ src/hdfs/org/apache/hadoop/hdfs/web/ src/hdfs/org/apache/hadoop/hdfs/web/resources/
Date Tue, 08 Nov 2011 19:31:08 GMT
Author: szetszwo
Date: Tue Nov  8 19:31:07 2011
New Revision: 1199402

URL: http://svn.apache.org/viewvc?rev=1199402&view=rev
Log:
svn merge -c 1199401 from branch-0.20-security for HDFS-2540.

Modified:
    hadoop/common/branches/branch-0.20-security-205/   (props changed)
    hadoop/common/branches/branch-0.20-security-205/CHANGES.txt   (contents, props changed)
    hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
    hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/JsonUtil.java
    hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
    hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java

Propchange: hadoop/common/branches/branch-0.20-security-205/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Nov  8 19:31:07 2011
@@ -1,6 +1,6 @@
 /hadoop/common/branches/branch-0.20:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946,990003,1044225
 /hadoop/common/branches/branch-0.20-append:955380,955398,955448,956329
-/hadoop/common/branches/branch-0.20-security:1170042,1170087,1170997,1171137,1171380,1171613,1171891,1171905,1172184,1172188,1172190,1172192,1173470,1174471,1175114,1176179,1176720,1177907,1179036,1179171,1179519,1179857,1183086,1183101,1183176,1183556,1186509,1187141,1189029,1189361,1190079,1190089,1190100,1195737,1197330,1198906
+/hadoop/common/branches/branch-0.20-security:1170042,1170087,1170997,1171137,1171380,1171613,1171891,1171905,1172184,1172188,1172190,1172192,1173470,1174471,1175114,1176179,1176720,1177907,1179036,1179171,1179519,1179857,1183086,1183101,1183176,1183556,1186509,1187141,1189029,1189361,1190079,1190089,1190100,1195737,1197330,1198906,1199401
 /hadoop/common/branches/branch-0.20-security-203:1096071,1097011,1097249,1097269,1097281,1097966,1098816,1098819,1098823,1098827,1098832,1098839,1098854,1098863,1099088,1099191,1099324,1099330,1099333,1102071,1128115
 /hadoop/common/branches/branch-0.20-security-204:1128390,1147228,1148069,1149316,1154413
 /hadoop/core/branches/branch-0.19:713112

Modified: hadoop/common/branches/branch-0.20-security-205/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/CHANGES.txt?rev=1199402&r1=1199401&r2=1199402&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-205/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security-205/CHANGES.txt Tue Nov  8 19:31:07 2011
@@ -76,6 +76,10 @@ Release 0.20.205.1 - unreleased
     HDFS-2528. Webhdfs: set delegation kind to WEBHDFS and add a HDFS token
     when http requests are redirected to datanode.  (szetszwo)
 
+    HDFS-2540. Webhdfs: change "Expect: 100-continue" to two-step write; change 
+    "HdfsFileStatus" and "localName" respectively to "FileStatus" and
+    "pathSuffix" in JSON response.  (szetszwo)
+
 Release 0.20.205.0 - 2011.10.06
 
   NEW FEATURES

Propchange: hadoop/common/branches/branch-0.20-security-205/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Nov  8 19:31:07 2011
@@ -1,6 +1,6 @@
 /hadoop/common/branches/branch-0.20/CHANGES.txt:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946,990003,1044225
 /hadoop/common/branches/branch-0.20-append/CHANGES.txt:955380,955398,955448,956329
-/hadoop/common/branches/branch-0.20-security/CHANGES.txt:1170042,1170087,1170997,1171137,1171181,1171380,1171613,1171891,1171905,1172184,1172188,1172190,1172192,1173470,1173843,1174326,1174471,1174476,1174482,1175114,1176179,1176182,1176270,1176276,1176675,1176720,1177031,1177036,1177098,1177101,1177907,1178074,1179036,1179171,1179471,1179519,1179713,1179722,1179857,1179919,1183086,1183101,1183176,1183556,1186509,1187141,1189029,1189361,1190079,1190089,1190100,1195737,1197330,1198906
+/hadoop/common/branches/branch-0.20-security/CHANGES.txt:1170042,1170087,1170997,1171137,1171181,1171380,1171613,1171891,1171905,1172184,1172188,1172190,1172192,1173470,1173843,1174326,1174471,1174476,1174482,1175114,1176179,1176182,1176270,1176276,1176675,1176720,1177031,1177036,1177098,1177101,1177907,1178074,1179036,1179171,1179471,1179519,1179713,1179722,1179857,1179919,1183086,1183101,1183176,1183556,1186509,1187141,1189029,1189361,1190079,1190089,1190100,1195737,1197330,1198906,1199401
 /hadoop/common/branches/branch-0.20-security-203/CHANGES.txt:1096071,1097011,1097249,1097269,1097281,1097966,1098816,1098819,1098823,1098827,1098832,1098839,1098854,1098863,1099088,1099191,1099324,1099330,1099333,1102071,1128115
 /hadoop/common/branches/branch-0.20-security-204/CHANGES.txt:1128390,1147228,1148069,1149316,1154413,1159730,1161741
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226

Modified: hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1199402&r1=1199401&r2=1199402&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
(original)
+++ hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
Tue Nov  8 19:31:07 2011
@@ -47,6 +47,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -555,8 +556,8 @@ public class NamenodeWebHdfsMethods {
       @Override
       public void write(final OutputStream outstream) throws IOException {
         final PrintStream out = new PrintStream(outstream);
-        out.println("{\"" + HdfsFileStatus.class.getSimpleName() + "es\":{\""
-            + HdfsFileStatus.class.getSimpleName() + "\":[");
+        out.println("{\"" + FileStatus.class.getSimpleName() + "es\":{\""
+            + FileStatus.class.getSimpleName() + "\":[");
 
         final HdfsFileStatus[] partial = first.getPartialListing();
         if (partial.length > 0) {

Modified: hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/JsonUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/JsonUtil.java?rev=1199402&r1=1199401&r2=1199402&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/JsonUtil.java
(original)
+++ hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/JsonUtil.java
Tue Nov  8 19:31:07 2011
@@ -28,6 +28,7 @@ import java.util.TreeMap;
 
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSUtil;
@@ -149,7 +150,7 @@ public class JsonUtil {
       return null;
     }
     final Map<String, Object> m = new TreeMap<String, Object>();
-    m.put("localName", status.getLocalName());
+    m.put("pathSuffix", status.getLocalName());
     m.put("type", PathType.valueOf(status));
     m.put("length", status.getLen());
     m.put("owner", status.getOwner());
@@ -159,8 +160,7 @@ public class JsonUtil {
     m.put("modificationTime", status.getModificationTime());
     m.put("blockSize", status.getBlockSize());
     m.put("replication", status.getReplication());
-    return includeType ? toJsonString(HdfsFileStatus.class, m) : 
-      JSON.toString(m);
+    return includeType ? toJsonString(FileStatus.class, m): JSON.toString(m);
   }
 
   /** Convert a Json map to a HdfsFileStatus object. */
@@ -170,8 +170,8 @@ public class JsonUtil {
     }
 
     final Map<?, ?> m = includesType ? 
-        (Map<?, ?>)json.get(HdfsFileStatus.class.getSimpleName()) : json;
-    final String localName = (String) m.get("localName");
+        (Map<?, ?>)json.get(FileStatus.class.getSimpleName()) : json;
+    final String localName = (String) m.get("pathSuffix");
     final PathType type = PathType.valueOf((String) m.get("type"));
 
     final long len = (Long) m.get("length");

Modified: hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1199402&r1=1199401&r2=1199402&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
(original)
+++ hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
Tue Nov  8 19:31:07 2011
@@ -325,14 +325,13 @@ public class WebHdfsFileSystem extends F
     final URL url = toUrl(op, fspath, parameters);
 
     //connect and get response
-    final HttpURLConnection conn = getHttpUrlConnection(url);
+    HttpURLConnection conn = getHttpUrlConnection(url);
     try {
       conn.setRequestMethod(op.getType().toString());
-      conn.setDoOutput(op.getDoOutput());
       if (op.getDoOutput()) {
-        conn.setRequestProperty("Expect", "100-Continue");
-        conn.setInstanceFollowRedirects(true);
+        conn = twoStepWrite(conn, op);
       }
+      conn.setDoOutput(op.getDoOutput());
       conn.connect();
       return conn;
     } catch (IOException e) {
@@ -340,6 +339,35 @@ public class WebHdfsFileSystem extends F
       throw e;
     }
   }
+  
+  /**
+   * Two-step Create/Append:
+   * Step 1) Submit a Http request with neither auto-redirect nor data. 
+   * Step 2) Submit Http PUT with the URL from the Location header with data.
+   * 
+   * The reason of having two-step create/append is for preventing clients to
+   * send out the data before the redirect. This issue is addressed by the
+   * "Expect: 100-continue" header in HTTP/1.1; see RFC 2616, Section 8.2.3.
+   * Unfortunately, there are software library bugs (e.g. Jetty 6 http server
+   * and Java 6 http client), which do not correctly implement "Expect:
+   * 100-continue". The two-step create/append is a temporary workaround for
+   * the software library bugs.
+   */
+  private static HttpURLConnection twoStepWrite(HttpURLConnection conn,
+      final HttpOpParam.Op op) throws IOException {
+    //Step 1) Submit a Http request with neither auto-redirect nor data. 
+    conn.setInstanceFollowRedirects(false);
+    conn.setDoOutput(false);
+    conn.connect();
+    validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op), conn);
+    final String redirect = conn.getHeaderField("Location");
+    conn.disconnect();
+
+    //Step 2) Submit Http PUT with the URL from the Location header with data.
+    conn = (HttpURLConnection)new URL(redirect).openConnection();
+    conn.setRequestMethod(op.getType().toString());
+    return conn;
+  }
 
   /**
    * Run a http operation.
@@ -605,8 +633,8 @@ public class WebHdfsFileSystem extends F
 
     final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
     final Map<?, ?> json  = run(op, f);
-    final Map<?, ?> rootmap = (Map<?, ?>)json.get(HdfsFileStatus.class.getSimpleName()
+ "es");
-    final Object[] array = (Object[])rootmap.get(HdfsFileStatus.class.getSimpleName());
+    final Map<?, ?> rootmap = (Map<?, ?>)json.get(FileStatus.class.getSimpleName()
+ "es");
+    final Object[] array = (Object[])rootmap.get(FileStatus.class.getSimpleName());
 
     //convert FileStatus
     final FileStatus[] statuses = new FileStatus[array.length];

Modified: hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java?rev=1199402&r1=1199401&r2=1199402&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java
(original)
+++ hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java
Tue Nov  8 19:31:07 2011
@@ -17,6 +17,9 @@
  */
 package org.apache.hadoop.hdfs.web.resources;
 
+import javax.ws.rs.core.Response;
+
+
 /** Http operation parameter. */
 public abstract class HttpOpParam<E extends Enum<E> & HttpOpParam.Op>
     extends EnumParam<E> {
@@ -46,6 +49,49 @@ public abstract class HttpOpParam<E exte
     public String toQueryString();
   }
 
+  /** Expects HTTP response 307 "Temporary Redirect". */
+  public static class TemporaryRedirectOp implements Op {
+    static final TemporaryRedirectOp CREATE = new TemporaryRedirectOp(PutOpParam.Op.CREATE);
+    static final TemporaryRedirectOp APPEND = new TemporaryRedirectOp(PostOpParam.Op.APPEND);
+    
+    /** Get an object for the given op. */
+    public static TemporaryRedirectOp valueOf(final Op op) {
+      if (op == CREATE.op) {
+        return CREATE;
+      } if (op == APPEND.op) {
+        return APPEND;
+      }
+      throw new IllegalArgumentException(op + " not found.");
+    }
+
+    private final Op op;
+
+    private TemporaryRedirectOp(final Op op) {
+      this.op = op;
+    }
+
+    @Override
+    public Type getType() {
+      return op.getType();
+    }
+
+    @Override
+    public boolean getDoOutput() {
+      return op.getDoOutput();
+    }
+
+    /** Override the original expected response with "Temporary Redirect". */
+    @Override
+    public int getExpectedHttpResponseCode() {
+      return Response.Status.TEMPORARY_REDIRECT.getStatusCode();
+    }
+
+    @Override
+    public String toQueryString() {
+      return op.toQueryString();
+    }
+  }
+
   HttpOpParam(final Domain<E> domain, final E value) {
     super(domain, value);
   }



Mime
View raw message