hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject svn commit: r1179484 [2/2] - in /hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/ src/main/java/org/apache/hadoop/hdfs/ src/main/java/org/apache/hadoop/hdfs/protocol/ src/main/java/org/apache/hadoop/hdfs/server/datano...
Date Thu, 06 Oct 2011 01:17:05 GMT
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java Thu Oct  6 01:16:48 2011
@@ -547,7 +547,7 @@ class NameNodeRpcServer implements Namen
   }
   
   @Override // ClientProtocol
-  public void rename(String src, String dst, Options.Rename... options)
+  public void rename2(String src, String dst, Options.Rename... options)
       throws IOException {
     nn.checkOperation(OperationCategory.WRITE);
     if(stateChangeLog.isDebugEnabled()) {
@@ -561,13 +561,6 @@ class NameNodeRpcServer implements Namen
     metrics.incrFilesRenamed();
   }
 
-  @Deprecated
-  @Override // ClientProtocol
-  public boolean delete(String src) throws IOException {
-    nn.checkOperation(OperationCategory.WRITE);
-    return delete(src, true);
-  }
-
   @Override // ClientProtocol
   public boolean delete(String src, boolean recursive) throws IOException {
     nn.checkOperation(OperationCategory.WRITE);

Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java Thu Oct  6 01:16:48 2011
@@ -46,6 +46,7 @@ import javax.ws.rs.core.StreamingOutput;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
@@ -112,7 +113,9 @@ public class NamenodeWebHdfsMethods {
   private static DatanodeInfo chooseDatanode(final NameNode namenode,
       final String path, final HttpOpParam.Op op, final long openOffset
       ) throws IOException {
-    if (op == GetOpParam.Op.OPEN || op == PostOpParam.Op.APPEND) {
+    if (op == GetOpParam.Op.OPEN
+        || op == GetOpParam.Op.GETFILECHECKSUM
+        || op == PostOpParam.Op.APPEND) {
       final NamenodeProtocols np = namenode.getRpcServer();
       final HdfsFileStatus status = np.getFileInfo(path);
       final long len = status.getLen();
@@ -245,7 +248,7 @@ public class NamenodeWebHdfsMethods {
     case MKDIRS:
     {
       final boolean b = np.mkdirs(fullpath, permission.getFsPermission(), true);
-      final String js = JsonUtil.toJsonString(PutOpParam.Op.MKDIRS, b);
+      final String js = JsonUtil.toJsonString("boolean", b);
       return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
     }
     case RENAME:
@@ -254,10 +257,10 @@ public class NamenodeWebHdfsMethods {
       if (s.isEmpty()) {
         @SuppressWarnings("deprecation")
         final boolean b = np.rename(fullpath, dstPath.getValue());
-        final String js = JsonUtil.toJsonString(PutOpParam.Op.RENAME, b);
+        final String js = JsonUtil.toJsonString("boolean", b);
         return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
       } else {
-        np.rename(fullpath, dstPath.getValue(),
+        np.rename2(fullpath, dstPath.getValue(),
             s.toArray(new Options.Rename[s.size()]));
         return Response.ok().type(MediaType.APPLICATION_JSON).build();
       }
@@ -265,7 +268,7 @@ public class NamenodeWebHdfsMethods {
     case SETREPLICATION:
     {
       final boolean b = np.setReplication(fullpath, replication.getValue());
-      final String js = JsonUtil.toJsonString(PutOpParam.Op.SETREPLICATION, b);
+      final String js = JsonUtil.toJsonString("boolean", b);
       return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
     }
     case SETOWNER:
@@ -431,6 +434,18 @@ public class NamenodeWebHdfsMethods {
       final StreamingOutput streaming = getListingStream(np, fullpath);
       return Response.ok(streaming).type(MediaType.APPLICATION_JSON).build();
     }
+    case GETCONTENTSUMMARY:
+    {
+      final ContentSummary contentsummary = np.getContentSummary(fullpath);
+      final String js = JsonUtil.toJsonString(contentsummary);
+      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+    }
+    case GETFILECHECKSUM:
+    {
+      final URI uri = redirectURI(namenode, ugi, delegation, fullpath,
+          op.getValue(), -1L);
+      return Response.temporaryRedirect(uri).build();
+    }
     case GETDELEGATIONTOKEN:
     {
       final Token<? extends TokenIdentifier> token = generateDelegationToken(
@@ -467,7 +482,7 @@ public class NamenodeWebHdfsMethods {
       @Override
       public void write(final OutputStream outstream) throws IOException {
         final PrintStream out = new PrintStream(outstream);
-        out.print('[');
+        out.println("{\"" + HdfsFileStatus[].class.getSimpleName() + "\":[");
 
         final HdfsFileStatus[] partial = first.getPartialListing();
         if (partial.length > 0) {
@@ -486,7 +501,7 @@ public class NamenodeWebHdfsMethods {
           }
         }
         
-        out.println(']');
+        out.println("]}");
       }
     };
   }
@@ -522,7 +537,7 @@ public class NamenodeWebHdfsMethods {
         case DELETE:
         {
           final boolean b = namenode.getRpcServer().delete(fullpath, recursive.getValue());
-          final String js = JsonUtil.toJsonString(DeleteOpParam.Op.DELETE, b);
+          final String js = JsonUtil.toJsonString("boolean", b);
           return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
         }
         default:

Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java Thu Oct  6 01:16:48 2011
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdfs.web;
 
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -24,6 +26,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -34,45 +38,33 @@ import org.apache.hadoop.hdfs.protocol.L
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.io.MD5Hash;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.StringUtils;
 import org.mortbay.util.ajax.JSON;
 
 /** JSON Utilities */
 public class JsonUtil {
-  private static class ThreadLocalMap extends ThreadLocal<Map<String, Object>> {
-    @Override
-    protected Map<String, Object> initialValue() {
-      return new TreeMap<String, Object>();
-    }
-
-    @Override
-    public Map<String, Object> get() {
-      final Map<String, Object> m = super.get();
-      m.clear();
-      return m;
-    }
-  }
-
-  private static final ThreadLocalMap jsonMap = new ThreadLocalMap();
-  private static final ThreadLocalMap tokenMap = new ThreadLocalMap();
-  private static final ThreadLocalMap datanodeInfoMap = new ThreadLocalMap();
-  private static final ThreadLocalMap extendedBlockMap = new ThreadLocalMap();
-  private static final ThreadLocalMap locatedBlockMap = new ThreadLocalMap();
-
+  private static final Object[] EMPTY_OBJECT_ARRAY = {};
   private static final DatanodeInfo[] EMPTY_DATANODE_INFO_ARRAY = {};
 
   /** Convert a token object to a Json string. */
   public static String toJsonString(final Token<? extends TokenIdentifier> token
       ) throws IOException {
+    return toJsonString(Token.class, toJsonMap(token));
+  }
+
+  private static Map<String, Object> toJsonMap(
+      final Token<? extends TokenIdentifier> token) throws IOException {
     if (token == null) {
       return null;
     }
 
-    final Map<String, Object> m = tokenMap.get();
+    final Map<String, Object> m = new TreeMap<String, Object>();
     m.put("urlString", token.encodeToUrlString());
-    return JSON.toString(m);
+    return m;
   }
 
   /** Convert a Json map to a Token. */
@@ -91,46 +83,52 @@ public class JsonUtil {
   /** Convert a Json map to a Token of DelegationTokenIdentifier. */
   @SuppressWarnings("unchecked")
   public static Token<DelegationTokenIdentifier> toDelegationToken(
-      final Map<?, ?> m) throws IOException {
+      final Map<?, ?> json) throws IOException {
+    final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName());
     return (Token<DelegationTokenIdentifier>)toToken(m);
   }
 
   /** Convert a Json map to a Token of BlockTokenIdentifier. */
   @SuppressWarnings("unchecked")
-  public static Token<BlockTokenIdentifier> toBlockToken(
+  private static Token<BlockTokenIdentifier> toBlockToken(
       final Map<?, ?> m) throws IOException {
     return (Token<BlockTokenIdentifier>)toToken(m);
   }
 
   /** Convert an exception object to a Json string. */
   public static String toJsonString(final Exception e) {
-    final Map<String, Object> m = jsonMap.get();
+    final Map<String, Object> m = new TreeMap<String, Object>();
     m.put("className", e.getClass().getName());
     m.put("message", e.getMessage());
-    return JSON.toString(m);
+    return toJsonString(RemoteException.class, m);
   }
 
   /** Convert a Json map to a RemoteException. */
-  public static RemoteException toRemoteException(final Map<String, Object> m) {
+  public static RemoteException toRemoteException(final Map<?, ?> json) {
+    final Map<?, ?> m = (Map<?, ?>)json.get(RemoteException.class.getSimpleName());
     final String className = (String)m.get("className");
     final String message = (String)m.get("message");
     return new RemoteException(className, message);
   }
 
+  private static String toJsonString(final Class<?> clazz, final Object value) {
+    return toJsonString(clazz.getSimpleName(), value);
+  }
+
   /** Convert a key-value pair to a Json string. */
-  public static String toJsonString(final Object key, final Object value) {
-    final Map<String, Object> m = jsonMap.get();
-    m.put(key instanceof String ? (String) key : key.toString(), value);
+  public static String toJsonString(final String key, final Object value) {
+    final Map<String, Object> m = new TreeMap<String, Object>();
+    m.put(key, value);
     return JSON.toString(m);
   }
 
   /** Convert a FsPermission object to a string. */
-  public static String toString(final FsPermission permission) {
+  private static String toString(final FsPermission permission) {
     return String.format("%o", permission.toShort());
   }
 
   /** Convert a string to a FsPermission object. */
-  public static FsPermission toFsPermission(final String s) {
+  private static FsPermission toFsPermission(final String s) {
     return new FsPermission(Short.parseShort(s, 8));
   }
 
@@ -139,7 +137,7 @@ public class JsonUtil {
     if (status == null) {
       return null;
     } else {
-      final Map<String, Object> m = jsonMap.get();
+      final Map<String, Object> m = new TreeMap<String, Object>();
       m.put("localName", status.getLocalName());
       m.put("isDir", status.isDir());
       m.put("isSymlink", status.isSymlink());
@@ -155,21 +153,17 @@ public class JsonUtil {
       m.put("modificationTime", status.getModificationTime());
       m.put("blockSize", status.getBlockSize());
       m.put("replication", status.getReplication());
-      return JSON.toString(m);
+      return toJsonString(HdfsFileStatus.class, m);
     }
   }
 
-  @SuppressWarnings("unchecked")
-  static Map<String, Object> parse(String jsonString) {
-    return (Map<String, Object>) JSON.parse(jsonString);
-  }
-
   /** Convert a Json map to a HdfsFileStatus object. */
-  public static HdfsFileStatus toFileStatus(final Map<String, Object> m) {
-    if (m == null) {
+  public static HdfsFileStatus toFileStatus(final Map<?, ?> json) {
+    if (json == null) {
       return null;
     }
 
+    final Map<?, ?> m = (Map<?, ?>)json.get(HdfsFileStatus.class.getSimpleName());
     final String localName = (String) m.get("localName");
     final boolean isDir = (Boolean) m.get("isDir");
     final boolean isSymlink = (Boolean) m.get("isSymlink");
@@ -189,22 +183,22 @@ public class JsonUtil {
         symlink, DFSUtil.string2Bytes(localName));
   }
 
-  /** Convert a LocatedBlock to a Json string. */
-  public static String toJsonString(final ExtendedBlock extendedblock) {
+  /** Convert an ExtendedBlock to a Json map. */
+  private static Map<String, Object> toJsonMap(final ExtendedBlock extendedblock) {
     if (extendedblock == null) {
       return null;
     }
 
-    final Map<String, Object> m = extendedBlockMap.get();
+    final Map<String, Object> m = new TreeMap<String, Object>();
     m.put("blockPoolId", extendedblock.getBlockPoolId());
     m.put("blockId", extendedblock.getBlockId());
     m.put("numBytes", extendedblock.getNumBytes());
     m.put("generationStamp", extendedblock.getGenerationStamp());
-    return JSON.toString(m);
+    return m;
   }
 
   /** Convert a Json map to an ExtendedBlock object. */
-  public static ExtendedBlock toExtendedBlock(final Map<?, ?> m) {
+  private static ExtendedBlock toExtendedBlock(final Map<?, ?> m) {
     if (m == null) {
       return null;
     }
@@ -216,13 +210,13 @@ public class JsonUtil {
     return new ExtendedBlock(blockPoolId, blockId, numBytes, generationStamp);
   }
   
-  /** Convert a DatanodeInfo to a Json string. */
-  public static String toJsonString(final DatanodeInfo datanodeinfo) {
+  /** Convert a DatanodeInfo to a Json map. */
+  private static Map<String, Object> toJsonMap(final DatanodeInfo datanodeinfo) {
     if (datanodeinfo == null) {
       return null;
     }
 
-    final Map<String, Object> m = datanodeInfoMap.get();
+    final Map<String, Object> m = new TreeMap<String, Object>();
     m.put("name", datanodeinfo.getName());
     m.put("storageID", datanodeinfo.getStorageID());
     m.put("infoPort", datanodeinfo.getInfoPort());
@@ -238,11 +232,11 @@ public class JsonUtil {
     m.put("networkLocation", datanodeinfo.getNetworkLocation());
     m.put("hostName", datanodeinfo.getHostName());
     m.put("adminState", datanodeinfo.getAdminState().name());
-    return JSON.toString(m);
+    return m;
   }
 
   /** Convert a Json map to an DatanodeInfo object. */
-  public static DatanodeInfo toDatanodeInfo(final Map<?, ?> m) {
+  private static DatanodeInfo toDatanodeInfo(final Map<?, ?> m) {
     if (m == null) {
       return null;
     }
@@ -264,25 +258,23 @@ public class JsonUtil {
         AdminStates.valueOf((String)m.get("adminState")));
   }
 
-  /** Convert a DatanodeInfo[] to a Json string. */
-  public static String toJsonString(final DatanodeInfo[] array
-      ) throws IOException {
+  /** Convert a DatanodeInfo[] to a Json array. */
+  private static Object[] toJsonArray(final DatanodeInfo[] array) {
     if (array == null) {
       return null;
     } else if (array.length == 0) {
-      return "[]";
+      return EMPTY_OBJECT_ARRAY;
     } else {
-      final StringBuilder b = new StringBuilder().append('[').append(
-          toJsonString(array[0]));
-      for(int i = 1; i < array.length; i++) {
-        b.append(", ").append(toJsonString(array[i]));
+      final Object[] a = new Object[array.length];
+      for(int i = 0; i < array.length; i++) {
+        a[i] = toJsonMap(array[i]);
       }
-      return b.append(']').toString();
+      return a;
     }
   }
 
   /** Convert an Object[] to a DatanodeInfo[]. */
-  public static DatanodeInfo[] toDatanodeInfoArray(final Object[] objects) {
+  private static DatanodeInfo[] toDatanodeInfoArray(final Object[] objects) {
     if (objects == null) {
       return null;
     } else if (objects.length == 0) {
@@ -290,66 +282,63 @@ public class JsonUtil {
     } else {
       final DatanodeInfo[] array = new DatanodeInfo[objects.length];
       for(int i = 0; i < array.length; i++) {
-        array[i] = (DatanodeInfo)toDatanodeInfo((Map<?, ?>) objects[i]);
+        array[i] = toDatanodeInfo((Map<?, ?>) objects[i]);
       }
       return array;
     }
   }
 
-  /** Convert a LocatedBlock to a Json string. */
-  public static String toJsonString(final LocatedBlock locatedblock
+  /** Convert a LocatedBlock to a Json map. */
+  private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock
       ) throws IOException {
     if (locatedblock == null) {
       return null;
     }
  
-    final Map<String, Object> m = locatedBlockMap.get();
-    m.put("blockToken", toJsonString(locatedblock.getBlockToken()));
+    final Map<String, Object> m = new TreeMap<String, Object>();
+    m.put("blockToken", toJsonMap(locatedblock.getBlockToken()));
     m.put("isCorrupt", locatedblock.isCorrupt());
     m.put("startOffset", locatedblock.getStartOffset());
-    m.put("block", toJsonString(locatedblock.getBlock()));
-
-    m.put("locations", toJsonString(locatedblock.getLocations()));
-    return JSON.toString(m);
+    m.put("block", toJsonMap(locatedblock.getBlock()));
+    m.put("locations", toJsonArray(locatedblock.getLocations()));
+    return m;
   }
 
   /** Convert a Json map to LocatedBlock. */
-  public static LocatedBlock toLocatedBlock(final Map<?, ?> m) throws IOException {
+  private static LocatedBlock toLocatedBlock(final Map<?, ?> m) throws IOException {
     if (m == null) {
       return null;
     }
 
-    final ExtendedBlock b = toExtendedBlock((Map<?, ?>)JSON.parse((String)m.get("block")));
+    final ExtendedBlock b = toExtendedBlock((Map<?, ?>)m.get("block"));
     final DatanodeInfo[] locations = toDatanodeInfoArray(
-        (Object[])JSON.parse((String)m.get("locations")));
+        (Object[])m.get("locations"));
     final long startOffset = (Long)m.get("startOffset");
     final boolean isCorrupt = (Boolean)m.get("isCorrupt");
 
     final LocatedBlock locatedblock = new LocatedBlock(b, locations, startOffset, isCorrupt);
-    locatedblock.setBlockToken(toBlockToken((Map<?, ?>)JSON.parse((String)m.get("blockToken"))));
+    locatedblock.setBlockToken(toBlockToken((Map<?, ?>)m.get("blockToken")));
     return locatedblock;
   }
 
-  /** Convert a LocatedBlock[] to a Json string. */
-  public static String toJsonString(final List<LocatedBlock> array
+  /** Convert a LocatedBlock[] to a Json array. */
+  private static Object[] toJsonArray(final List<LocatedBlock> array
       ) throws IOException {
     if (array == null) {
       return null;
     } else if (array.size() == 0) {
-      return "[]";
+      return EMPTY_OBJECT_ARRAY;
     } else {
-      final StringBuilder b = new StringBuilder().append('[').append(
-          toJsonString(array.get(0)));
-      for(int i = 1; i < array.size(); i++) {
-        b.append(",\n  ").append(toJsonString(array.get(i)));
+      final Object[] a = new Object[array.size()];
+      for(int i = 0; i < array.size(); i++) {
+        a[i] = toJsonMap(array.get(0));
       }
-      return b.append(']').toString();
+      return a;
     }
   }
 
-  /** Convert an Object[] to a List of LocatedBlock. 
-   * @throws IOException */
-  public static List<LocatedBlock> toLocatedBlockList(final Object[] objects
+  /** Convert an Object[] to a List of LocatedBlock. */
+  private static List<LocatedBlock> toLocatedBlockList(final Object[] objects
       ) throws IOException {
     if (objects == null) {
       return null;
@@ -358,7 +347,7 @@ public class JsonUtil {
     } else {
       final List<LocatedBlock> list = new ArrayList<LocatedBlock>(objects.length);
       for(int i = 0; i < objects.length; i++) {
-        list.add((LocatedBlock)toLocatedBlock((Map<?, ?>)objects[i]));
+        list.add(toLocatedBlock((Map<?, ?>)objects[i]));
       }
       return list;
     }
@@ -371,31 +360,115 @@ public class JsonUtil {
       return null;
     }
 
-    final Map<String, Object> m = jsonMap.get();
+    final Map<String, Object> m = new TreeMap<String, Object>();
     m.put("fileLength", locatedblocks.getFileLength());
     m.put("isUnderConstruction", locatedblocks.isUnderConstruction());
 
-    m.put("locatedBlocks", toJsonString(locatedblocks.getLocatedBlocks()));
-    m.put("lastLocatedBlock", toJsonString(locatedblocks.getLastLocatedBlock()));
+    m.put("locatedBlocks", toJsonArray(locatedblocks.getLocatedBlocks()));
+    m.put("lastLocatedBlock", toJsonMap(locatedblocks.getLastLocatedBlock()));
     m.put("isLastBlockComplete", locatedblocks.isLastBlockComplete());
-    return JSON.toString(m);
+    return toJsonString(LocatedBlocks.class, m);
   }
 
   /** Convert a Json map to LocatedBlock. */
-  public static LocatedBlocks toLocatedBlocks(final Map<String, Object> m
+  public static LocatedBlocks toLocatedBlocks(final Map<?, ?> json
       ) throws IOException {
-    if (m == null) {
+    if (json == null) {
       return null;
     }
-    
+
+    final Map<?, ?> m = (Map<?, ?>)json.get(LocatedBlocks.class.getSimpleName());
     final long fileLength = (Long)m.get("fileLength");
     final boolean isUnderConstruction = (Boolean)m.get("isUnderConstruction");
     final List<LocatedBlock> locatedBlocks = toLocatedBlockList(
-        (Object[])JSON.parse((String) m.get("locatedBlocks")));
+        (Object[])m.get("locatedBlocks"));
     final LocatedBlock lastLocatedBlock = toLocatedBlock(
-        (Map<?, ?>)JSON.parse((String)m.get("lastLocatedBlock")));
+        (Map<?, ?>)m.get("lastLocatedBlock"));
     final boolean isLastBlockComplete = (Boolean)m.get("isLastBlockComplete");
     return new LocatedBlocks(fileLength, isUnderConstruction, locatedBlocks,
         lastLocatedBlock, isLastBlockComplete);
   }
+
+  /** Convert a ContentSummary to a Json string. */
+  public static String toJsonString(final ContentSummary contentsummary) {
+    if (contentsummary == null) {
+      return null;
+    }
+
+    final Map<String, Object> m = new TreeMap<String, Object>();
+    m.put("length", contentsummary.getLength());
+    m.put("fileCount", contentsummary.getFileCount());
+    m.put("directoryCount", contentsummary.getDirectoryCount());
+    m.put("quota", contentsummary.getQuota());
+    m.put("spaceConsumed", contentsummary.getSpaceConsumed());
+    m.put("spaceQuota", contentsummary.getSpaceQuota());
+    return toJsonString(ContentSummary.class, m);
+  }
+
+  /** Convert a Json map to a ContentSummary. */
+  public static ContentSummary toContentSummary(final Map<?, ?> json) {
+    if (json == null) {
+      return null;
+    }
+
+    final Map<?, ?> m = (Map<?, ?>)json.get(ContentSummary.class.getSimpleName());
+    final long length = (Long)m.get("length");
+    final long fileCount = (Long)m.get("fileCount");
+    final long directoryCount = (Long)m.get("directoryCount");
+    final long quota = (Long)m.get("quota");
+    final long spaceConsumed = (Long)m.get("spaceConsumed");
+    final long spaceQuota = (Long)m.get("spaceQuota");
+
+    return new ContentSummary(length, fileCount, directoryCount,
+        quota, spaceConsumed, spaceQuota);
+  }
+
+  /** Convert a MD5MD5CRC32FileChecksum to a Json string. */
+  public static String toJsonString(final MD5MD5CRC32FileChecksum checksum) {
+    if (checksum == null) {
+      return null;
+    }
+
+    final Map<String, Object> m = new TreeMap<String, Object>();
+    m.put("algorithm", checksum.getAlgorithmName());
+    m.put("length", checksum.getLength());
+    m.put("bytes", StringUtils.byteToHexString(checksum.getBytes()));
+    return toJsonString(MD5MD5CRC32FileChecksum.class, m);
+  }
+
+  /** Convert a Json map to a MD5MD5CRC32FileChecksum. */
+  public static MD5MD5CRC32FileChecksum toMD5MD5CRC32FileChecksum(
+      final Map<?, ?> json) throws IOException {
+    if (json == null) {
+      return null;
+    }
+
+    final Map<?, ?> m = (Map<?, ?>)json.get(
+        MD5MD5CRC32FileChecksum.class.getSimpleName());
+    final String algorithm = (String)m.get("algorithm");
+    final int length = (int)(long)(Long)m.get("length");
+    final byte[] bytes = StringUtils.hexStringToByte((String)m.get("bytes"));
+
+    final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes));
+    final int bytesPerCRC = in.readInt();
+    final long crcPerBlock = in.readLong();
+    final MD5Hash md5 = MD5Hash.read(in);
+    final MD5MD5CRC32FileChecksum checksum = new MD5MD5CRC32FileChecksum(
+        bytesPerCRC, crcPerBlock, md5);
+
+    //check algorithm name
+    final String alg = "MD5-of-" + crcPerBlock + "MD5-of-" + bytesPerCRC + "CRC32";
+    if (!alg.equals(algorithm)) {
+      throw new IOException("Algorithm not matched: algorithm=" + algorithm
+          + ", crcPerBlock=" + crcPerBlock
+          + ", bytesPerCRC=" + bytesPerCRC);
+    }
+    //check length
+    if (length != checksum.getLength()) {
+      throw new IOException("Length not matched: length=" + length
+          + ", checksum.getLength()=" + checksum.getLength());
+    }
+
+    return checksum;
+  }
 }
\ No newline at end of file

Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Thu Oct  6 01:16:48 2011
@@ -33,10 +33,12 @@ import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
 import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.ParentNotDirectoryException;
 import org.apache.hadoop.fs.Path;
@@ -278,7 +280,7 @@ public class WebHdfsFileSystem extends H
     final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
     final Map<String, Object> json = run(op, f,
         new PermissionParam(applyUMask(permission)));
-    return (Boolean)json.get(op.toString());
+    return (Boolean)json.get("boolean");
   }
 
   @Override
@@ -287,7 +289,7 @@ public class WebHdfsFileSystem extends H
     final HttpOpParam.Op op = PutOpParam.Op.RENAME;
     final Map<String, Object> json = run(op, src,
         new DstPathParam(makeQualified(dst).toUri().getPath()));
-    return (Boolean)json.get(op.toString());
+    return (Boolean)json.get("boolean");
   }
 
   @SuppressWarnings("deprecation")
@@ -327,7 +329,7 @@ public class WebHdfsFileSystem extends H
     final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
     final Map<String, Object> json = run(op, p,
         new ReplicationParam(replication));
-    return (Boolean)json.get(op.toString());
+    return (Boolean)json.get("boolean");
   }
 
   @Override
@@ -384,7 +386,7 @@ public class WebHdfsFileSystem extends H
   public boolean delete(Path f, boolean recursive) throws IOException {
     final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
     final Map<String, Object> json = run(op, f, new RecursiveParam(recursive));
-    return (Boolean)json.get(op.toString());
+    return (Boolean)json.get("boolean");
   }
 
   @Override
@@ -401,7 +403,9 @@ public class WebHdfsFileSystem extends H
     statistics.incrementReadOps(1);
 
     final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
-    final Object[] array = run(op, f);
+    final Map<?, ?> json  = run(op, f);
+    final Object[] array = (Object[])json.get(
+        HdfsFileStatus[].class.getSimpleName());
 
     //convert FileStatus
     final FileStatus[] statuses = new FileStatus[array.length];
@@ -449,4 +453,23 @@ public class WebHdfsFileSystem extends H
         new LengthParam(length));
     return DFSUtil.locatedBlocks2Locations(JsonUtil.toLocatedBlocks(m));
   }
+
+  @Override
+  public ContentSummary getContentSummary(final Path p) throws IOException {
+    statistics.incrementReadOps(1);
+
+    final HttpOpParam.Op op = GetOpParam.Op.GETCONTENTSUMMARY;
+    final Map<String, Object> m = run(op, p);
+    return JsonUtil.toContentSummary(m);
+  }
+
+  @Override
+  public MD5MD5CRC32FileChecksum getFileChecksum(final Path p
+      ) throws IOException {
+    statistics.incrementReadOps(1);
+  
+    final HttpOpParam.Op op = GetOpParam.Op.GETFILECHECKSUM;
+    final Map<String, Object> m = run(op, p);
+    return JsonUtil.toMD5MD5CRC32FileChecksum(m);
+  }
 }
\ No newline at end of file

Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java Thu Oct  6 01:16:48 2011
@@ -28,6 +28,8 @@ public class GetOpParam extends HttpOpPa
 
     GETFILESTATUS(HttpURLConnection.HTTP_OK),
     LISTSTATUS(HttpURLConnection.HTTP_OK),
+    GETCONTENTSUMMARY(HttpURLConnection.HTTP_OK),
+    GETFILECHECKSUM(HttpURLConnection.HTTP_OK),
 
     GETDELEGATIONTOKEN(HttpURLConnection.HTTP_OK),
 

Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Oct  6 01:16:48 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:1152502-1177128
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:1152502-1179483
 /hadoop/core/branches/branch-0.19/hdfs/src/main/native:713112
 /hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
 /hadoop/core/trunk/src/c++/libhdfs:776175-784663

Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/packages/deb/hadoop.control/preinst
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/packages/deb/hadoop.control/preinst?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/packages/deb/hadoop.control/preinst (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/packages/deb/hadoop.control/preinst Thu Oct  6 01:16:48 2011
@@ -15,4 +15,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-/usr/sbin/useradd --comment "Hadoop HDFS" --shell /bin/bash -M -r --groups hadoop --home /var/lib/hadoop/hdfs hdfs 2> /dev/null || :
+/usr/sbin/useradd --comment "Hadoop HDFS" -u 201 --shell /bin/bash -M -r --groups hadoop --home /var/lib/hadoop/hdfs hdfs 2> /dev/null || :

Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/packages/rpm/spec/hadoop-hdfs.spec
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/packages/rpm/spec/hadoop-hdfs.spec?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/packages/rpm/spec/hadoop-hdfs.spec (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/packages/rpm/spec/hadoop-hdfs.spec Thu Oct  6 01:16:48 2011
@@ -144,8 +144,8 @@ mv ${RPM_BUILD_DIR}/%{_final_name}/share
 rm -rf ${RPM_BUILD_DIR}/%{_final_name}/etc
 
 %pre
-getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hadoop
-/usr/sbin/useradd --comment "Hadoop HDFS" --shell /bin/bash -M -r --groups hadoop --home %{_var_dir}/hdfs hdfs 2> /dev/null || :
+getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -g 123 -r hadoop
+/usr/sbin/useradd --comment "Hadoop HDFS" -u 201 --shell /bin/bash -M -r --groups hadoop --home %{_var_dir}/hdfs hdfs 2> /dev/null || :
 
 %post
 bash ${RPM_INSTALL_PREFIX0}/sbin/update-hdfs-env.sh \

Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Oct  6 01:16:48 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:1159757-1177128
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:1159757-1179483
 /hadoop/core/branches/branch-0.19/hdfs/src/main/webapps/datanode:713112
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112
 /hadoop/core/trunk/src/webapps/datanode:776175-784663

Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Oct  6 01:16:48 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:1152502-1177128
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:1152502-1179483
 /hadoop/core/branches/branch-0.19/hdfs/src/main/webapps/hdfs:713112
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112
 /hadoop/core/trunk/src/webapps/hdfs:776175-784663

Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Oct  6 01:16:48 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:1152502-1177128
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:1152502-1179483
 /hadoop/core/branches/branch-0.19/hdfs/src/main/webapps/secondary:713112
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112
 /hadoop/core/trunk/src/webapps/secondary:776175-784663

Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Oct  6 01:16:48 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:1159757-1177128
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:1159757-1179483
 /hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs:713112
 /hadoop/core/trunk/src/test/hdfs:776175-785643
 /hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:987665-1095512

Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java Thu Oct  6 01:16:48 2011
@@ -26,6 +26,7 @@ import static org.junit.Assert.fail;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
+import java.security.PrivilegedExceptionAction;
 import java.util.Random;
 
 import org.apache.commons.logging.impl.Log4JLogger;
@@ -37,6 +38,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.log4j.Level;
 import org.junit.Test;
@@ -399,15 +401,40 @@ public class TestDistributedFileSystem {
     RAN.setSeed(seed);
 
     final Configuration conf = getTestConfiguration();
+    conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
     conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "localhost");
 
     final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
     final FileSystem hdfs = cluster.getFileSystem();
-    final String hftpuri = "hftp://" + conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
+
+    final String nnAddr = conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
+    final UserGroupInformation current = UserGroupInformation.getCurrentUser();
+    final UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
+        current.getShortUserName() + "x", new String[]{"user"});
+    
+    //hftp
+    final String hftpuri = "hftp://" + nnAddr;
     System.out.println("hftpuri=" + hftpuri);
-    final FileSystem hftp = new Path(hftpuri).getFileSystem(conf);
+    final FileSystem hftp = ugi.doAs(
+        new PrivilegedExceptionAction<FileSystem>() {
+      @Override
+      public FileSystem run() throws Exception {
+        return new Path(hftpuri).getFileSystem(conf);
+      }
+    });
+
+    //webhdfs
+    final String webhdfsuri = WebHdfsFileSystem.SCHEME  + "://" + nnAddr;
+    System.out.println("webhdfsuri=" + webhdfsuri);
+    final FileSystem webhdfs = ugi.doAs(
+        new PrivilegedExceptionAction<FileSystem>() {
+      @Override
+      public FileSystem run() throws Exception {
+        return new Path(webhdfsuri).getFileSystem(conf);
+      }
+    });
 
-    final String dir = "/filechecksum";
+    final Path dir = new Path("/filechecksum");
     final int block_size = 1024;
     final int buffer_size = conf.getInt("io.file.buffer.size", 4096);
     conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 512);
@@ -431,7 +458,8 @@ public class TestDistributedFileSystem {
       //compute checksum
       final FileChecksum hdfsfoocs = hdfs.getFileChecksum(foo);
       System.out.println("hdfsfoocs=" + hdfsfoocs);
-      
+
+      //hftp
       final FileChecksum hftpfoocs = hftp.getFileChecksum(foo);
       System.out.println("hftpfoocs=" + hftpfoocs);
 
@@ -439,6 +467,14 @@ public class TestDistributedFileSystem {
       final FileChecksum qfoocs = hftp.getFileChecksum(qualified);
       System.out.println("qfoocs=" + qfoocs);
 
+      //webhdfs
+      final FileChecksum webhdfsfoocs = webhdfs.getFileChecksum(foo);
+      System.out.println("webhdfsfoocs=" + webhdfsfoocs);
+
+      final Path webhdfsqualified = new Path(webhdfsuri + dir, "foo" + n);
+      final FileChecksum webhdfs_qfoocs = webhdfs.getFileChecksum(webhdfsqualified);
+      System.out.println("webhdfs_qfoocs=" + webhdfs_qfoocs);
+
       //write another file
       final Path bar = new Path(dir, "bar" + n);
       {
@@ -454,24 +490,40 @@ public class TestDistributedFileSystem {
         assertEquals(hdfsfoocs.hashCode(), barhashcode);
         assertEquals(hdfsfoocs, barcs);
 
+        //hftp
         assertEquals(hftpfoocs.hashCode(), barhashcode);
         assertEquals(hftpfoocs, barcs);
 
         assertEquals(qfoocs.hashCode(), barhashcode);
         assertEquals(qfoocs, barcs);
+
+        //webhdfs
+        assertEquals(webhdfsfoocs.hashCode(), barhashcode);
+        assertEquals(webhdfsfoocs, barcs);
+
+        assertEquals(webhdfs_qfoocs.hashCode(), barhashcode);
+        assertEquals(webhdfs_qfoocs, barcs);
       }
 
+      hdfs.setPermission(dir, new FsPermission((short)0));
       { //test permission error on hftp 
-        hdfs.setPermission(new Path(dir), new FsPermission((short)0));
         try {
-          final String username = UserGroupInformation.getCurrentUser().getShortUserName() + "1";
-          final HftpFileSystem hftp2 = cluster.getHftpFileSystemAs(username, conf, 0, "somegroup");
-          hftp2.getFileChecksum(qualified);
+          hftp.getFileChecksum(qualified);
+          fail();
+        } catch(IOException ioe) {
+          FileSystem.LOG.info("GOOD: getting an exception", ioe);
+        }
+      }
+
+      { //test permission error on webhdfs 
+        try {
+          webhdfs.getFileChecksum(webhdfsqualified);
           fail();
         } catch(IOException ioe) {
           FileSystem.LOG.info("GOOD: getting an exception", ioe);
         }
       }
+      hdfs.setPermission(dir, new FsPermission((short)0777));
     }
     cluster.shutdown();
   }

Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java Thu Oct  6 01:16:48 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.protocol.N
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
 import org.apache.hadoop.hdfs.tools.DFSAdmin;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.junit.Test;
@@ -770,6 +771,11 @@ public class TestQuota {
     }
   }
 
+  private static void checkContentSummary(final ContentSummary expected,
+      final ContentSummary computed) {
+    assertEquals(expected.toString(), computed.toString());
+  }
+
   /**
    * Violate a space quota using files of size < 1 block. Test that block
    * allocation conservatively assumes that for quota checking the entire
@@ -781,12 +787,18 @@ public class TestQuota {
     Configuration conf = new HdfsConfiguration();
     final int BLOCK_SIZE = 6 * 1024;
     conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+    conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
     MiniDFSCluster cluster = 
       new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
     cluster.waitActive();
     FileSystem fs = cluster.getFileSystem();
     DFSAdmin admin = new DFSAdmin(conf);
-    
+
+    final String nnAddr = conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
+    final String webhdfsuri = WebHdfsFileSystem.SCHEME  + "://" + nnAddr;
+    System.out.println("webhdfsuri=" + webhdfsuri);
+    final FileSystem webhdfs = new Path(webhdfsuri).getFileSystem(conf);
+
     try {
       Path dir = new Path("/test");
       Path file1 = new Path("/test/test1");
@@ -806,6 +818,7 @@ public class TestQuota {
       DFSTestUtil.createFile(fs, file1, FILE_SIZE, (short) 3, 1L);
       DFSTestUtil.waitReplication(fs, file1, (short) 3);
       c = fs.getContentSummary(dir);
+      checkContentSummary(c, webhdfs.getContentSummary(dir));
       assertEquals("Quota is half consumed", QUOTA_SIZE / 2,
                    c.getSpaceConsumed());
 
@@ -836,12 +849,18 @@ public class TestQuota {
     Configuration conf = new HdfsConfiguration();
     final int BLOCK_SIZE = 6 * 1024;
     conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+    conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
     MiniDFSCluster cluster = 
       new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
     cluster.waitActive();
     FileSystem fs = cluster.getFileSystem();
     DFSAdmin admin = new DFSAdmin(conf);
 
+    final String nnAddr = conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
+    final String webhdfsuri = WebHdfsFileSystem.SCHEME  + "://" + nnAddr;
+    System.out.println("webhdfsuri=" + webhdfsuri);
+    final FileSystem webhdfs = new Path(webhdfsuri).getFileSystem(conf);
+    
     try {
       
       //Test for deafult NameSpace Quota
@@ -882,6 +901,7 @@ public class TestQuota {
 
       // Should account for all 59 files (almost QUOTA_SIZE)
       c = fs.getContentSummary(dir);
+      checkContentSummary(c, webhdfs.getContentSummary(dir));
       assertEquals("Invalid space consumed", 59 * FILE_SIZE * 3,
           c.getSpaceConsumed());
       assertEquals("Invalid space consumed", QUOTA_SIZE - (59 * FILE_SIZE * 3),

Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java Thu Oct  6 01:16:48 2011
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.na
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.spy;
@@ -33,6 +34,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -73,7 +75,7 @@ public class TestEditLogJournalFailures 
   public void testSingleFailedEditsDirOnFlush() throws IOException {
     assertTrue(doAnEdit());
     // Invalidate one edits journal.
-    invalidateEditsDirAtIndex(0, true);
+    invalidateEditsDirAtIndex(0, true, false);
     // Make sure runtime.exit(...) hasn't been called at all yet.
     assertExitInvocations(0);
     assertTrue(doAnEdit());
@@ -86,8 +88,22 @@ public class TestEditLogJournalFailures 
   public void testAllEditsDirsFailOnFlush() throws IOException {
     assertTrue(doAnEdit());
     // Invalidate both edits journals.
-    invalidateEditsDirAtIndex(0, true);
-    invalidateEditsDirAtIndex(1, true);
+    invalidateEditsDirAtIndex(0, true, false);
+    invalidateEditsDirAtIndex(1, true, false);
+    // Make sure runtime.exit(...) hasn't been called at all yet.
+    assertExitInvocations(0);
+    assertTrue(doAnEdit());
+    // The previous edit could not be synced to any persistent storage, should
+    // have halted the NN.
+    assertExitInvocations(1);
+  }
+  
+  @Test
+  public void testAllEditsDirFailOnWrite() throws IOException {
+    assertTrue(doAnEdit());
+    // Invalidate both edits journals.
+    invalidateEditsDirAtIndex(0, true, true);
+    invalidateEditsDirAtIndex(1, true, true);
     // Make sure runtime.exit(...) hasn't been called at all yet.
     assertExitInvocations(0);
     assertTrue(doAnEdit());
@@ -100,7 +116,7 @@ public class TestEditLogJournalFailures 
   public void testSingleFailedEditsDirOnSetReadyToFlush() throws IOException {
     assertTrue(doAnEdit());
     // Invalidate one edits journal.
-    invalidateEditsDirAtIndex(0, false);
+    invalidateEditsDirAtIndex(0, false, false);
     // Make sure runtime.exit(...) hasn't been called at all yet.
     assertExitInvocations(0);
     assertTrue(doAnEdit());
@@ -117,16 +133,18 @@ public class TestEditLogJournalFailures 
    * @return the original <code>EditLogOutputStream</code> of the journal.
    */
   private EditLogOutputStream invalidateEditsDirAtIndex(int index,
-      boolean failOnFlush) throws IOException {
+      boolean failOnFlush, boolean failOnWrite) throws IOException {
     FSImage fsimage = cluster.getNamesystem().getFSImage();
     FSEditLog editLog = fsimage.getEditLog();
-    
 
-    FSEditLog.JournalAndStream jas = editLog.getJournals().get(index);
+    JournalAndStream jas = editLog.getJournals().get(index);
     EditLogFileOutputStream elos =
       (EditLogFileOutputStream) jas.getCurrentStream();
     EditLogFileOutputStream spyElos = spy(elos);
-    
+    if (failOnWrite) {
+      doThrow(new IOException("fail on write()")).when(spyElos).write(
+          (FSEditLogOp) any());
+    }
     if (failOnFlush) {
       doThrow(new IOException("fail on flush()")).when(spyElos).flush();
     } else {
@@ -151,7 +169,7 @@ public class TestEditLogJournalFailures 
     FSImage fsimage = cluster.getNamesystem().getFSImage();
     FSEditLog editLog = fsimage.getEditLog();
 
-    FSEditLog.JournalAndStream jas = editLog.getJournals().get(index);
+    JournalAndStream jas = editLog.getJournals().get(index);
     jas.setCurrentStreamForTests(elos);
   }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java Thu Oct  6 01:16:48 2011
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.protocol.H
 import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
 import org.apache.log4j.Level;
 
@@ -356,7 +357,7 @@ public class TestEditLogRace {
       FSImage fsimage = namesystem.getFSImage();
       FSEditLog editLog = fsimage.getEditLog();
 
-      FSEditLog.JournalAndStream jas = editLog.getJournals().get(0);
+      JournalAndStream jas = editLog.getJournals().get(0);
       EditLogFileOutputStream spyElos =
           spy((EditLogFileOutputStream)jas.getCurrentStream());
       jas.setCurrentStreamForTests(spyElos);

Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java Thu Oct  6 01:16:48 2011
@@ -28,7 +28,6 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.Set;
 
-import static org.mockito.Matchers.anyByte;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.spy;
 
@@ -45,7 +44,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
 
 import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getInProgressEditsFileName;
 import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getFinalizedEditsFileName;
@@ -123,7 +122,7 @@ public class TestStorageRestore {
     // simulate an error
     fi.getStorage().reportErrorsOnDirectories(al);
     
-    for (FSEditLog.JournalAndStream j : fi.getEditLog().getJournals()) {
+    for (JournalAndStream j : fi.getEditLog().getJournals()) {
       if (j.getManager() instanceof FileJournalManager) {
         FileJournalManager fm = (FileJournalManager)j.getManager();
         if (fm.getStorageDirectory().getRoot().equals(path2)

Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java Thu Oct  6 01:16:48 2011
@@ -17,14 +17,16 @@
  */
 package org.apache.hadoop.hdfs.web;
 
+import java.util.Map;
+
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.web.JsonUtil;
 import org.junit.Assert;
 import org.junit.Test;
+import org.mortbay.util.ajax.JSON;
 
 public class TestJsonUtil {
   static FileStatus toFileStatus(HdfsFileStatus f, String parent) {
@@ -46,7 +48,7 @@ public class TestJsonUtil {
     System.out.println("fstatus = " + fstatus);
     final String json = JsonUtil.toJsonString(status);
     System.out.println("json    = " + json.replace(",", ",\n  "));
-    final HdfsFileStatus s2 = JsonUtil.toFileStatus(JsonUtil.parse(json));
+    final HdfsFileStatus s2 = JsonUtil.toFileStatus((Map<?, ?>)JSON.parse(json));
     final FileStatus fs2 = toFileStatus(s2, parent);
     System.out.println("s2      = " + s2);
     System.out.println("fs2     = " + fs2);



Mime
View raw message