hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1416603 [1/2] - in /hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/ src/main/native/ s...
Date Mon, 03 Dec 2012 18:05:06 GMT
Author: suresh
Date: Mon Dec  3 18:04:51 2012
New Revision: 1416603

URL: http://svn.apache.org/viewvc?rev=1416603&view=rev
Log:
Merging trunk to HDFS-2802 branch.

Added:
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupState.java
      - copied unchanged from r1416602, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupState.java
Modified:
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/   (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/   (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/   (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/   (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/   (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/   (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/   (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
    hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1415804-1416602

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Dec  3 18:04:51 2012
@@ -176,6 +176,9 @@ Trunk (Unreleased)
     HDFS-4209. Clean up the addNode/addChild/addChildNoQuotaCheck methods in
     FSDirectory and INodeDirectory. (szetszwo)
 
+    HDFS-3358. Specify explicitly that the NN UI status total is talking
+    of persistent objects on heap. (harsh)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -646,6 +649,8 @@ Release 2.0.3-alpha - Unreleased 
     of it is undefined after the iteration or modifications of the map.
     (szetszwo)
 
+    HDFS-4231. BackupNode: Introduce BackupState. (shv)
+
 Release 2.0.2-alpha - 2012-09-07 
 
   INCOMPATIBLE CHANGES
@@ -2035,6 +2040,11 @@ Release 0.23.6 - UNRELEASED
 
   BUG FIXES
 
+    HDFS-4247. saveNamespace should be tolerant of dangling lease (daryn)
+
+    HDFS-4248. Renaming directories may incorrectly remove the paths in leases
+    under the tree.  (daryn via szetszwo)
+
 Release 0.23.5 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1415804-1416602

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java Mon Dec  3 18:04:51 2012
@@ -24,6 +24,7 @@ import java.net.SocketTimeoutException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.ha.ServiceFailedException;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.NameNodeProxies;
@@ -35,6 +36,7 @@ import org.apache.hadoop.hdfs.protocolPB
 import org.apache.hadoop.hdfs.protocolPB.JournalProtocolServerSideTranslatorPB;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
 import org.apache.hadoop.hdfs.server.protocol.FenceResponse;
 import org.apache.hadoop.hdfs.server.protocol.JournalInfo;
 import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
@@ -414,14 +416,23 @@ public class BackupNode extends NameNode
       + HdfsConstants.LAYOUT_VERSION + " actual "+ nsInfo.getLayoutVersion();
     return nsInfo;
   }
-  
+
   @Override
+  protected String getNameServiceId(Configuration conf) {
+    return DFSUtil.getBackupNameServiceId(conf);
+  }
+
+  protected HAState createHAState() {
+    return new BackupState();
+  }
+
+  @Override // NameNode
   protected NameNodeHAContext createHAContext() {
     return new BNHAContext();
   }
-  
+
   private class BNHAContext extends NameNodeHAContext {
-    @Override // NameNode
+    @Override // NameNodeHAContext
     public void checkOperation(OperationCategory op)
         throws StandbyException {
       if (op == OperationCategory.UNCHECKED ||
@@ -435,10 +446,42 @@ public class BackupNode extends NameNode
         throw new StandbyException(msg);
       }
     }
-  }
-  
-  @Override
-  protected String getNameServiceId(Configuration conf) {
-    return DFSUtil.getBackupNameServiceId(conf);
+
+    @Override // NameNodeHAContext
+    public void prepareToStopStandbyServices() throws ServiceFailedException {
+    }
+
+    /**
+     * Start services for BackupNode.
+     * <p>
+     * The following services should be muted
+     * (not run or not pass any control commands to DataNodes)
+     * on BackupNode:
+     * {@link LeaseManager.Monitor} protected by SafeMode.
+     * {@link BlockManager.ReplicationMonitor} protected by SafeMode.
+     * {@link HeartbeatManager.Monitor} protected by SafeMode.
+     * {@link DecommissionManager.Monitor} need to prohibit refreshNodes().
+     * {@link PendingReplicationBlocks.PendingReplicationMonitor} harmless,
+     * because ReplicationMonitor is muted.
+     */
+    @Override
+    public void startActiveServices() throws IOException {
+      try {
+        namesystem.startActiveServices();
+      } catch (Throwable t) {
+        doImmediateShutdown(t);
+      }
+    }
+
+    @Override
+    public void stopActiveServices() throws IOException {
+      try {
+        if (namesystem != null) {
+          namesystem.stopActiveServices();
+        }
+      } catch (Throwable t) {
+        doImmediateShutdown(t);
+      }
+    }
   }
 }

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Mon Dec  3 18:04:51 2012
@@ -383,7 +383,7 @@ public class FSDirectory implements Clos
     writeLock();
     try {
       // file is closed
-      file.setModificationTimeForce(now);
+      file.setModificationTime(now);
       fsImage.getEditLog().logCloseFile(path, file);
       if (NameNode.stateChangeLog.isDebugEnabled()) {
         NameNode.stateChangeLog.debug("DIR* FSDirectory.closeFile: "
@@ -585,8 +585,10 @@ public class FSDirectory implements Clos
               + src + " is renamed to " + dst);
         }
         // update modification time of dst and the parent of src
-        srcInodes[srcInodes.length-2].setModificationTime(timestamp);
-        dstInodes[dstInodes.length-2].setModificationTime(timestamp);
+        srcInodes[srcInodes.length-2].updateModificationTime(timestamp);
+        dstInodes[dstInodes.length-2].updateModificationTime(timestamp);
+        // update moved leases with new filename
+        getFSNamesystem().unprotectedChangeLease(src, dst);        
         return true;
       }
     } finally {
@@ -750,8 +752,10 @@ public class FSDirectory implements Clos
               "DIR* FSDirectory.unprotectedRenameTo: " + src
               + " is renamed to " + dst);
         }
-        srcInodes[srcInodes.length - 2].setModificationTime(timestamp);
-        dstInodes[dstInodes.length - 2].setModificationTime(timestamp);
+        srcInodes[srcInodes.length - 2].updateModificationTime(timestamp);
+        dstInodes[dstInodes.length - 2].updateModificationTime(timestamp);
+        // update moved lease with new filename
+        getFSNamesystem().unprotectedChangeLease(src, dst);
 
         // Collect the blocks and remove the lease for previous dst
         int filesDeleted = 0;
@@ -986,12 +990,12 @@ public class FSDirectory implements Clos
       if(nodeToRemove == null) continue;
       
       nodeToRemove.setBlocks(null);
-      trgParent.removeChild(nodeToRemove);
+      trgParent.removeChild(nodeToRemove, trgINodesInPath.getLatestSnapshot());
       count++;
     }
     
-    trgInode.setModificationTimeForce(timestamp);
-    trgParent.setModificationTime(timestamp);
+    trgInode.setModificationTime(timestamp);
+    trgParent.updateModificationTime(timestamp);
     // update quota on the parent directory ('count' files removed, 0 space)
     unprotectedUpdateCount(trgINodesInPath, trgINodes.length-1, -count, 0);
   }
@@ -1129,7 +1133,7 @@ public class FSDirectory implements Clos
       return 0;
     }
     // set the parent's modification time
-    inodes[inodes.length - 2].setModificationTime(mtime);
+    inodes[inodes.length - 2].updateModificationTime(mtime);
     int filesRemoved = targetNode.collectSubtreeBlocksAndClear(collectedBlocks);
     if (NameNode.stateChangeLog.isDebugEnabled()) {
       NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: "
@@ -1165,10 +1169,10 @@ public class FSDirectory implements Clos
   /**
    * Replaces the specified INode.
    */
-  private void replaceINodeUnsynced(String path, INode oldnode, INode newnode
-      ) throws IOException {    
+  private void replaceINodeUnsynced(String path, INode oldnode, INode newnode,
+      Snapshot latestSnapshot) throws IOException {    
     //remove the old node from the namespace 
-    if (!oldnode.removeNode()) {
+    if (!oldnode.removeNode(latestSnapshot)) {
       final String mess = "FSDirectory.replaceINodeUnsynced: failed to remove "
           + path;
       NameNode.stateChangeLog.warn("DIR* " + mess);
@@ -1183,10 +1187,10 @@ public class FSDirectory implements Clos
    * Replaces the specified INodeDirectory.
    */
   public void replaceINodeDirectory(String path, INodeDirectory oldnode,
-      INodeDirectory newnode) throws IOException {    
+      INodeDirectory newnode, Snapshot latestSnapshot) throws IOException {    
     writeLock();
     try {
-      replaceINodeUnsynced(path, oldnode, newnode);
+      replaceINodeUnsynced(path, oldnode, newnode, latestSnapshot);
 
       //update children's parent directory
       for(INode i : newnode.getChildrenList(null)) {
@@ -1200,11 +1204,11 @@ public class FSDirectory implements Clos
   /**
    * Replaces the specified INodeFile with the specified one.
    */
-  public void replaceNode(String path, INodeFile oldnode, INodeFile newnode
-      ) throws IOException {    
+  public void replaceNode(String path, INodeFile oldnode, INodeFile newnode,
+      Snapshot latestSnapshot) throws IOException {
     writeLock();
     try {
-      replaceINodeUnsynced(path, oldnode, newnode);
+      replaceINodeUnsynced(path, oldnode, newnode, latestSnapshot);
       
       //Currently, oldnode and newnode are assumed to contain the same blocks.
       //Otherwise, blocks need to be removed from the blocksMap.
@@ -1273,13 +1277,9 @@ public class FSDirectory implements Clos
     String srcs = normalizePath(src);
     readLock();
     try {
-      INode targetNode = rootDir.getNode(srcs, resolveLink);
-      if (targetNode == null) {
-        return null;
-      }
-      else {
-        return createFileStatus(HdfsFileStatus.EMPTY_NAME, targetNode);
-      }
+      final INodesInPath inodesInPath = rootDir.getINodesInPath(srcs, resolveLink);
+      final INode i = inodesInPath.getINode(0);
+      return i == null? null: createFileStatus(HdfsFileStatus.EMPTY_NAME, i);
     } finally {
       readUnlock();
     }
@@ -1303,9 +1303,16 @@ public class FSDirectory implements Clos
    * Get {@link INode} associated with the file / directory.
    */
   public INode getINode(String src) throws UnresolvedLinkException {
+    return getINodesInPath(src).getINode(0);
+  }
+
+  /**
+   * Get {@link INode} associated with the file / directory.
+   */
+  public INodesInPath getINodesInPath(String src) throws UnresolvedLinkException {
     readLock();
     try {
-      return rootDir.getNode(src, true);
+      return rootDir.getINodesInPath(src, true);
     } finally {
       readUnlock();
     }
@@ -1800,7 +1807,8 @@ public class FSDirectory implements Clos
     if (inodes[pos-1] == null) {
       throw new NullPointerException("Panic: parent does not exist");
     }
-    final boolean added = ((INodeDirectory)inodes[pos-1]).addChild(child, true);
+    final boolean added = ((INodeDirectory)inodes[pos-1]).addChild(child, true,
+        inodesInPath.getLatestSnapshot());
     if (!added) {
       updateCount(inodesInPath, pos, -counts.getNsCount(), -counts.getDsCount(), true);
     }
@@ -1824,7 +1832,8 @@ public class FSDirectory implements Clos
   private INode removeLastINode(final INodesInPath inodesInPath) {
     final INode[] inodes = inodesInPath.getINodes();
     final int pos = inodes.length - 1;
-    INode removedNode = ((INodeDirectory)inodes[pos-1]).removeChild(inodes[pos]);
+    INode removedNode = ((INodeDirectory)inodes[pos-1]).removeChild(inodes[pos],
+        inodesInPath.getLatestSnapshot());
     if (removedNode != null) {
       INode.DirCounts counts = new INode.DirCounts();
       removedNode.spaceConsumedInTree(counts);
@@ -1965,8 +1974,8 @@ public class FSDirectory implements Clos
     }
     
     String srcs = normalizePath(src);
-    final INode[] inodes = rootDir.getMutableINodesInPath(srcs, true)
-        .getINodes();
+    final INodesInPath inodesInPath = rootDir.getMutableINodesInPath(srcs, true);
+    final INode[] inodes = inodesInPath.getINodes();
     INodeDirectory dirNode = INodeDirectory.valueOf(inodes[inodes.length-1], srcs);
     if (dirNode.isRoot() && nsQuota == HdfsConstants.QUOTA_RESET) {
       throw new IllegalArgumentException("Cannot clear namespace quota on root.");
@@ -1988,7 +1997,7 @@ public class FSDirectory implements Clos
           INodeDirectory newNode = new INodeDirectory(dirNode);
           INodeDirectory parent = (INodeDirectory)inodes[inodes.length-2];
           dirNode = newNode;
-          parent.replaceChild(newNode);
+          parent.replaceChild(newNode, inodesInPath.getLatestSnapshot());
         }
       } else {
         // a non-quota directory; so replace it with a directory with quota
@@ -1997,7 +2006,7 @@ public class FSDirectory implements Clos
         // non-root directory node; parent != null
         INodeDirectory parent = (INodeDirectory)inodes[inodes.length-2];
         dirNode = newNode;
-        parent.replaceChild(newNode);
+        parent.replaceChild(newNode, inodesInPath.getLatestSnapshot());
       }
       return (oldNsQuota != nsQuota || oldDsQuota != dsQuota) ? dirNode : null;
     }
@@ -2061,7 +2070,7 @@ public class FSDirectory implements Clos
     assert hasWriteLock();
     boolean status = false;
     if (mtime != -1) {
-      inode.setModificationTimeForce(mtime);
+      inode.setModificationTime(mtime);
       status = true;
     }
     if (atime != -1) {

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Mon Dec  3 18:04:51 2012
@@ -31,7 +31,6 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
@@ -58,6 +57,7 @@ import org.apache.hadoop.hdfs.server.nam
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.TimesOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateBlocksOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateMasterKeyOp;
+import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
 import org.apache.hadoop.hdfs.util.Holder;
 
@@ -246,7 +246,8 @@ public class FSEditLogLoader {
       // 3. OP_ADD to open file for append
 
       // See if the file already exists (persistBlocks call)
-      INodeFile oldFile = getINodeFile(fsDir, addCloseOp.path);
+      final INodesInPath inodesInPath = fsDir.getINodesInPath(addCloseOp.path);
+      INodeFile oldFile = toINodeFile(inodesInPath.getINode(0), addCloseOp.path);
       INodeFile newFile = oldFile;
       if (oldFile == null) { // this is OP_ADD on a new file (case 1)
         // versions > 0 support per file replication
@@ -272,7 +273,7 @@ public class FSEditLogLoader {
           }
           fsNamesys.prepareFileForWrite(addCloseOp.path, oldFile,
               addCloseOp.clientName, addCloseOp.clientMachine, null,
-              false);
+              false, inodesInPath.getLatestSnapshot());
           newFile = getINodeFile(fsDir, addCloseOp.path);
         }
       }
@@ -282,7 +283,7 @@ public class FSEditLogLoader {
       
       // Update the salient file attributes.
       newFile.setAccessTime(addCloseOp.atime);
-      newFile.setModificationTimeForce(addCloseOp.mtime);
+      newFile.setModificationTime(addCloseOp.mtime);
       updateBlocks(fsDir, addCloseOp, newFile);
       break;
     }
@@ -296,7 +297,8 @@ public class FSEditLogLoader {
             " clientMachine " + addCloseOp.clientMachine);
       }
 
-      INodeFile oldFile = getINodeFile(fsDir, addCloseOp.path);
+      final INodesInPath inodesInPath = fsDir.getINodesInPath(addCloseOp.path);
+      INodeFile oldFile = toINodeFile(inodesInPath.getINode(0), addCloseOp.path);
       if (oldFile == null) {
         throw new IOException("Operation trying to close non-existent file " +
             addCloseOp.path);
@@ -304,7 +306,7 @@ public class FSEditLogLoader {
       
       // Update the salient file attributes.
       oldFile.setAccessTime(addCloseOp.atime);
-      oldFile.setModificationTimeForce(addCloseOp.mtime);
+      oldFile.setModificationTime(addCloseOp.mtime);
       updateBlocks(fsDir, addCloseOp, oldFile);
 
       // Now close the file
@@ -322,7 +324,8 @@ public class FSEditLogLoader {
         INodeFileUnderConstruction ucFile = (INodeFileUnderConstruction) oldFile;
         fsNamesys.leaseManager.removeLeaseWithPrefixPath(addCloseOp.path);
         INodeFile newFile = ucFile.convertToInodeFile();
-        fsDir.replaceNode(addCloseOp.path, ucFile, newFile);
+        fsDir.replaceNode(addCloseOp.path, ucFile, newFile,
+            inodesInPath.getLatestSnapshot());
       }
       break;
     }
@@ -360,10 +363,8 @@ public class FSEditLogLoader {
     }
     case OP_RENAME_OLD: {
       RenameOldOp renameOp = (RenameOldOp)op;
-      HdfsFileStatus dinfo = fsDir.getFileInfo(renameOp.dst, false);
       fsDir.unprotectedRenameTo(renameOp.src, renameOp.dst,
                                 renameOp.timestamp);
-      fsNamesys.unprotectedChangeLease(renameOp.src, renameOp.dst, dinfo);
       break;
     }
     case OP_DELETE: {
@@ -433,11 +434,8 @@ public class FSEditLogLoader {
     }
     case OP_RENAME: {
       RenameOp renameOp = (RenameOp)op;
-
-      HdfsFileStatus dinfo = fsDir.getFileInfo(renameOp.dst, false);
       fsDir.unprotectedRenameTo(renameOp.src, renameOp.dst,
                                 renameOp.timestamp, renameOp.options);
-      fsNamesys.unprotectedChangeLease(renameOp.src, renameOp.dst, dinfo);
       break;
     }
     case OP_GET_DELEGATION_TOKEN: {
@@ -512,7 +510,11 @@ public class FSEditLogLoader {
   
   private static INodeFile getINodeFile(FSDirectory fsDir, String path)
       throws IOException {
-    INode inode = fsDir.getINode(path);
+    return toINodeFile(fsDir.getINode(path), path);
+  }
+
+  private static INodeFile toINodeFile(INode inode, String path)
+      throws IOException {
     if (inode != null) {
       if (!(inode instanceof INodeFile)) {
         throw new IOException("Operation trying to get non-file " + path);

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Mon Dec  3 18:04:51 2012
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.protocol.L
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
+import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
 import org.apache.hadoop.hdfs.util.ReadOnlyList;
 import org.apache.hadoop.io.MD5Hash;
 import org.apache.hadoop.io.Text;
@@ -202,7 +203,7 @@ class FSImageFormat {
     if (nsQuota != -1 || dsQuota != -1) {
       fsDir.rootDir.setQuota(nsQuota, dsQuota);
     }
-    fsDir.rootDir.setModificationTime(root.getModificationTime());
+    fsDir.rootDir.cloneModificationTime(root);
     fsDir.rootDir.clonePermissionStatus(root);    
   }
 
@@ -305,7 +306,7 @@ class FSImageFormat {
    */
   void addToParent(INodeDirectory parent, INode child) {
     // NOTE: This does not update space counts for parents
-    if (!parent.addChild(child, false)) {
+    if (!parent.addChild(child, false, null)) {
       return;
     }
     namesystem.dir.cacheName(child);
@@ -388,8 +389,9 @@ class FSImageFormat {
 
         // verify that file exists in namespace
         String path = cons.getLocalName();
-        INodeFile oldnode = INodeFile.valueOf(fsDir.getINode(path), path);
-        fsDir.replaceNode(path, oldnode, cons);
+        final INodesInPath inodesInPath = fsDir.getINodesInPath(path);
+        INodeFile oldnode = INodeFile.valueOf(inodesInPath.getINode(0), path);
+        fsDir.replaceNode(path, oldnode, cons, inodesInPath.getLatestSnapshot());
         namesystem.leaseManager.addLease(cons.getClientName(), path); 
       }
     }

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Mon Dec  3 18:04:51 2012
@@ -121,6 +121,7 @@ import org.apache.hadoop.fs.UnresolvedLi
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
 import org.apache.hadoop.ha.ServiceFailedException;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.HAUtil;
@@ -165,7 +166,6 @@ import org.apache.hadoop.hdfs.server.nam
 import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
 import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
-import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
 import org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
@@ -173,7 +173,9 @@ import org.apache.hadoop.hdfs.server.nam
 import org.apache.hadoop.hdfs.server.namenode.ha.StandbyState;
 import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
 import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeFileWithLink;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
 import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
@@ -1036,7 +1038,8 @@ public class FSNamesystem implements Nam
     long totalInodes = this.dir.totalInodes();
     long totalBlocks = this.getBlocksTotal();
     out.println(totalInodes + " files and directories, " + totalBlocks
-        + " blocks = " + (totalInodes + totalBlocks) + " total");
+        + " blocks = " + (totalInodes + totalBlocks)
+        + " total filesystem objects");
 
     blockManager.metaSave(out);
   }
@@ -1820,7 +1823,9 @@ public class FSNamesystem implements Nam
     try {
       blockManager.verifyReplication(src, replication, clientMachine);
       boolean create = flag.contains(CreateFlag.CREATE);
-      final INode myFile = dir.getINode(src);
+      
+      final INodesInPath inodesInPath = dir.getINodesInPath(src);
+      final INode myFile = inodesInPath.getINode(0);
       if (myFile == null) {
         if (!create) {
           throw new FileNotFoundException("failed to overwrite or append to non-existent file "
@@ -1847,8 +1852,8 @@ public class FSNamesystem implements Nam
 
       if (append && myFile != null) {
         final INodeFile f = INodeFile.valueOf(myFile, src); 
-        return prepareFileForWrite(
-            src, f, holder, clientMachine, clientNode, true);
+        return prepareFileForWrite(src, f, holder, clientMachine, clientNode,
+            true, inodesInPath.getLatestSnapshot());
       } else {
        // Now we can add the name to the filesystem. This file has no
        // blocks associated with it.
@@ -1896,7 +1901,7 @@ public class FSNamesystem implements Nam
    */
   LocatedBlock prepareFileForWrite(String src, INodeFile file,
       String leaseHolder, String clientMachine, DatanodeDescriptor clientNode,
-      boolean writeToEditLog) throws IOException {
+      boolean writeToEditLog, Snapshot latestSnapshot) throws IOException {
     //TODO SNAPSHOT: INodeFileUnderConstruction with link
     INodeFileUnderConstruction cons = new INodeFileUnderConstruction(
                                     file.getLocalNameBytes(),
@@ -1908,7 +1913,7 @@ public class FSNamesystem implements Nam
                                     leaseHolder,
                                     clientMachine,
                                     clientNode);
-    dir.replaceNode(src, file, cons);
+    dir.replaceNode(src, file, cons, latestSnapshot);
     leaseManager.addLease(cons.getClientName(), src);
     
     LocatedBlock ret = blockManager.convertLastBlockToUnderConstruction(cons);
@@ -2158,7 +2163,8 @@ public class FSNamesystem implements Nam
       // have we exceeded the configured limit of fs objects.
       checkFsObjectLimit();
 
-      INodeFileUnderConstruction pendingFile = checkLease(src, clientName);
+      final INodeFileUnderConstruction pendingFile = checkLease(
+          src, clientName, dir.getINode(src));
       BlockInfo lastBlockInFile = pendingFile.getLastBlock();
       if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) {
         // The block that the client claims is the current last block
@@ -2294,7 +2300,8 @@ public class FSNamesystem implements Nam
       }
 
       //check lease
-      final INodeFileUnderConstruction file = checkLease(src, clientName);
+      final INodeFileUnderConstruction file = checkLease(
+          src, clientName, dir.getINode(src));
       clientnode = file.getClientNode();
       preferredblocksize = file.getPreferredBlockSize();
 
@@ -2340,7 +2347,9 @@ public class FSNamesystem implements Nam
         throw new SafeModeException("Cannot abandon block " + b +
                                     " for fle" + src, safeMode);
       }
-      INodeFileUnderConstruction file = checkLease(src, holder);
+      final INodesInPath inodesInPath = checkLease(src, holder);
+      final INodeFileUnderConstruction file
+          = (INodeFileUnderConstruction)inodesInPath.getINode(0); 
       dir.removeBlock(src, file, ExtendedBlock.getLocalBlock(b));
       if(NameNode.stateChangeLog.isDebugEnabled()) {
         NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: "
@@ -2357,11 +2366,13 @@ public class FSNamesystem implements Nam
     return true;
   }
   
-  // make sure that we still have the lease on this file.
-  private INodeFileUnderConstruction checkLease(String src, String holder) 
+  /** make sure that we still have the lease on this file. */
+  private INodesInPath checkLease(String src, String holder) 
       throws LeaseExpiredException, UnresolvedLinkException {
     assert hasReadOrWriteLock();
-    return checkLease(src, holder, dir.getINode(src));
+    final INodesInPath inodesInPath = dir.getINodesInPath(src);
+    checkLease(src, holder, inodesInPath.getINode(0));
+    return inodesInPath;
   }
 
   private INodeFileUnderConstruction checkLease(String src, String holder,
@@ -2424,9 +2435,11 @@ public class FSNamesystem implements Nam
       throw new SafeModeException("Cannot complete file " + src, safeMode);
     }
 
-    INodeFileUnderConstruction pendingFile;
+    final INodesInPath inodesInPath;
+    final INodeFileUnderConstruction pendingFile;
     try {
-      pendingFile = checkLease(src, holder);
+      inodesInPath = checkLease(src, holder);
+      pendingFile = (INodeFileUnderConstruction)inodesInPath.getINode(0); 
     } catch (LeaseExpiredException lee) {
       final INode inode = dir.getINode(src);
       if (inode != null && inode instanceof INodeFile && !inode.isUnderConstruction()) {
@@ -2454,7 +2467,8 @@ public class FSNamesystem implements Nam
       return false;
     }
 
-    finalizeINodeFileUnderConstruction(src, pendingFile);
+    finalizeINodeFileUnderConstruction(src, pendingFile,
+        inodesInPath.getLatestSnapshot());
 
     NameNode.stateChangeLog.info("DIR* completeFile: " + src + " is closed by "
         + holder);
@@ -2595,15 +2609,15 @@ public class FSNamesystem implements Nam
     if (isPermissionEnabled) {
       //We should not be doing this.  This is move() not renameTo().
       //but for now,
+      //NOTE: yes, this is bad!  it's assuming much lower level behavior
+      //      of rewriting the dst
       String actualdst = dir.isDir(dst)?
           dst + Path.SEPARATOR + new Path(src).getName(): dst;
       checkParentAccess(src, FsAction.WRITE);
       checkAncestorAccess(actualdst, FsAction.WRITE);
     }
 
-    HdfsFileStatus dinfo = dir.getFileInfo(dst, false);
     if (dir.renameTo(src, dst)) {
-      unprotectedChangeLease(src, dst, dinfo);     // update lease with new filename
       return true;
     }
     return false;
@@ -2654,9 +2668,7 @@ public class FSNamesystem implements Nam
       checkAncestorAccess(dst, FsAction.WRITE);
     }
 
-    HdfsFileStatus dinfo = dir.getFileInfo(dst, false);
     dir.renameTo(src, dst, options);
-    unprotectedChangeLease(src, dst, dinfo); // update lease with new filename
   }
   
   /**
@@ -3036,7 +3048,8 @@ public class FSNamesystem implements Nam
       if (isInSafeMode()) {
         throw new SafeModeException("Cannot fsync file " + src, safeMode);
       }
-      INodeFileUnderConstruction pendingFile  = checkLease(src, clientName);
+      final INodeFileUnderConstruction pendingFile  = checkLease(
+          src, clientName, dir.getINode(src));
       if (lastBlockLength > 0) {
         pendingFile.updateLengthOfLastBlock(lastBlockLength);
       }
@@ -3068,8 +3081,9 @@ public class FSNamesystem implements Nam
     assert !isInSafeMode();
     assert hasWriteLock();
 
+    final INodesInPath inodesInPath = dir.getINodesInPath(src);
     final INodeFileUnderConstruction pendingFile
-        = INodeFileUnderConstruction.valueOf(dir.getINode(src), src);
+        = INodeFileUnderConstruction.valueOf(inodesInPath.getINode(0), src);
     int nrBlocks = pendingFile.numBlocks();
     BlockInfo[] blocks = pendingFile.getBlocks();
 
@@ -3086,7 +3100,8 @@ public class FSNamesystem implements Nam
     // If there are no incomplete blocks associated with this file,
     // then reap lease immediately and close the file.
     if(nrCompleteBlocks == nrBlocks) {
-      finalizeINodeFileUnderConstruction(src, pendingFile);
+      finalizeINodeFileUnderConstruction(src, pendingFile,
+          inodesInPath.getLatestSnapshot());
       NameNode.stateChangeLog.warn("BLOCK*"
         + " internalReleaseLease: All existing blocks are COMPLETE,"
         + " lease removed, file closed.");
@@ -3134,7 +3149,8 @@ public class FSNamesystem implements Nam
       // Close file if committed blocks are minimally replicated
       if(penultimateBlockMinReplication &&
           blockManager.checkMinReplication(lastBlock)) {
-        finalizeINodeFileUnderConstruction(src, pendingFile);
+        finalizeINodeFileUnderConstruction(src, pendingFile,
+            inodesInPath.getLatestSnapshot());
         NameNode.stateChangeLog.warn("BLOCK*"
           + " internalReleaseLease: Committed blocks are minimally replicated,"
           + " lease removed, file closed.");
@@ -3212,7 +3228,7 @@ public class FSNamesystem implements Nam
   }
 
   private void finalizeINodeFileUnderConstruction(String src, 
-      INodeFileUnderConstruction pendingFile) 
+      INodeFileUnderConstruction pendingFile, Snapshot latestSnapshot) 
       throws IOException, UnresolvedLinkException {
     assert hasWriteLock();
     leaseManager.removeLease(pendingFile.getClientName(), src);
@@ -3220,7 +3236,7 @@ public class FSNamesystem implements Nam
     // The file is no longer pending.
     // Create permanent INode, update blocks
     INodeFile newFile = pendingFile.convertToInodeFile();
-    dir.replaceNode(src, pendingFile, newFile);
+    dir.replaceNode(src, pendingFile, newFile, latestSnapshot);
 
     // close file and persist block allocations for this file
     dir.closeFile(src, newFile);
@@ -3312,7 +3328,8 @@ public class FSNamesystem implements Nam
         commitOrCompleteLastBlock(pendingFile, storedBlock);
 
         //remove lease, close file
-        finalizeINodeFileUnderConstruction(src, pendingFile);
+        finalizeINodeFileUnderConstruction(src, pendingFile,
+            INodeDirectorySnapshottable.findLatestSnapshot(pendingFile));
       } else {
         // If this commit does not want to close the file, persist blocks
         dir.persistBlocks(src, pendingFile);
@@ -3483,9 +3500,9 @@ public class FSNamesystem implements Nam
   private NNHAStatusHeartbeat createHaStatusHeartbeat() {
     HAState state = haContext.getState();
     NNHAStatusHeartbeat.State hbState;
-    if (state instanceof ActiveState) {
+    if (state.getServiceState() == HAServiceState.ACTIVE) {
       hbState = NNHAStatusHeartbeat.State.ACTIVE;
-    } else if (state instanceof StandbyState) {
+    } else if (state.getServiceState() == HAServiceState.STANDBY) {
       hbState = NNHAStatusHeartbeat.State.STANDBY;      
     } else {
       throw new AssertionError("Invalid state: " + state.getClass());
@@ -4939,31 +4956,9 @@ public class FSNamesystem implements Nam
 
   // rename was successful. If any part of the renamed subtree had
   // files that were being written to, update with new filename.
-  void unprotectedChangeLease(String src, String dst, HdfsFileStatus dinfo) {
-    String overwrite;
-    String replaceBy;
+  void unprotectedChangeLease(String src, String dst) {
     assert hasWriteLock();
-
-    boolean destinationExisted = true;
-    if (dinfo == null) {
-      destinationExisted = false;
-    }
-
-    if (destinationExisted && dinfo.isDir()) {
-      Path spath = new Path(src);
-      Path parent = spath.getParent();
-      if (parent.isRoot()) {
-        overwrite = parent.toString();
-      } else {
-        overwrite = parent.toString() + Path.SEPARATOR;
-      }
-      replaceBy = dst + Path.SEPARATOR;
-    } else {
-      overwrite = src;
-      replaceBy = dst;
-    }
-
-    leaseManager.changeLease(src, dst, overwrite, replaceBy);
+    leaseManager.changeLease(src, dst);
   }
 
   /**
@@ -4974,19 +4969,13 @@ public class FSNamesystem implements Nam
     // lock on our behalf. If we took the read lock here, we could block
     // for fairness if a writer is waiting on the lock.
     synchronized (leaseManager) {
-      out.writeInt(leaseManager.countPath()); // write the size
-
-      for (Lease lease : leaseManager.getSortedLeases()) {
-        for(String path : lease.getPaths()) {
-          // verify that path exists in namespace
-          final INodeFileUnderConstruction cons;
-          try {
-            cons = INodeFileUnderConstruction.valueOf(dir.getINode(path), path);
-          } catch (UnresolvedLinkException e) {
-            throw new AssertionError("Lease files should reside on this FS");
-          }
-          FSImageSerialization.writeINodeUnderConstruction(out, cons, path);
-        }
+      Map<String, INodeFileUnderConstruction> nodes =
+          leaseManager.getINodesUnderConstruction();
+      out.writeInt(nodes.size()); // write the size    
+      for (Map.Entry<String, INodeFileUnderConstruction> entry
+           : nodes.entrySet()) {
+        FSImageSerialization.writeINodeUnderConstruction(
+            out, entry.getValue(), entry.getKey());
       }
     }
   }

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java Mon Dec  3 18:04:51 2012
@@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.hdfs.util.ReadOnlyList;
 import org.apache.hadoop.util.StringUtils;
 
@@ -46,8 +47,33 @@ import com.google.common.primitives.Sign
  */
 @InterfaceAudience.Private
 public abstract class INode implements Comparable<byte[]> {
+  /** A dummy INode which can be used as a probe object. */
+  public static final INode DUMMY = new INode() {
+    @Override
+    int collectSubtreeBlocksAndClear(BlocksMapUpdateInfo info) {
+      throw new UnsupportedOperationException();
+    }
+    @Override
+    long[] computeContentSummary(long[] summary) {
+      throw new UnsupportedOperationException();
+    }
+    @Override
+    DirCounts spaceConsumedInTree(DirCounts counts) {
+      throw new UnsupportedOperationException();
+    }
+  };
   static final ReadOnlyList<INode> EMPTY_READ_ONLY_LIST
       = ReadOnlyList.Util.emptyList();
+  /**
+   * Assert that the snapshot parameter must be null since
+   * this class only take care current state. 
+   * Subclasses should override the methods for handling the snapshot states.
+   */
+  static void assertNull(Snapshot snapshot) {
+    if (snapshot != null) {
+      throw new AssertionError("snapshot is not null: " + snapshot);
+    }
+  }
 
 
   /** Wrapper of two counters for namespace consumed and diskspace consumed. */
@@ -120,9 +146,12 @@ public abstract class INode implements C
    * should not modify it.
    */
   private long permission = 0L;
-  protected INodeDirectory parent = null;
-  protected long modificationTime = 0L;
-  protected long accessTime = 0L;
+  INodeDirectory parent = null;
+  private long modificationTime = 0L;
+  private long accessTime = 0L;
+  
+  /** For creating the a {@link #DUMMY} object. */
+  private INode() {}
 
   private INode(byte[] name, long permission, INodeDirectory parent,
       long modificationTime, long accessTime) {
@@ -149,8 +178,8 @@ public abstract class INode implements C
   
   /** @param other Other node to be copied */
   INode(INode other) {
-    this(other.getLocalNameBytes(), other.permission, other.getParent(), 
-        other.getModificationTime(), other.getAccessTime());
+    this(other.name, other.permission, other.parent, 
+        other.modificationTime, other.accessTime);
   }
 
   /**
@@ -290,13 +319,13 @@ public abstract class INode implements C
    * Set local file name
    */
   public void setLocalName(String name) {
-    this.name = DFSUtil.string2Bytes(name);
+    setLocalName(DFSUtil.string2Bytes(name));
   }
 
   /**
    * Set local file name
    */
-  void setLocalName(byte[] name) {
+  public void setLocalName(byte[] name) {
     this.name = name;
   }
 
@@ -316,7 +345,7 @@ public abstract class INode implements C
    * Get parent directory 
    * @return parent INode
    */
-  INodeDirectory getParent() {
+  public INodeDirectory getParent() {
     return this.parent;
   }
 
@@ -336,17 +365,21 @@ public abstract class INode implements C
   /**
    * Set last modification time of inode.
    */
-  public void setModificationTime(long modtime) {
+  public void updateModificationTime(long modtime) {
     assert isDirectory();
     if (this.modificationTime <= modtime) {
-      this.modificationTime = modtime;
+      setModificationTime(modtime);
     }
   }
 
+  void cloneModificationTime(INode that) {
+    this.modificationTime = that.modificationTime;
+  }
+
   /**
    * Always set the last modification time of inode.
    */
-  void setModificationTimeForce(long modtime) {
+  void setModificationTime(long modtime) {
     this.modificationTime = modtime;
   }
 
@@ -431,11 +464,11 @@ public abstract class INode implements C
     return buf.toString();
   }
 
-  public boolean removeNode() {
+  public boolean removeNode(Snapshot latestSnapshot) {
     if (parent == null) {
       return false;
     } else {
-      parent.removeChild(this);
+      parent.removeChild(this, latestSnapshot);
       parent = null;
       return true;
     }

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java Mon Dec  3 18:04:51 2012
@@ -54,7 +54,7 @@ public class INodeDirectory extends INod
     }
     return (INodeDirectory)inode; 
   }
-
+  
   protected static final int DEFAULT_FILES_PER_DIRECTORY = 5;
   final static String ROOT_NAME = "";
 
@@ -99,32 +99,56 @@ public class INodeDirectory extends INod
     }
   }
 
-  private int searchChildren(INode inode) {
+  public int searchChildren(INode inode) {
     return Collections.binarySearch(children, inode.getLocalNameBytes());
   }
 
-  INode removeChild(INode node) {
+  public int searchChildrenForExistingINode(INode inode) {
+    final int i = searchChildren(inode);
+    if (i < 0) {
+      throw new AssertionError("Child not found: inode=" + inode);
+    }
+    return i;
+  }
+
+  public INode removeChild(INode node, Snapshot latestSnapshot) {
     assertChildrenNonNull();
+
+    if (latestSnapshot != null) {
+      final INodeDirectoryWithSnapshot dir
+          = INodeDirectoryWithSnapshot.replaceDir(this, latestSnapshot);
+      return dir.removeChild(node, latestSnapshot);
+    }
+
     final int i = searchChildren(node);
     return i >= 0? children.remove(i): null;
   }
-
   /** Replace a child that has the same name as newChild by newChild.
    * 
    * @param newChild Child node to be added
    */
-  void replaceChild(INode newChild) {
+  public INode replaceChild(INodeDirectory newChild, Snapshot latestSnapshot) {
     assertChildrenNonNull();
 
-    final int low = searchChildren(newChild);
-    if (low>=0) { // an old child exists so replace by the newChild
-      children.set(low, newChild);
-    } else {
-      throw new IllegalArgumentException("No child exists to be replaced");
+    if (latestSnapshot != null) {
+      final INodeDirectoryWithSnapshot dir
+          = INodeDirectoryWithSnapshot.replaceDir(this, latestSnapshot);
+      return dir.replaceChild(newChild, latestSnapshot);
+    }
+
+    // find the old child and replace it
+    final int low = searchChildrenForExistingINode(newChild);
+    final INode oldChild = children.set(low, newChild);
+    // set the parent of the children of the child.
+    for(INode i : newChild.getChildrenList(null)) {
+      i.parent = newChild;
     }
+    return oldChild;
   }
 
-  private INode getChild(byte[] name, Snapshot snapshot) {
+  public INode getChild(byte[] name, Snapshot snapshot) {
+    assertNull(snapshot);
+
     final ReadOnlyList<INode> c = getChildrenList(snapshot);
     final int i = ReadOnlyList.Util.binarySearch(c, name);
     return i < 0? null: c.get(i);
@@ -361,7 +385,14 @@ public class INodeDirectory extends INod
    * @return false if the child with this name already exists; 
    *         otherwise, return true;
    */
-  public boolean addChild(final INode node, final boolean setModTime) {
+  public boolean addChild(final INode node, final boolean setModTime,
+      Snapshot latestSnapshot) {
+    if (latestSnapshot != null) {
+      final INodeDirectoryWithSnapshot dir
+          = INodeDirectoryWithSnapshot.replaceDir(this, latestSnapshot);
+      return dir.addChild(node, setModTime, latestSnapshot);
+    }
+
     if (children == null) {
       children = new ArrayList<INode>(DEFAULT_FILES_PER_DIRECTORY);
     }
@@ -372,8 +403,9 @@ public class INodeDirectory extends INod
     node.parent = this;
     children.add(-low - 1, node);
     // update modification time of the parent directory
-    if (setModTime)
-      setModificationTime(node.getModificationTime());
+    if (setModTime) {
+      updateModificationTime(node.getModificationTime());
+    }
     if (node.getGroupName() == null) {
       node.setGroup(getGroupName());
     }
@@ -400,20 +432,28 @@ public class INodeDirectory extends INod
     }
     newNode.setLocalName(pathComponents[pathComponents.length - 1]);
     // insert into the parent children list
-    INodeDirectory parent = getParent(pathComponents);
-    return parent.addChild(newNode, true);
+    INodesInPath inodes =  getExistingPathINodes(pathComponents, 2, false);
+    INodeDirectory parent = INodeDirectory.valueOf(inodes.inodes[0], pathComponents);
+    return parent.addChild(newNode, true, inodes.getLatestSnapshot());
   }
 
   INodeDirectory getParent(byte[][] pathComponents
       ) throws FileNotFoundException, PathIsNotDirectoryException,
       UnresolvedLinkException {
+    return (INodeDirectory)getParentINodesInPath(pathComponents).getINode(0);
+  }
+
+  INodesInPath getParentINodesInPath(byte[][] pathComponents
+      ) throws FileNotFoundException, PathIsNotDirectoryException,
+      UnresolvedLinkException {
     if (pathComponents.length < 2)  // add root
       return null;
     // Gets the parent INode
     INodesInPath inodes =  getExistingPathINodes(pathComponents, 2, false);
-    return INodeDirectory.valueOf(inodes.inodes[0], pathComponents);
+    INodeDirectory.valueOf(inodes.inodes[0], pathComponents);
+    return inodes;
   }
-
+  
   @Override
   DirCounts spaceConsumedInTree(DirCounts counts) {
     counts.nsCount += 1;
@@ -462,10 +502,11 @@ public class INodeDirectory extends INod
    *         Note that the returned list is never null.
    */
   public ReadOnlyList<INode> getChildrenList(final Snapshot snapshot) {
-    //TODO: use snapshot to select children list
+    assertNull(snapshot);
     return children == null ? EMPTY_READ_ONLY_LIST
         : ReadOnlyList.Util.asReadOnlyList(children);
   }
+
   /** Set the children list. */
   public void setChildren(List<INode> children) {
     this.children = children;
@@ -490,7 +531,7 @@ public class INodeDirectory extends INod
    * {@link INodeDirectory#getExistingPathINodes(byte[][], int, boolean)}.
    * Contains INodes information resolved from a given path.
    */
-  static class INodesInPath {
+  public static class INodesInPath {
     /**
      * Array with the specified number of INodes resolved for a given path.
      */
@@ -570,7 +611,7 @@ public class INodeDirectory extends INod
     }
     
     /** @return the i-th inode. */
-    INode getINode(int i) {
+    public INode getINode(int i) {
       return inodes[i];
     }
     
@@ -676,4 +717,13 @@ public class INodeDirectory extends INod
       }
     }
   }
+
+  /** 
+   * Get last modification time of inode.
+   * @return access time
+   */
+  public long getModificationTime(Snapshot snapshot) {
+    assertNull(snapshot);
+    return getModificationTime();
+  }
 }

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Mon Dec  3 18:04:51 2012
@@ -89,16 +89,22 @@ public class INodeFile extends INode imp
   INodeFile(PermissionStatus permissions, BlockInfo[] blklist,
                       short replication, long modificationTime,
                       long atime, long preferredBlockSize) {
-    super(permissions, modificationTime, atime);
+    this(null, permissions, modificationTime, atime, blklist, replication,
+        preferredBlockSize);
+  }
+
+  INodeFile(byte[] name, PermissionStatus permissions, long mtime, long atime,
+      BlockInfo[] blklist, short replication, long preferredBlockSize) {
+    super(name, permissions, null, mtime, atime);
     header = HeaderFormat.combineReplication(header, replication);
     header = HeaderFormat.combinePreferredBlockSize(header, preferredBlockSize);
     this.blocks = blklist;
   }
 
-  protected INodeFile(INodeFile f) {
-    this(f.getPermissionStatus(), f.getBlocks(), f.getFileReplication(),
-        f.getModificationTime(), f.getAccessTime(), f.getPreferredBlockSize());
-    this.setLocalName(f.getLocalNameBytes());
+  protected INodeFile(INodeFile that) {
+    super(that);
+    this.header = that.header;
+    this.blocks = that.blocks;
   }
 
   /** @return true unconditionally. */

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java Mon Dec  3 18:04:51 2012
@@ -72,9 +72,8 @@ public class INodeFileUnderConstruction 
                              String clientName,
                              String clientMachine,
                              DatanodeDescriptor clientNode) {
-    super(perm, blocks, blockReplication, modificationTime, modificationTime,
-          preferredBlockSize);
-    setLocalName(name);
+    super(name, perm, modificationTime, modificationTime,
+        blocks, blockReplication, preferredBlockSize);
     this.clientName = clientName;
     this.clientMachine = clientMachine;
     this.clientNode = clientNode;

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java Mon Dec  3 18:04:51 2012
@@ -331,22 +331,19 @@ public class LeaseManager {
     }
   }
 
-  synchronized void changeLease(String src, String dst,
-      String overwrite, String replaceBy) {
+  synchronized void changeLease(String src, String dst) {
     if (LOG.isDebugEnabled()) {
       LOG.debug(getClass().getSimpleName() + ".changelease: " +
-               " src=" + src + ", dest=" + dst + 
-               ", overwrite=" + overwrite +
-               ", replaceBy=" + replaceBy);
+               " src=" + src + ", dest=" + dst);
     }
 
-    final int len = overwrite.length();
+    final int len = src.length();
     for(Map.Entry<String, Lease> entry
         : findLeaseWithPrefixPath(src, sortedLeasesByPath).entrySet()) {
       final String oldpath = entry.getKey();
       final Lease lease = entry.getValue();
-      //overwrite must be a prefix of oldpath
-      final String newpath = replaceBy + oldpath.substring(len);
+      // replace stem of src with new destination
+      final String newpath = dst + oldpath.substring(len);
       if (LOG.isDebugEnabled()) {
         LOG.debug("changeLease: replacing " + oldpath + " with " + newpath);
       }
@@ -429,6 +426,26 @@ public class LeaseManager {
     }
   }
 
+  /**
+   * Get the list of inodes corresponding to valid leases.
+   * @return list of inodes
+   * @throws UnresolvedLinkException
+   */
+  Map<String, INodeFileUnderConstruction> getINodesUnderConstruction() {
+    Map<String, INodeFileUnderConstruction> inodes =
+        new TreeMap<String, INodeFileUnderConstruction>();
+    for (String p : sortedLeasesByPath.keySet()) {
+      // verify that path exists in namespace
+      try {
+        INode node = fsnamesystem.dir.getINode(p);
+        inodes.put(p, INodeFileUnderConstruction.valueOf(node, p));
+      } catch (IOException ioe) {
+        LOG.error(ioe);
+      }
+    }
+    return inodes;
+  }
+  
   /** Check the leases beginning from the oldest.
    *  @return true is sync is needed.
    */

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Mon Dec  3 18:04:51 2012
@@ -598,11 +598,7 @@ public class NameNode {
     String nsId = getNameServiceId(conf);
     String namenodeId = HAUtil.getNameNodeId(conf, nsId);
     this.haEnabled = HAUtil.isHAEnabled(conf, nsId);
-    if (!haEnabled) {
-      state = ACTIVE_STATE;
-    } else {
-      state = STANDBY_STATE;
-    }
+    state = createHAState();
     this.allowStaleStandbyReads = HAUtil.shouldAllowStandbyReads(conf);
     this.haContext = createHAContext();
     try {
@@ -619,6 +615,10 @@ public class NameNode {
     }
   }
 
+  protected HAState createHAState() {
+    return !haEnabled ? ACTIVE_STATE : STANDBY_STATE;
+  }
+
   protected HAContext createHAContext() {
     return new NameNodeHAContext();
   }
@@ -1298,7 +1298,7 @@ public class NameNode {
    *          before exit.
    * @throws ExitException thrown only for testing.
    */
-  private synchronized void doImmediateShutdown(Throwable t)
+  protected synchronized void doImmediateShutdown(Throwable t)
       throws ExitException {
     String message = "Error encountered requiring NN shutdown. " +
         "Shutting down immediately.";

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java Mon Dec  3 18:04:51 2012
@@ -102,7 +102,7 @@ class NamenodeJspHelper {
     long usedNonHeap = (totalNonHeap * 100) / commitedNonHeap;
 
     String str = "<div>" + inodes + " files and directories, " + blocks + " blocks = "
-        + (inodes + blocks) + " total";
+        + (inodes + blocks) + " total filesystem objects";
     if (maxobjects != 0) {
       long pct = ((inodes + blocks) * 100) / maxobjects;
       str += " / " + maxobjects + " (" + pct + "%)";

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java Mon Dec  3 18:04:51 2012
@@ -65,6 +65,19 @@ public class INodeDirectorySnapshottable
     }
     return (INodeDirectorySnapshottable)dir;
   }
+  
+  public static Snapshot findLatestSnapshot(INode inode) {
+    Snapshot latest = null;
+    for(; inode != null; inode = inode.getParent()) {
+      if (inode instanceof INodeDirectorySnapshottable) {
+        final Snapshot s = ((INodeDirectorySnapshottable)inode).getLastSnapshot();
+        if (Snapshot.ID_COMPARATOR.compare(latest, s) < 0) {
+          latest = s;
+        }
+      }
+    }
+    return latest;
+  }
 
   /** Snapshots of this directory in ascending order of snapshot id. */
   private final List<Snapshot> snapshots = new ArrayList<Snapshot>();
@@ -196,8 +209,8 @@ public class INodeDirectorySnapshottable
 
     //set modification time
     final long timestamp = Time.now();
-    s.getRoot().setModificationTime(timestamp);
-    setModificationTime(timestamp);
+    s.getRoot().updateModificationTime(timestamp);
+    updateModificationTime(timestamp);
     return s;
   }
   

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java Mon Dec  3 18:04:51 2012
@@ -19,10 +19,15 @@ package org.apache.hadoop.hdfs.server.na
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 
+import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
+import org.apache.hadoop.hdfs.util.ReadOnlyList;
+
+import com.google.common.base.Preconditions;
 
 /** The directory with snapshots. */
 public class INodeDirectoryWithSnapshot extends INodeDirectory {
@@ -250,9 +255,223 @@ public class INodeDirectoryWithSnapshot 
           + "\n  deleted=" + toString(deleted);
     }
   }
+  
+  private class SnapshotDiff implements Comparable<Snapshot> {
+    /** The snapshot will obtain after applied this diff. */
+    final Snapshot snapshot;
+    /** The size of the children list which is never changed. */
+    final int size;
+    /**
+     * Posterior diff is the diff happened after this diff.
+     * The posterior diff should be first applied to obtain the posterior
+     * snapshot and then apply this diff in order to obtain this snapshot.
+     * If the posterior diff is null, the posterior state is the current state. 
+     */
+    private SnapshotDiff posteriorDiff;
+    /** The data of this diff. */
+    private final Diff diff = new Diff();
+    /** The snapshot version of the inode. */
+    private INode snapshotINode;
+
+    SnapshotDiff(Snapshot snapshot, int size) {
+      if (size < 0) {
+        throw new HadoopIllegalArgumentException("size = " + size + " < 0");
+      }
+      this.snapshot = snapshot;
+      this.size = size;
+    }
+
+    @Override
+    public int compareTo(final Snapshot that_snapshot) {
+      return Snapshot.ID_COMPARATOR.compare(this.snapshot, that_snapshot);
+    }
+
+    /**
+     * @return The children list of a directory in a snapshot.
+     *         Since the snapshot is read-only, the logical view of the list is
+     *         never changed although the internal data structure may mutate.
+     */
+    ReadOnlyList<INode> getChildrenList() {
+      return new ReadOnlyList<INode>() {
+        private List<INode> children = null;
+
+        private List<INode> initChildren() {
+          if (children == null) {
+            final ReadOnlyList<INode> posterior = posteriorDiff != null?
+                posteriorDiff.getChildrenList()
+                : INodeDirectoryWithSnapshot.this.getChildrenList(null);
+            children = diff.apply2Current(ReadOnlyList.Util.asList(posterior));
+          }
+          return children;
+        }
+
+        @Override
+        public Iterator<INode> iterator() {
+          return initChildren().iterator();
+        }
+    
+        @Override
+        public boolean isEmpty() {
+          return size == 0;
+        }
+    
+        @Override
+        public int size() {
+          return size;
+        }
+    
+        @Override
+        public INode get(int i) {
+          return initChildren().get(i);
+        }
+      };
+    }
+    
+    INode getChild(byte[] name) {
+      final INode i = diff.accessPrevious(name, INode.DUMMY);
+      if (i != INode.DUMMY) {
+        // this diff is able to find it
+        return i; 
+      } else {
+        // should return the posterior INode.
+        return posteriorDiff != null? posteriorDiff.getChild(name)
+            : INodeDirectoryWithSnapshot.this.getChild(name, null);
+      }
+    }
+  }
+  
+  /** Replace the given directory to an {@link INodeDirectoryWithSnapshot}. */
+  public static INodeDirectoryWithSnapshot replaceDir(INodeDirectory oldDir,
+      Snapshot latestSnapshot) {
+    Preconditions.checkArgument(!(oldDir instanceof INodeDirectoryWithSnapshot),
+        "oldDir is already an INodeDirectoryWithSnapshot, oldDir=%s", oldDir);
+
+    final INodeDirectory parent = oldDir.getParent();
+    Preconditions.checkArgument(parent != null,
+        "parent is null, oldDir=%s", oldDir);
+
+    final INodeDirectoryWithSnapshot newDir = new INodeDirectoryWithSnapshot(
+        oldDir, latestSnapshot);
+    parent.replaceChild(newDir, null);
+    return newDir;
+  }
+  
+  /** Diff list sorted by snapshot IDs, i.e. in chronological order. */
+  private final List<SnapshotDiff> diffs = new ArrayList<SnapshotDiff>();
 
-  INodeDirectoryWithSnapshot(String name, INodeDirectory dir) {
-    super(name, dir.getPermissionStatus());
-    parent = dir;
+  INodeDirectoryWithSnapshot(INodeDirectory that, Snapshot s) {
+    super(that);
+
+    // add a diff for the snapshot
+    addSnapshotDiff(s, that.getChildrenList(null).size());
+  }
+
+  INodeDirectoryWithSnapshot(String name, INodeDirectory dir, Snapshot s) {
+    this(dir, s);
+    setLocalName(name);
+    setParent(dir);
+  }
+  
+  SnapshotDiff addSnapshotDiff(Snapshot snapshot, int childrenSize) {
+    final SnapshotDiff d = new SnapshotDiff(snapshot, childrenSize); 
+    diffs.add(d);
+    return d;
+  }
+
+  /**
+   * Check if the latest snapshot diff exist.  If not, add it.
+   * @return the latest snapshot diff, which is never null.
+   */
+  private SnapshotDiff checkAndAddLatestSnapshotDiff(Snapshot latest) {
+    final SnapshotDiff last = getLastSnapshotDiff();
+    if (last != null && last.snapshot.equals(latest)) {
+      return last;
+    }
+
+    final int size = getChildrenList(null).size();
+    final SnapshotDiff d = addSnapshotDiff(latest, size);
+    if (last != null) {
+      last.posteriorDiff = d;
+    }
+    return d;
+  }
+  
+  Diff getLatestDiff(Snapshot latest) {
+    return checkAndAddLatestSnapshotDiff(latest).diff;
+  }
+
+  /**
+   * @return the diff corresponding to the snapshot.
+   *         When the diff is not found, it means that the current state and
+   *         the snapshot state are the same. 
+   */
+  SnapshotDiff getSnapshotDiff(Snapshot snapshot) {
+    if (snapshot == null) {
+      return null;
+    }
+    final int i = Collections.binarySearch(diffs, snapshot);
+    if (i >= 0) {
+      // exact match
+      return diffs.get(i);
+    } else {
+      // Exact match not found means that there were no changes between
+      // given snapshot and the next state so that the diff for the given
+      // snapshot is not recorded.  Thus, use the next state.
+      final int j = -i - 1;
+      return j < diffs.size()? diffs.get(j): null;
+    }
+  }
+  
+  SnapshotDiff getLastSnapshotDiff() {
+    return diffs.get(diffs.size() - 1);
+  }
+
+  @Override
+  public ReadOnlyList<INode> getChildrenList(Snapshot snapshot) {
+    final SnapshotDiff diff = getSnapshotDiff(snapshot);
+    if (diff != null) {
+      return diff.getChildrenList();
+    }
+    return super.getChildrenList(null);
+  }
+
+  @Override
+  public INode getChild(byte[] name, Snapshot snapshot) {
+    final SnapshotDiff diff = getSnapshotDiff(snapshot);
+    if (diff != null) {
+      return diff.getChild(name);
+    }
+    return super.getChild(name, null);
+  }
+  
+  @Override
+  public boolean addChild(INode inode, boolean setModTime,
+      Snapshot latestSnapshot) {
+    getLatestDiff(latestSnapshot).create(inode);
+    return super.addChild(inode, setModTime, null);
+  }
+
+  @Override
+  public INode removeChild(INode inode, Snapshot latestSnapshot) {
+    getLatestDiff(latestSnapshot).delete(inode);
+    return super.removeChild(inode, null);
+  }
+
+  @Override
+  public INode replaceChild(INodeDirectory newChild, Snapshot latestSnapshot) {
+    final INode oldChild = super.replaceChild(newChild, null);
+    final Diff diff = getLatestDiff(latestSnapshot);
+    diff.delete(oldChild);
+    diff.create(newChild);
+    return oldChild;
+  }
+
+  @Override
+  public long getModificationTime(Snapshot snapshot) {
+    final SnapshotDiff diff = getSnapshotDiff(snapshot);
+    if (diff != null) {
+      return diff.snapshotINode.getModificationTime();
+    }
+    return getModificationTime();
   }
 }

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java Mon Dec  3 18:04:51 2012
@@ -109,7 +109,7 @@ public class INodeFileWithLink extends I
       this.setFileReplication(maxReplication);
       this.next = null;
       // clear parent
-      parent = null;
+      setParent(null);
     }
     return 1;
   }

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java Mon Dec  3 18:04:51 2012
@@ -44,7 +44,7 @@ public class Snapshot implements Compara
 
   Snapshot(int id, String name, INodeDirectorySnapshottable dir) {
     this.id = id;
-    this.root = new INodeDirectoryWithSnapshot(name, dir);
+    this.root = new INodeDirectoryWithSnapshot(name, dir, this);
   }
 
   /** @return the root directory of the snapshot. */
@@ -58,6 +58,21 @@ public class Snapshot implements Compara
   }
   
   @Override
+  public boolean equals(Object that) {
+    if (this == that) {
+      return true;
+    } else if (that == null || !(that instanceof Snapshot)) {
+      return false;
+    }
+    return this.id == ((Snapshot)that).id;
+  }
+  
+  @Override
+  public int hashCode() {
+    return id;
+  }
+  
+  @Override
   public String toString() {
     return getClass().getSimpleName() + ":" + root.getLocalName();
   }

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java Mon Dec  3 18:04:51 2012
@@ -24,12 +24,8 @@ import java.util.concurrent.atomic.Atomi
 
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
-import org.apache.hadoop.hdfs.server.namenode.INodeFile;
-import org.apache.hadoop.hdfs.server.namenode.INodeFileUnderConstruction;
-import org.apache.hadoop.hdfs.server.namenode.INodeSymlink;
-import org.apache.hadoop.hdfs.util.ReadOnlyList;
+import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
 
 /**
  * Manage snapshottable directories and their snapshots.
@@ -68,7 +64,9 @@ public class SnapshotManager implements 
    */
   public void setSnapshottable(final String path, final int snapshotQuota
       ) throws IOException {
-    final INodeDirectory d = INodeDirectory.valueOf(fsdir.getINode(path), path);
+    final INodesInPath inodesInPath = fsdir.getINodesInPath(path);
+    final INodeDirectory d = INodeDirectory.valueOf(
+        inodesInPath.getINode(0), path);
     if (d.isSnapshottable()) {
       //The directory is already a snapshottable directory.
       ((INodeDirectorySnapshottable)d).setSnapshotQuota(snapshotQuota);
@@ -77,7 +75,7 @@ public class SnapshotManager implements 
 
     final INodeDirectorySnapshottable s
         = INodeDirectorySnapshottable.newInstance(d, snapshotQuota);
-    fsdir.replaceINodeDirectory(path, d, s);
+    fsdir.replaceINodeDirectory(path, d, s, inodesInPath.getLatestSnapshot());
     snapshottables.add(s);
 
     numSnapshottableDirs.getAndIncrement();
@@ -90,15 +88,16 @@ public class SnapshotManager implements 
    */
   public void resetSnapshottable(final String path
       ) throws IOException {
+    final INodesInPath inodesInPath = fsdir.getINodesInPath(path);
     final INodeDirectorySnapshottable s = INodeDirectorySnapshottable.valueOf(
-        fsdir.getINode(path), path);
+        inodesInPath.getINode(0), path);
     if (s.getNumSnapshots() > 0) {
       throw new SnapshotException("The directory " + path + " has snapshot(s). "
           + "Please redo the operation after removing all the snapshots.");
     }
 
     final INodeDirectory d = new INodeDirectory(s);
-    fsdir.replaceINodeDirectory(path, s, d);
+    fsdir.replaceINodeDirectory(path, s, d, inodesInPath.getLatestSnapshot());
     snapshottables.remove(s);
 
     numSnapshottableDirs.getAndDecrement();
@@ -121,9 +120,8 @@ public class SnapshotManager implements 
     // Find the source root directory path where the snapshot is taken.
     final INodeDirectorySnapshottable srcRoot
         = INodeDirectorySnapshottable.valueOf(fsdir.getINode(path), path);
-    final Snapshot s = srcRoot.addSnapshot(snapshotID, snapshotName);
-    new SnapshotCreation().processRecursively(srcRoot, s.getRoot());
-      
+    srcRoot.addSnapshot(snapshotID, snapshotName);
+
     //create success, update id
     snapshotID++;
     numSnapshots.getAndIncrement();
@@ -154,85 +152,6 @@ public class SnapshotManager implements 
     srcRoot.renameSnapshot(path, oldSnapshotName, newSnapshotName);
   }
   
-  /**
-   * Create a snapshot of subtrees by recursively coping the directory
-   * structure from the source directory to the snapshot destination directory.
-   * This creation algorithm requires O(N) running time and O(N) memory,
-   * where N = # files + # directories + # symlinks. 
-   */
-  class SnapshotCreation {
-    /** Process snapshot creation recursively. */
-    private void processRecursively(final INodeDirectory srcDir,
-        final INodeDirectory dstDir) throws IOException {
-      final ReadOnlyList<INode> children = srcDir.getChildrenList(null);
-      if (!children.isEmpty()) {
-        final List<INode> inodes = new ArrayList<INode>(children.size());
-        for(final INode c : new ArrayList<INode>(ReadOnlyList.Util.asList(children))) {
-          final INode i;
-          if (c == null) {
-            i = null;
-          } else if (c instanceof INodeDirectory) {
-            //also handle INodeDirectoryWithQuota
-            i = processINodeDirectory((INodeDirectory)c);
-          } else if (c instanceof INodeFileUnderConstruction) {
-            //TODO: support INodeFileUnderConstruction
-            throw new IOException("Not yet supported.");
-          } else if (c instanceof INodeFile) {
-            i = processINodeFile(srcDir, (INodeFile)c);
-          } else if (c instanceof INodeSymlink) {
-            i = new INodeSymlink((INodeSymlink)c);
-          } else {
-            throw new AssertionError("Unknow INode type: " + c.getClass()
-                + ", inode = " + c);
-          }
-          i.setParent(dstDir);
-          inodes.add(i);
-        }
-        dstDir.setChildren(inodes);
-      }
-    }
-    
-    /**
-     * Create destination INodeDirectory and make the recursive call. 
-     * @return destination INodeDirectory.
-     */
-    private INodeDirectory processINodeDirectory(final INodeDirectory srcChild
-        ) throws IOException {
-      final INodeDirectory dstChild = new INodeDirectory(srcChild);
-      dstChild.setChildren(null);
-      processRecursively(srcChild, dstChild);
-      return dstChild;
-    }
-
-    /**
-     * Create destination INodeFileSnapshot and update source INode type.
-     * @return destination INodeFileSnapshot.
-     */
-    private INodeFileSnapshot processINodeFile(final INodeDirectory parent,
-        final INodeFile file) {
-      final INodeFileSnapshot snapshot = new INodeFileSnapshot(
-          file, file.computeFileSize(true)); 
-
-      final INodeFileWithLink srcWithLink;
-      //check source INode type
-      if (file instanceof INodeFileWithLink) {
-        srcWithLink = (INodeFileWithLink)file;
-      } else {
-        //source is an INodeFile, replace the source.
-        srcWithLink = new INodeFileWithLink(file);
-        file.removeNode();
-        parent.addChild(srcWithLink, false);
-
-        //update block map
-        namesystem.getBlockManager().addBlockCollection(srcWithLink);
-      }
-      
-      //insert the snapshot to src's linked list.
-      srcWithLink.insert(snapshot);
-      return snapshot;
-    }
-  }
-
   @Override
   public long getNumSnapshottableDirs() {
     return numSnapshottableDirs.get();
@@ -243,4 +162,4 @@ public class SnapshotManager implements 
     return numSnapshots.get();
   }
   
-}
\ No newline at end of file
+}

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1415804-1416602

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1415804-1416602

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1415804-1416602

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1415804-1416602

Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1415804-1416602



Mime
View raw message