hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1437256 [2/2] - in /hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/main/java/org/apache/hadoop/hdfs/server/namenode/snaps...
Date Wed, 23 Jan 2013 02:48:02 GMT
Added: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java?rev=1437256&view=auto
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
(added)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
Wed Jan 23 02:48:01 2013
@@ -0,0 +1,351 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode.snapshot;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
+import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
+import org.apache.hadoop.hdfs.server.namenode.FSImageFormat.Loader;
+import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
+import org.apache.hadoop.hdfs.server.namenode.INode;
+import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot.Diff;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot.SnapshotDiff;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.Root;
+import org.apache.hadoop.hdfs.util.ReadOnlyList;
+
+/**
+ * A helper class defining static methods for reading/writing snapshot related
+ * information from/to FSImage.
+ */
+public class SnapshotFSImageFormat {
+
+  /**
+   * Save snapshots and snapshot quota for a snapshottable directory.
+   * @param current The directory that the snapshots belongs to.
+   * @param out The {@link DataOutputStream} to write.
+   * @throws IOException
+   */
+  public static void saveSnapshots(INodeDirectorySnapshottable current,
+      DataOutputStream out) throws IOException {
+    // list of snapshots in snapshotsByNames
+    ReadOnlyList<Snapshot> snapshots = current.getSnapshotsByNames();
+    out.writeInt(snapshots.size());
+    for (Snapshot ss : snapshots) {
+      // write the snapshot
+      ss.write(out);
+    }
+    // snapshot quota
+    out.writeInt(current.getSnapshotQuota());
+  }
+  
+  /**
+   * Save SnapshotDiff list for an INodeDirectoryWithSnapshot.
+   * @param sNode The directory that the SnapshotDiff list belongs to.
+   * @param out The {@link DataOutputStream} to write.
+   */
+  public static void saveSnapshotDiffs(INodeDirectoryWithSnapshot sNode,
+      DataOutputStream out) throws IOException {
+    // # of SnapshotDiff
+    List<SnapshotDiff> diffs = sNode.getSnapshotDiffs();
+    // Record the SnapshotDiff in reversed order, so that we can find the
+    // correct reference for INodes in the created list when loading the
+    // FSImage
+    out.writeInt(diffs.size());
+    for (int i = diffs.size() - 1; i >= 0; i--) {
+      SnapshotDiff sdiff = diffs.get(i);
+      sdiff.write(out);
+    }
+  }
+  
+  /**
+   * Load a node stored in the created list from fsimage.
+   * @param createdNodeName The name of the created node.
+   * @param parent The directory that the created list belongs to.
+   * @return The created node.
+   */
+  private static INode loadCreated(byte[] createdNodeName,
+      INodeDirectoryWithSnapshot parent) throws IOException {
+    // the INode in the created list should be a reference to another INode
+    // in posterior SnapshotDiffs or one of the current children
+    for (SnapshotDiff postDiff : parent.getSnapshotDiffs()) {
+      INode created = findCreated(createdNodeName, postDiff.getDiff());
+      if (created != null) {
+        return created;
+      } // else go to the next SnapshotDiff
+    } 
+    // use the current child
+    INode currentChild = parent.getChild(createdNodeName, null);
+    if (currentChild == null) {
+      throw new IOException("Cannot find an INode associated with the INode "
+          + DFSUtil.bytes2String(createdNodeName)
+          + " in created list while loading FSImage.");
+    }
+    return currentChild;
+  }
+  
+  /**
+   * Search the given {@link Diff} to find an inode matching the specific name.
+   * @param createdNodeName The name of the node for searching.
+   * @param diff The given {@link Diff} where to search the node.
+   * @return The matched inode. Return null if no matched inode can be found.
+   */
+  private static INode findCreated(byte[] createdNodeName, Diff diff) {
+    INode c = diff.searchCreated(createdNodeName);
+    INode d = diff.searchDeleted(createdNodeName);
+    if (c == null && d != null) {
+      // if an INode with the same name is only contained in the deleted
+      // list, then the node should be the snapshot copy of a deleted
+      // node, and the node in the created list should be its reference 
+      return d;
+    } else if (c != null && d != null) {
+      // in a posterior SnapshotDiff, if the created/deleted lists both
+      // contains nodes with the same name (c & d), there are two
+      // possibilities:
+      // 
+      // 1) c and d are used to represent a modification, and 
+      // 2) d indicates the deletion of the node, while c was originally
+      // contained in the created list of a later snapshot, but c was
+      // moved here because of the snapshot deletion.
+      // 
+      // For case 1), c and d should be both INodeFile and should share
+      // the same blockInfo list.
+      if (c.isFile()
+          && ((INodeFile) c).getBlocks() == ((INodeFile) d).getBlocks()) {
+        return c;
+      } else {
+        return d;
+      }
+    }
+    return null;
+  }
+  
+  /**
+   * Load the created list from fsimage.
+   * @param parent The directory that the created list belongs to.
+   * @param in The {@link DataInputStream} to read.
+   * @return The created list.
+   */
+  private static List<INode> loadCreatedList(INodeDirectoryWithSnapshot parent,
+      DataInputStream in) throws IOException {
+    // read the size of the created list
+    int createdSize = in.readInt();
+    List<INode> createdList = new ArrayList<INode>(createdSize);
+    for (int i = 0; i < createdSize; i++) {
+      byte[] createdNodeName = new byte[in.readShort()];
+      in.readFully(createdNodeName);
+      INode created = loadCreated(createdNodeName, parent);
+      createdList.add(created);
+    }
+    return createdList;
+  }
+    
+  /**
+   * Load the deleted list from the fsimage.
+   * 
+   * @param parent The directory that the deleted list belongs to.
+   * @param createdList The created list associated with the deleted list in 
+   *                    the same Diff.
+   * @param in The {@link DataInputStream} to read.
+   * @param loader The {@link Loader} instance. Used to call the
+   *               {@link Loader#loadINode(DataInputStream)} method.
+   * @return The deleted list.
+   */
+  private static List<INode> loadDeletedList(INodeDirectoryWithSnapshot parent,
+      List<INode> createdList, DataInputStream in, FSImageFormat.Loader loader)
+      throws IOException {
+    int deletedSize = in.readInt();
+    List<INode> deletedList = new ArrayList<INode>(deletedSize);
+    for (int i = 0; i < deletedSize; i++) {
+      byte[] deletedNodeName = new byte[in.readShort()];
+      in.readFully(deletedNodeName);
+      INode deleted = loader.loadINode(in);
+      deleted.setLocalName(deletedNodeName);
+      deletedList.add(deleted);
+      // set parent: the parent field of an INode in the deleted list is not 
+      // useful, but set the parent here to be consistent with the original 
+      // fsdir tree.
+      deleted.setParent(parent);
+      if (deleted instanceof INodeFile
+          && ((INodeFile) deleted).getBlocks() == null) {
+        // if deleted is an INodeFile, and its blocks is null, then deleted
+        // must be an INodeFileWithLink, and we need to rebuild its next link
+        int c = Collections.binarySearch(createdList, deletedNodeName);
+        if (c < 0) {
+          throw new IOException(
+              "Cannot find the INode linked with the INode "
+                  + DFSUtil.bytes2String(deletedNodeName)
+                  + " in deleted list while loading FSImage.");
+        }
+        // deleted must be an INodeFileSnapshot
+        INodeFileSnapshot deletedWithLink = (INodeFileSnapshot) deleted;
+        INodeFile cNode = (INodeFile) createdList.get(c);
+        INodeFileWithSnapshot cNodeWithLink = (INodeFileWithSnapshot) cNode;
+        deletedWithLink.setBlocks(cNode.getBlocks());
+        // insert deleted into the circular list
+        cNodeWithLink.insertBefore(deletedWithLink);
+      }
+    }
+    return deletedList;
+  }
+  
+  /**
+   * Load snapshots and snapshotQuota for a Snapshottable directory.
+   * @param snapshottableParent The snapshottable directory for loading.
+   * @param numSnapshots The number of snapshots that the directory has.
+   * @param in The {@link DataInputStream} instance to read.
+   * @param loader The {@link Loader} instance that this loading procedure is 
+   *               using.
+   */
+  public static void loadSnapshotList(
+      INodeDirectorySnapshottable snapshottableParent, int numSnapshots,
+      DataInputStream in, FSImageFormat.Loader loader) throws IOException {
+    for (int i = 0; i < numSnapshots; i++) {
+      // read snapshots
+      Snapshot ss = loadSnapshot(snapshottableParent, in, loader);
+      snapshottableParent.addSnapshot(ss);
+    }
+    int snapshotQuota = in.readInt();
+    snapshottableParent.setSnapshotQuota(snapshotQuota);
+  }
+  
+  /**
+   * Load a {@link Snapshot} from fsimage.
+   * @param parent The directory that the snapshot belongs to.
+   * @param in The {@link DataInputStream} instance to read.
+   * @param loader The {@link Loader} instance that this loading procedure is 
+   *               using.
+   * @return The snapshot.
+   */
+  private static Snapshot loadSnapshot(INodeDirectorySnapshottable parent,
+      DataInputStream in, FSImageFormat.Loader loader) throws IOException {
+    int snapshotId = in.readInt();
+    byte[] snapshotName = new byte[in.readShort()];
+    in.readFully(snapshotName);
+    INode rootNode = loader.loadINode(in);
+    rootNode.setLocalName(snapshotName);
+    rootNode.setParent(parent);
+    return new Snapshot(snapshotId, (INodeDirectory) rootNode);
+  }
+  
+  /**
+   * Load the {@link SnapshotDiff} list for the INodeDirectoryWithSnapshot
+   * directory.
+   * @param snapshottableParent The snapshottable directory for loading.
+   * @param numSnapshotDiffs The number of {@link SnapshotDiff} that the 
+   *                         directory has.
+   * @param in The {@link DataInputStream} instance to read.
+   * @param loader The {@link Loader} instance that this loading procedure is 
+   *               using.
+   */
+  public static void loadSnapshotDiffList(
+      INodeDirectoryWithSnapshot parentWithSnapshot, int numSnapshotDiffs,
+      DataInputStream in, FSImageFormat.Loader loader)
+      throws IOException {
+    for (int i = 0; i < numSnapshotDiffs; i++) {
+      SnapshotDiff diff = loadSnapshotDiff(parentWithSnapshot, in, loader);
+      parentWithSnapshot.insertDiff(diff);
+    }
+  }
+  
+  /**
+   * Use the given full path to a {@link Root} directory to find the
+   * associated snapshot.
+   */
+  private static Snapshot findSnapshot(String sRootFullPath, FSDirectory fsdir)
+      throws IOException {
+    // find the root
+    INode root = fsdir.getINode(sRootFullPath);
+    INodeDirectorySnapshottable snapshotRoot = INodeDirectorySnapshottable
+        .valueOf(root.getParent(), root.getParent().getFullPathName());
+    // find the snapshot
+    return snapshotRoot.getSnapshot(root.getLocalNameBytes());
+  }
+  
+  /**
+   * Load the snapshotINode field of {@link SnapshotDiff}.
+   * @param snapshot The Snapshot associated with the {@link SnapshotDiff}.
+   * @param in The {@link DataInputStream} to read.
+   * @param loader The {@link Loader} instance that this loading procedure is 
+   *               using.
+   * @return The snapshotINode.
+   */
+  private static INodeDirectory loadSnapshotINodeInSnapshotDiff(
+      Snapshot snapshot, DataInputStream in, FSImageFormat.Loader loader)
+      throws IOException {
+    // read the boolean indicating whether snapshotINode == Snapshot.Root
+    boolean useRoot = in.readBoolean();      
+    if (useRoot) {
+      return snapshot.getRoot();
+    } else {
+      // another boolean is used to indicate whether snapshotINode is non-null
+      if (in.readBoolean()) {
+        byte[] localName = new byte[in.readShort()];
+        in.readFully(localName);
+        INodeDirectory snapshotINode = (INodeDirectory) loader.loadINode(in);
+        snapshotINode.setLocalName(localName);
+        return snapshotINode;
+      }
+    }
+    return null;
+  }
+   
+  /**
+   * Load {@link SnapshotDiff} from fsimage.
+   * @param parent The directory that the SnapshotDiff belongs to.
+   * @param in The {@link DataInputStream} instance to read.
+   * @param loader The {@link Loader} instance that this loading procedure is 
+   *               using.
+   * @return A {@link SnapshotDiff}.
+   */
+  private static SnapshotDiff loadSnapshotDiff(
+      INodeDirectoryWithSnapshot parent, DataInputStream in,
+      FSImageFormat.Loader loader) throws IOException {
+    // 1. Load SnapshotDiff#childrenSize
+    int childrenSize = in.readInt();
+    // 2. Read the full path of the Snapshot's Root, identify 
+    //    SnapshotDiff#Snapshot
+    Snapshot snapshot = findSnapshot(FSImageSerialization.readString(in),
+        loader.getFSDirectoryInLoading());
+    
+    // 3. Load SnapshotDiff#snapshotINode 
+    INodeDirectory snapshotINode = loadSnapshotINodeInSnapshotDiff(snapshot,
+        in, loader);
+    
+    // 4. Load the created list in SnapshotDiff#Diff
+    List<INode> createdList = loadCreatedList(parent, in);
+    
+    // 5. Load the deleted list in SnapshotDiff#Diff
+    List<INode> deletedList = loadDeletedList(parent, createdList, in, loader);
+    
+    // 6. Compose the SnapshotDiff
+    SnapshotDiff sdiff = parent.new SnapshotDiff(snapshot, childrenSize,
+        snapshotINode, parent.getSnapshotDiffs().isEmpty() ? null : parent
+            .getSnapshotDiffs().get(0), createdList, deletedList);
+    return sdiff;
+  }
+  
+}

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java?rev=1437256&r1=1437255&r2=1437256&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
Wed Jan 23 02:48:01 2013
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.snapshot;
 
+import java.io.DataInput;
+import java.io.DataOutput;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -48,7 +50,7 @@ public class SnapshotManager implements 
   private final AtomicInteger numSnapshottableDirs = new AtomicInteger();
   private final AtomicInteger numSnapshots = new AtomicInteger();
 
-  private int snapshotID = 0;
+  private int snapshotCounter = 0;
   
   /** All snapshottable directories in the namesystem. */
   private final List<INodeDirectorySnapshottable> snapshottables
@@ -117,10 +119,10 @@ public class SnapshotManager implements 
     final INodesInPath i = fsdir.getMutableINodesInPath(path);
     final INodeDirectorySnapshottable srcRoot
         = INodeDirectorySnapshottable.valueOf(i.getLastINode(), path);
-    srcRoot.addSnapshot(snapshotID, snapshotName);
+    srcRoot.addSnapshot(snapshotCounter, snapshotName);
       
     //create success, update id
-    snapshotID++;
+    snapshotCounter++;
     numSnapshots.getAndIncrement();
   }
   
@@ -181,6 +183,26 @@ public class SnapshotManager implements 
   }
   
   /**
+   * Write {@link #snapshotCounter}, {@link #numSnapshots}, and
+   * {@link #numSnapshottableDirs} to the DataOutput.
+   */
+  public void write(DataOutput out) throws IOException {
+    out.writeInt(snapshotCounter);
+    out.writeInt(numSnapshots.get());
+    out.writeInt(numSnapshottableDirs.get());
+  }
+  
+  /**
+   * Read values of {@link #snapshotCounter}, {@link #numSnapshots}, and
+   * {@link #numSnapshottableDirs} from the DataInput
+   */
+  public void read(DataInput in) throws IOException {
+    snapshotCounter = in.readInt();
+    numSnapshots.set(in.readInt());
+    numSnapshottableDirs.set(in.readInt());
+  }
+  
+  /**
    * @return All the current snapshottable directories
    */
   public SnapshottableDirectoryStatus[] getSnapshottableDirListing() {

Added: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java?rev=1437256&view=auto
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java
(added)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java
Wed Jan 23 02:48:01 2013
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
+import org.apache.hadoop.hdfs.util.Canceler;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test FSImage save/load when Snapshot is supported
+ */
+public class TestFSImageWithSnapshot {
+  static final long seed = 0;
+  static final short REPLICATION = 3;
+  static final long BLOCKSIZE = 1024;
+  static final long txid = 1;
+
+  private final Path dir = new Path("/TestSnapshot");
+  private static String testDir =
+      System.getProperty("test.build.data", "build/test/data");
+  
+  Configuration conf;
+  MiniDFSCluster cluster;
+  FSNamesystem fsn;
+  DistributedFileSystem hdfs;
+  
+  @Before
+  public void setUp() throws Exception {
+    conf = new Configuration();
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION)
+        .build();
+    cluster.waitActive();
+    fsn = cluster.getNamesystem();
+    hdfs = cluster.getFileSystem();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+  }
+  
+  /**
+   * Testing steps:
+   * <pre>
+   * 1. Creating/modifying directories/files while snapshots are being taken.
+   * 2. Dump the FSDirectory tree of the namesystem.
+   * 3. Save the namesystem to a temp file (FSImage saving).
+   * 4. Restart the cluster and format the namesystem.
+   * 5. Load the namesystem from the temp file (FSImage loading).
+   * 6. Dump the FSDirectory again and compare the two dumped string.
+   * </pre>
+   */
+  @Test
+  public void testSaveLoadImage() throws Exception {
+    // make changes to the namesystem
+    hdfs.mkdirs(dir);
+    hdfs.allowSnapshot(dir.toString());
+    hdfs.createSnapshot(dir, "s0");
+    
+    Path sub1 = new Path(dir, "sub1");
+    Path sub1file1 = new Path(sub1, "sub1file1");
+    Path sub1file2 = new Path(sub1, "sub1file2");
+    DFSTestUtil.createFile(hdfs, sub1file1, BLOCKSIZE, REPLICATION, seed);
+    DFSTestUtil.createFile(hdfs, sub1file2, BLOCKSIZE, REPLICATION, seed);
+    
+    hdfs.createSnapshot(dir, "s1");
+    
+    Path sub2 = new Path(dir, "sub2");
+    Path sub2file1 = new Path(sub2, "sub2file1");
+    Path sub2file2 = new Path(sub2, "sub2file2");
+    DFSTestUtil.createFile(hdfs, sub2file1, BLOCKSIZE, REPLICATION, seed);
+    DFSTestUtil.createFile(hdfs, sub2file2, BLOCKSIZE, REPLICATION, seed);
+    hdfs.setReplication(sub1file1, (short) (REPLICATION - 1));
+    hdfs.delete(sub1file2, true);
+    
+    hdfs.createSnapshot(dir, "s2");
+    hdfs.setOwner(sub2, "dr.who", "unknown");
+    hdfs.delete(sub2file2, true);
+    
+    // dump the fsdir tree
+    StringBuffer fsnStrBefore = fsn.getFSDirectory().rootDir
+        .dumpTreeRecursively();
+    
+    // save the namesystem to a temp file
+    SaveNamespaceContext context = new SaveNamespaceContext(fsn, txid,
+        new Canceler());
+    FSImageFormat.Saver saver = new FSImageFormat.Saver(context);
+    FSImageCompression compression = FSImageCompression.createCompression(conf);
+    File dstFile = getStorageFile(testDir, txid);
+    fsn.readLock();
+    try {
+      saver.save(dstFile, compression);
+    } finally {
+      fsn.readUnlock();
+    }
+
+    // restart the cluster, and format the cluster
+    cluster.shutdown();
+    cluster = new MiniDFSCluster.Builder(conf).format(true)
+        .numDataNodes(REPLICATION).build();
+    cluster.waitActive();
+    fsn = cluster.getNamesystem();
+    hdfs = cluster.getFileSystem();
+    
+    // load the namesystem from the temp file
+    FSImageFormat.Loader loader = new FSImageFormat.Loader(conf, fsn);
+    fsn.writeLock();
+    try {
+      loader.load(dstFile);
+    } finally {
+      fsn.writeUnlock();
+    }
+    
+    // dump the fsdir tree again
+    StringBuffer fsnStrAfter = fsn.getFSDirectory().rootDir
+        .dumpTreeRecursively();
+    
+    // compare two dumped tree
+    System.out.println(fsnStrBefore.toString());
+    System.out.println("\n" + fsnStrAfter.toString());
+    assertEquals(fsnStrBefore.toString(), fsnStrAfter.toString());
+  }
+  
+  /**
+   * Create a temp fsimage file for testing.
+   * @param dir The directory where the fsimage file resides
+   * @param imageTxId The transaction id of the fsimage
+   * @return The file of the image file
+   */
+  private File getStorageFile(String dir, long imageTxId) {
+    return new File(dir, String.format("%s_%019d", NameNodeFile.IMAGE,
+        imageTxId));
+  }
+}
\ No newline at end of file

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java?rev=1437256&r1=1437255&r2=1437256&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
Wed Jan 23 02:48:01 2013
@@ -127,7 +127,6 @@ public class SnapshotTestHelper {
   static class TestDirectoryTree {
     /** Height of the directory tree */
     final int height;
-    final FileSystem fs;
     /** Top node of the directory tree */
     final Node topNode;
     /** A map recording nodes for each tree level */
@@ -138,12 +137,11 @@ public class SnapshotTestHelper {
      */
     TestDirectoryTree(int height, FileSystem fs) throws Exception {
       this.height = height;
-      this.fs = fs;
       this.topNode = new Node(new Path("/TestSnapshot"), 0,
           null, fs);
       this.levelMap = new HashMap<Integer, ArrayList<Node>>();
       addDirNode(topNode, 0);
-      genChildren(topNode, height - 1);
+      genChildren(topNode, height - 1, fs);
     }
 
     /**
@@ -164,9 +162,11 @@ public class SnapshotTestHelper {
      * 
      * @param parent The parent node
      * @param level The remaining levels to generate
+     * @param fs The FileSystem where to generate the files/dirs
      * @throws Exception
      */
-    void genChildren(Node parent, int level) throws Exception {
+    private void genChildren(Node parent, int level, FileSystem fs)
+        throws Exception {
       if (level == 0) {
         return;
       }
@@ -176,8 +176,8 @@ public class SnapshotTestHelper {
           "right" + ++id), height - level, parent, fs);
       addDirNode(parent.leftChild, parent.leftChild.level);
       addDirNode(parent.rightChild, parent.rightChild.level);
-      genChildren(parent.leftChild, level - 1);
-      genChildren(parent.rightChild, level - 1);
+      genChildren(parent.leftChild, level - 1, fs);
+      genChildren(parent.rightChild, level - 1, fs);
     }
 
     /**
@@ -246,7 +246,6 @@ public class SnapshotTestHelper {
        * directory creation/deletion
        */
       final ArrayList<Node> nonSnapshotChildren;
-      final FileSystem fs;
 
       Node(Path path, int level, Node parent,
           FileSystem fs) throws Exception {
@@ -254,7 +253,6 @@ public class SnapshotTestHelper {
         this.level = level;
         this.parent = parent;
         this.nonSnapshotChildren = new ArrayList<Node>();
-        this.fs = fs;
         fs.mkdirs(nodePath);
       }
 
@@ -262,8 +260,8 @@ public class SnapshotTestHelper {
        * Create files and add them in the fileList. Initially the last element
        * in the fileList is set to null (where we start file creation).
        */
-      void initFileList(String namePrefix, long fileLen, short replication,
-          long seed, int numFiles) throws Exception {
+      void initFileList(FileSystem fs, String namePrefix, long fileLen,
+          short replication, long seed, int numFiles) throws Exception {
         fileList = new ArrayList<Path>(numFiles);
         for (int i = 0; i < numFiles; i++) {
           Path file = new Path(nodePath, namePrefix + "-f" + i);

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java?rev=1437256&r1=1437255&r2=1437256&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java
Wed Jan 23 02:48:01 2013
@@ -148,7 +148,7 @@ public class TestINodeDirectoryWithSnaps
     // combine all diffs
     final Diff combined = diffs[0];
     for(int i = 1; i < diffs.length; i++) {
-      combined.combinePostDiff(diffs[i], null);
+      combined.combinePostDiff(diffs[i], null, false);
     }
 
     {
@@ -284,7 +284,7 @@ public class TestINodeDirectoryWithSnaps
         before = toString(diff);
       }
 
-      final Triple<Integer, INode, Integer> undoInfo = diff.delete(inode);
+      final Triple<Integer, INode, Integer> undoInfo = diff.delete(inode, true);
 
       if (testUndo) {
         final String after = toString(diff);
@@ -292,7 +292,7 @@ public class TestINodeDirectoryWithSnaps
         diff.undoDelete(inode, undoInfo);
         assertDiff(before, diff);
         //re-do
-        diff.delete(inode);
+        diff.delete(inode, true);
         assertDiff(after, diff);
       }
     }
@@ -314,7 +314,7 @@ public class TestINodeDirectoryWithSnaps
         before = toString(diff);
       }
 
-      final Triple<Integer, INode, Integer> undoInfo = diff.modify(oldinode, newinode);
+      final Triple<Integer, INode, Integer> undoInfo = diff.modify(oldinode, newinode,
true);
 
       if (testUndo) {
         final String after = toString(diff);
@@ -322,7 +322,7 @@ public class TestINodeDirectoryWithSnaps
         diff.undoModify(oldinode, newinode, undoInfo);
         assertDiff(before, diff);
         //re-do
-        diff.modify(oldinode, newinode);
+        diff.modify(oldinode, newinode, true);
         assertDiff(after, diff);
       }
     }

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java?rev=1437256&r1=1437255&r2=1437256&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java
Wed Jan 23 02:48:01 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.permission.F
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper.TestDirectoryTree;
 import org.apache.hadoop.ipc.RemoteException;
@@ -55,7 +56,7 @@ public class TestSnapshot {
   private static final long seed = Time.now();
   protected static final short REPLICATION = 3;
   protected static final long BLOCKSIZE = 1024;
-  /** The number of times snapshots are created for a snapshottable directory  */
+  /** The number of times snapshots are created for a snapshottable directory */
   public static final int SNAPSHOT_ITERATION_NUMBER = 20;
   /** Height of directory tree used for testing */
   public static final int DIRECTORY_TREE_LEVEL = 5;
@@ -143,6 +144,49 @@ public class TestSnapshot {
     return nodes;
   }
 
+  /**
+   * Restart the cluster to check edit log applying and fsimage saving/loading
+   */
+  private void checkFSImage() throws Exception {
+    String rootDir = "/";
+    StringBuffer fsnStrBefore = fsn.getFSDirectory().getINode(rootDir)
+        .dumpTreeRecursively();
+    
+    cluster.shutdown();
+    cluster = new MiniDFSCluster.Builder(conf).format(false)
+        .numDataNodes(REPLICATION).build();
+    cluster.waitActive();
+    fsn = cluster.getNamesystem();
+    hdfs = cluster.getFileSystem();
+    // later check fsnStrMiddle to see if the edit log is recorded and applied
+    // correctly 
+    StringBuffer fsnStrMiddle = fsn.getFSDirectory().getINode(rootDir)
+        .dumpTreeRecursively();
+    
+    // save namespace and restart cluster
+    hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+    hdfs.saveNamespace();
+    hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+    cluster.shutdown();
+    cluster = new MiniDFSCluster.Builder(conf).format(false)
+        .numDataNodes(REPLICATION).build();
+    cluster.waitActive();
+    fsn = cluster.getNamesystem();
+    hdfs = cluster.getFileSystem();
+    // dump the namespace loaded from fsimage
+    StringBuffer fsnStrAfter = fsn.getFSDirectory().getINode(rootDir)
+        .dumpTreeRecursively();
+    
+    System.out.println("================== Original FSDir ==================");
+    System.out.println(fsnStrBefore.toString());
+    System.out.println("================== FSDir After Applying Edit Logs ==================");
+    System.out.println(fsnStrMiddle.toString());
+    System.out.println("================ FSDir After FSImage Saving/Loading ================");
+    System.out.println(fsnStrAfter.toString());
+    System.out.println("====================================================");
+    assertEquals(fsnStrBefore.toString(), fsnStrMiddle.toString());
+    assertEquals(fsnStrBefore.toString(), fsnStrAfter.toString());
+  }
   
   /**
    * Main test, where we will go in the following loop:
@@ -184,6 +228,9 @@ public class TestSnapshot {
       Modification chown = new FileChown(chownDir.nodePath, hdfs, userGroup[0],
           userGroup[1]);
       modifyCurrentDirAndCheckSnapshots(new Modification[]{chmod, chown});
+      
+      // check fsimage saving/loading
+      checkFSImage();
     }
     System.out.println("XXX done:");
     SnapshotTestHelper.dumpTreeRecursively(fsn.getFSDirectory().getINode("/"));
@@ -244,7 +291,8 @@ public class TestSnapshot {
     for (TestDirectoryTree.Node node : nodes) {
       // If the node does not have files in it, create files
       if (node.fileList == null) {
-        node.initFileList(node.nodePath.getName(), BLOCKSIZE, REPLICATION, seed, 6);
+        node.initFileList(hdfs, node.nodePath.getName(), BLOCKSIZE,
+            REPLICATION, seed, 5);
       }
       
       //
@@ -270,18 +318,21 @@ public class TestSnapshot {
       Modification delete = new FileDeletion(
           node.fileList.get((node.nullFileIndex + 1) % node.fileList.size()),
           hdfs);
-      Modification append = new FileAppend(
-          node.fileList.get((node.nullFileIndex + 2) % node.fileList.size()),
-          hdfs, (int) BLOCKSIZE);
+
+      // TODO: temporarily disable file append testing before supporting
+      // INodeFileUnderConstructionWithSnapshot in FSImage saving/loading
+//      Modification append = new FileAppend(
+//          node.fileList.get((node.nullFileIndex + 2) % node.fileList.size()),
+//          hdfs, (int) BLOCKSIZE);
       Modification chmod = new FileChangePermission(
-          node.fileList.get((node.nullFileIndex + 3) % node.fileList.size()),
+          node.fileList.get((node.nullFileIndex + 2) % node.fileList.size()),
           hdfs, genRandomPermission());
       String[] userGroup = genRandomOwner();
       Modification chown = new FileChown(
-          node.fileList.get((node.nullFileIndex + 4) % node.fileList.size()),
+          node.fileList.get((node.nullFileIndex + 3) % node.fileList.size()),
           hdfs, userGroup[0], userGroup[1]);
       Modification replication = new FileChangeReplication(
-          node.fileList.get((node.nullFileIndex + 5) % node.fileList.size()),
+          node.fileList.get((node.nullFileIndex + 4) % node.fileList.size()),
           hdfs, (short) (random.nextInt(REPLICATION) + 1));
       node.nullFileIndex = (node.nullFileIndex + 1) % node.fileList.size();
       Modification dirChange = new DirCreationOrDeletion(node.nodePath, hdfs,
@@ -289,7 +340,8 @@ public class TestSnapshot {
       
       mList.add(create);
       mList.add(delete);
-      mList.add(append);
+      //TODO
+      //mList.add(append);
       mList.add(chmod);
       mList.add(chown);
       mList.add(replication);
@@ -606,7 +658,7 @@ public class TestSnapshot {
   /**
    * Directory creation or deletion.
    */
-  static class DirCreationOrDeletion extends Modification {
+  class DirCreationOrDeletion extends Modification {
     private final TestDirectoryTree.Node node;
     private final boolean isCreation;
     private final Path changedPath;
@@ -656,15 +708,16 @@ public class TestSnapshot {
       if (isCreation) {
         // creation
         TestDirectoryTree.Node newChild = new TestDirectoryTree.Node(
-            changedPath, node.level + 1, node, node.fs);
+            changedPath, node.level + 1, node, hdfs);
         // create file under the new non-snapshottable directory
-        newChild.initFileList(node.nodePath.getName(), BLOCKSIZE, REPLICATION, seed, 2);
+        newChild.initFileList(hdfs, node.nodePath.getName(), BLOCKSIZE,
+            REPLICATION, seed, 2);
         node.nonSnapshotChildren.add(newChild);
       } else {
         // deletion
         TestDirectoryTree.Node childToDelete = node.nonSnapshotChildren
             .remove(node.nonSnapshotChildren.size() - 1);
-        node.fs.delete(childToDelete.nodePath, true);
+        hdfs.delete(childToDelete.nodePath, true);
       }
     }
 

Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java?rev=1437256&r1=1437255&r2=1437256&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java
Wed Jan 23 02:48:01 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot.SnapshotDiff;
+import org.apache.hadoop.hdfs.util.ReadOnlyList;
 import org.apache.hadoop.ipc.RemoteException;
 import org.junit.After;
 import org.junit.Before;
@@ -85,7 +86,7 @@ public class TestSnapshotRename {
    */
   private void checkSnapshotList(INodeDirectorySnapshottable srcRoot,
       String[] sortedNames, String[] names) {
-    List<Snapshot> listByName = srcRoot.getSnapshotsByNames();
+    ReadOnlyList<Snapshot> listByName = srcRoot.getSnapshotsByNames();
     assertEquals(sortedNames.length, listByName.size());
     for (int i = 0; i < listByName.size(); i++) {
       assertEquals(sortedNames[i], listByName.get(i).getRoot().getLocalName());



Mime
View raw message