hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject svn commit: r653866 - in /hadoop/core/trunk: ./ src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/fs/permission/
Date Tue, 06 May 2008 18:13:49 GMT
Author: shv
Date: Tue May  6 11:13:48 2008
New Revision: 653866

URL: http://svn.apache.org/viewvc?rev=653866&view=rev
Log:
HADOOP-3248. Optimization of saveFSImage. Contributed by Dhruba.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/FsPermission.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/PermissionStatus.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=653866&r1=653865&r2=653866&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue May  6 11:13:48 2008
@@ -100,6 +100,8 @@
     On Linux DataNode takes 5 times less CPU while serving data. Results may
     vary on other platforms. (rangadi)
 
+    HADOOP-3248. Optimization of saveFSImage. (Dhruba via shv)
+
   BUG FIXES
 
     HADOOP-2905. 'fsck -move' triggers NPE in NameNode. 

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java?rev=653866&r1=653865&r2=653866&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java Tue May  6 11:13:48 2008
@@ -37,8 +37,10 @@
 import java.util.Properties;
 import java.util.Random;
 import java.lang.Math;
+import java.nio.ByteBuffer;
 
 import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.dfs.FSConstants.CheckpointStates;
 import org.apache.hadoop.dfs.FSConstants.StartupOption;
@@ -85,6 +87,13 @@
   volatile private CheckpointStates ckptState = CheckpointStates.START; 
 
   /**
+   * Used for saving the image to disk
+   */
+  static private final FsPermission fileperm = new FsPermission((short)0);
+  static private final byte[] separator = INode.string2Bytes("/");
+  static private byte[] byteStore = null;
+
+  /**
    */
   FSImage() {
     super(NodeType.NAME_NODE);
@@ -785,7 +794,7 @@
   /**
    * Save the contents of the FS image to the file.
    */
-  void saveFSImage(File newFile ) throws IOException {
+  void saveFSImage(File newFile) throws IOException {
     FSNamesystem fsNamesys = FSNamesystem.getFSNamesystem();
     FSDirectory fsDir = fsNamesys.dir;
     //
@@ -799,8 +808,12 @@
       out.writeInt(namespaceID);
       out.writeInt(fsDir.rootDir.numItemsInTree() - 1);
       out.writeLong(fsNamesys.getGenerationStamp());
-      saveImage("", fsDir.rootDir, out);
+      byteStore = new byte[4*FSConstants.MAX_PATH_LENGTH];
+      ByteBuffer strbuf = ByteBuffer.wrap(byteStore);
+      saveImage(strbuf, 0, fsDir.rootDir, out);
       fsNamesys.saveFilesUnderConstruction(out);
+      byteStore = null;
+      strbuf = null;
     } finally {
       out.close();
     }
@@ -875,13 +888,16 @@
   /**
    * Save file tree image starting from the given root.
    */
-  private static void saveImage(String parentPrefix, 
+  private static void saveImage(ByteBuffer parentPrefix, 
+                                int prefixLength,
                                 INode inode, 
                                 DataOutputStream out) throws IOException {
-    String fullName = "";
+    int newPrefixLength = prefixLength;
     if (inode.getParent() != null) {
-      fullName = parentPrefix + "/" + inode.getLocalName();
-      new UTF8(fullName).write(out);
+      parentPrefix.put(separator).put(inode.getLocalNameBytes());
+      newPrefixLength += separator.length + inode.getLocalNameBytes().length;
+      out.writeShort(newPrefixLength);
+      out.write(byteStore, 0, newPrefixLength);
       if (!inode.isDirectory()) {  // write file inode
         INodeFile fileINode = (INodeFile)inode;
         out.writeShort(fileINode.getReplication());
@@ -891,7 +907,11 @@
         out.writeInt(blocks.length);
         for (Block blk : blocks)
           blk.write(out);
-        fileINode.getPermissionStatus().write(out);
+        fileperm.fromShort(fileINode.getFsPermissionShort());
+        PermissionStatus.write(out, fileINode.getUserName(),
+                               fileINode.getGroupName(),
+                               fileperm);
+        parentPrefix.position(prefixLength);
         return;
       }
       // write directory inode
@@ -899,11 +919,17 @@
       out.writeLong(inode.getModificationTime());
       out.writeLong(0);   // preferred block size
       out.writeInt(-1);    // # of blocks
-      inode.getPermissionStatus().write(out);
-    }
-    for(INode child : ((INodeDirectory)inode).getChildren()) {
-      saveImage(fullName, child, out);
+      fileperm.fromShort(inode.getFsPermissionShort());
+      PermissionStatus.write(out, inode.getUserName(),
+                             inode.getGroupName(),
+                             fileperm);
+    }
+    if (((INodeDirectory)inode).getChildrenRaw() != null) {
+      for(INode child : ((INodeDirectory)inode).getChildren()) {
+        saveImage(parentPrefix, newPrefixLength, child, out);
+      }
     }
+    parentPrefix.position(prefixLength);
   }
 
   void loadDatanodes(int version, DataInputStream in) throws IOException {

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java?rev=653866&r1=653865&r2=653866&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java Tue May  6 11:13:48 2008
@@ -125,6 +125,9 @@
     return new FsPermission(
         (short)PermissionStatusFormat.MODE.retrieve(permission));
   }
+  protected short getFsPermissionShort() {
+    return (short)PermissionStatusFormat.MODE.retrieve(permission);
+  }
   /** Set the {@link FsPermission} of this {@link INode} */
   protected void setPermission(FsPermission permission) {
     updatePermissionStatus(PermissionStatusFormat.MODE, permission.toShort());
@@ -156,6 +159,14 @@
   }
 
   /**
+   * Get local file name
+   * @return local file name
+   */
+  byte[] getLocalNameBytes() {
+    return name;
+  }
+
+  /**
    * Set local file name
    */
   void setLocalName(String name) {
@@ -565,6 +576,9 @@
   List<INode> getChildren() {
     return children==null ? new ArrayList<INode>() : children;
   }
+  List<INode> getChildrenRaw() {
+    return children;
+  }
 
   /**
    * Collect all the blocks in all children of this INode.

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/FsPermission.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/FsPermission.java?rev=653866&r1=653865&r2=653866&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/FsPermission.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/FsPermission.java Tue May 
6 11:13:48 2008
@@ -94,7 +94,7 @@
     groupaction = g;
     otheraction = o;
   }
-  private void fromShort(short n) {
+  public void fromShort(short n) {
     FsAction[] v = FsAction.values();
     set(v[(n >>> 6) & 7], v[(n >>> 3) & 7], v[n & 7]);
   }
@@ -194,4 +194,4 @@
     }
     return new FsPermission((short)n);
   }
-}
\ No newline at end of file
+}

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/PermissionStatus.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/PermissionStatus.java?rev=653866&r1=653865&r2=653866&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/PermissionStatus.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/PermissionStatus.java Tue May
 6 11:13:48 2008
@@ -87,9 +87,7 @@
 
   /** {@inheritDoc} */
   public void write(DataOutput out) throws IOException {
-    Text.writeString(out, username);
-    Text.writeString(out, groupname);
-    permission.write(out);
+    write(out, username, groupname, permission);
   }
 
   /**
@@ -101,8 +99,20 @@
     return p;
   }
 
+  /**
+   * Serialize a {@link PermissionStatus} from its base components.
+   */
+  public static void write(DataOutput out,
+                           String username, 
+                           String groupname,
+                           FsPermission permission) throws IOException {
+    Text.writeString(out, username);
+    Text.writeString(out, groupname);
+    permission.write(out);
+  }
+
   /** {@inheritDoc} */
   public String toString() {
     return username + ":" + groupname + ":" + permission;
   }
-}
\ No newline at end of file
+}



Mime
View raw message