hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jgho...@apache.org
Subject svn commit: r943214 - in /hadoop/hdfs/trunk: ./ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/unit/org/apache/hadoop/hdfs/server/namenode/
Date Tue, 11 May 2010 18:17:15 GMT
Author: jghoman
Date: Tue May 11 18:17:14 2010
New Revision: 943214

URL: http://svn.apache.org/viewvc?rev=943214&view=rev
Log:
HDFS-1061. Memory footprint optimization for INodeFile object. (Bharath Mundlapudi)

Added:
    hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=943214&r1=943213&r2=943214&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Tue May 11 18:17:14 2010
@@ -2,6 +2,11 @@ Hadoop HDFS Change Log
 
 Trunk (unreleased changes)
 
+  IMPROVEMENTS
+
+    HDFS-1061. Memory footprint optimization for INodeFile object. 
+    (Bharath Mundlapudi via jghoman)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=943214&r1=943213&r2=943214&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue
May 11 18:17:14 2010
@@ -870,7 +870,7 @@ public class FSNamesystem implements FSC
         throw new IllegalArgumentException("concat: "+ target + " file is empty");
       }
 
-      long blockSize = trgInode.preferredBlockSize;
+      long blockSize = trgInode.getPreferredBlockSize();
 
       // check the end block to be full
       if(blockSize != trgInode.blocks[trgInode.blocks.length-1].getNumBytes()) {
@@ -878,7 +878,7 @@ public class FSNamesystem implements FSC
       }
 
       si.add(trgInode);
-      short repl = trgInode.blockReplication;
+      short repl = trgInode.getReplication();
 
       // now check the srcs
       boolean endSrc = false; // final src file doesn't have to have full end block
@@ -898,10 +898,10 @@ public class FSNamesystem implements FSC
         }
 
         // check replication and blocks size
-        if(repl != srcInode.blockReplication) {
+        if(repl != srcInode.getReplication()) {
           throw new IllegalArgumentException(src + " and " + target + " " +
               "should have same replication: "
-              + repl + " vs. " + srcInode.blockReplication);
+              + repl + " vs. " + srcInode.getReplication());
         }
 
         //boolean endBlock=false;

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=943214&r1=943213&r2=943214&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Tue May
11 18:17:14 2010
@@ -28,9 +28,16 @@ import org.apache.hadoop.hdfs.protocol.B
 class INodeFile extends INode {
   static final FsPermission UMASK = FsPermission.createImmutable((short)0111);
 
+  //Number of bits for Block size
+  static final short BLOCKBITS = 48;
+
+  //Header mask 64-bit representation
+  //Format: [16 bits for replication][48 bits for PreferredBlockSize]
+  static final long HEADERMASK = 0xffffL << BLOCKBITS;
+
+  protected long header;
+
   protected BlockInfo blocks[] = null;
-  protected short blockReplication;
-  protected long preferredBlockSize;
 
   INodeFile(PermissionStatus permissions,
             int nrBlocks, short replication, long modificationTime,
@@ -41,16 +48,15 @@ class INodeFile extends INode {
 
   protected INodeFile() {
     blocks = null;
-    blockReplication = 0;
-    preferredBlockSize = 0;
+    header = 0;
   }
 
   protected INodeFile(PermissionStatus permissions, BlockInfo[] blklist,
                       short replication, long modificationTime,
                       long atime, long preferredBlockSize) {
     super(permissions, modificationTime, atime);
-    this.blockReplication = replication;
-    this.preferredBlockSize = preferredBlockSize;
+    this.setReplication(replication);
+    this.setPreferredBlockSize(preferredBlockSize);
     blocks = blklist;
   }
 
@@ -69,14 +75,31 @@ class INodeFile extends INode {
 
   /**
    * Get block replication for the file 
-   * @return block replication
+   * @return block replication value
    */
   public short getReplication() {
-    return this.blockReplication;
+    return (short) ((header & HEADERMASK) >> BLOCKBITS);
   }
 
-  void setReplication(short replication) {
-    this.blockReplication = replication;
+  public void setReplication(short replication) {
+    if(replication <= 0)
+       throw new IllegalArgumentException("Unexpected value for the replication");
+    header = ((long)replication << BLOCKBITS) | (header & ~HEADERMASK);
+  }
+
+  /**
+   * Get preferred block size for the file
+   * @return preferred block size in bytes
+   */
+  public long getPreferredBlockSize() {
+        return header & ~HEADERMASK;
+  }
+
+  public void setPreferredBlockSize(long preferredBlkSize)
+  {
+    if((preferredBlkSize < 0) || (preferredBlkSize > ~HEADERMASK ))
+       throw new IllegalArgumentException("Unexpected value for the block size");
+    header = (header & HEADERMASK) | (preferredBlkSize & ~HEADERMASK);
   }
 
   /**
@@ -195,20 +218,12 @@ class INodeFile extends INode {
      */
     if (blkArr.length > 0 && blkArr[blkArr.length-1] != null && 
         isUnderConstruction()) {
-      size += preferredBlockSize - blocks[blocks.length-1].getNumBytes();
+      size += getPreferredBlockSize() - blocks[blocks.length-1].getNumBytes();
     }
-    return size * blockReplication;
+    return size * getReplication();
   }
   
   /**
-   * Get the preferred block size of the file.
-   * @return the number of bytes
-   */
-  public long getPreferredBlockSize() {
-    return preferredBlockSize;
-  }
-
-  /**
    * Return the penultimate allocated block for this file.
    */
   BlockInfo getPenultimateBlock() {

Added: hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java?rev=943214&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
(added)
+++ hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
Tue May 11 18:17:14 2010
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.junit.Assert.*;
+
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+
+import org.junit.Test;
+
+public class TestINodeFile {
+
+  static final short BLOCKBITS = 48;
+  static final long BLKSIZE_MAXVALUE = ~(0xffffL << BLOCKBITS);
+
+  private String userName = "Test";
+  private short replication;
+  private long preferredBlockSize;
+
+  /**
+   * Test for the Replication value. Sets a value and checks if it was set
+   * correct.
+   */
+  @Test
+  public void testReplication () {
+    replication = 3;
+    preferredBlockSize = 128*1024*1024;
+    INodeFile inf = new INodeFile(new PermissionStatus(userName, null, 
+                                  FsPermission.getDefault()), null, replication,
+                                  0L, 0L, preferredBlockSize);
+    assertEquals("True has to be returned in this case", replication,
+                 inf.getReplication());
+  }
+
+  /**
+   * IllegalArgumentException is expected for setting below lower bound
+   * for Replication.
+   * @throws IllegalArgumentException as the result
+   */
+  @Test(expected=IllegalArgumentException.class)
+  public void testReplicationBelowLowerBound ()
+              throws IllegalArgumentException {
+    replication = -1;
+    preferredBlockSize = 128*1024*1024;
+    INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+                                  FsPermission.getDefault()), null, replication,
+                                  0L, 0L, preferredBlockSize);
+  }
+
+  /**
+   * Test for the PreferredBlockSize value. Sets a value and checks if it was
+   * set correct.
+   */
+  @Test
+  public void testPreferredBlockSize () {
+    replication = 3;
+    preferredBlockSize = 128*1024*1024;
+    INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+                                  FsPermission.getDefault()), null, replication,
+                                  0L, 0L, preferredBlockSize);
+    assertEquals("True has to be returned in this case", preferredBlockSize,
+           inf.getPreferredBlockSize());
+  }
+
+  @Test
+  public void testPreferredBlockSizeUpperBound () {
+    replication = 3;
+    preferredBlockSize = BLKSIZE_MAXVALUE;
+    INodeFile inf = new INodeFile(new PermissionStatus(userName, null, 
+                                  FsPermission.getDefault()), null, replication,
+                                  0L, 0L, preferredBlockSize);
+    assertEquals("True has to be returned in this case", BLKSIZE_MAXVALUE,
+                 inf.getPreferredBlockSize());
+  }
+
+  /**
+   * IllegalArgumentException is expected for setting below lower bound
+   * for PreferredBlockSize.
+   * @throws IllegalArgumentException as the result
+   */
+  @Test(expected=IllegalArgumentException.class)
+  public void testPreferredBlockSizeBelowLowerBound ()
+              throws IllegalArgumentException {
+    replication = 3;
+    preferredBlockSize = -1;
+    INodeFile inf = new INodeFile(new PermissionStatus(userName, null, 
+                                  FsPermission.getDefault()), null, replication,
+                                  0L, 0L, preferredBlockSize);
+  } 
+
+  /**
+   * IllegalArgumentException is expected for setting above upper bound
+   * for PreferredBlockSize.
+   * @throws IllegalArgumentException as the result
+   */
+  @Test(expected=IllegalArgumentException.class)
+  public void testPreferredBlockSizeAboveUpperBound ()
+              throws IllegalArgumentException {
+    replication = 3;
+    preferredBlockSize = BLKSIZE_MAXVALUE+1;
+    INodeFile inf = new INodeFile(new PermissionStatus(userName, null, 
+                                  FsPermission.getDefault()), null, replication,
+                                  0L, 0L, preferredBlockSize);
+  }
+
+}



Mime
View raw message