hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r573413 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/FSConstants.java src/java/org/apache/hadoop/dfs/FSEditLog.java src/java/org/apache/hadoop/dfs/FSImage.java src/java/org/apache/hadoop/dfs/FSNamesystem.java
Date Fri, 07 Sep 2007 00:04:58 GMT
Author: dhruba
Date: Thu Sep  6 17:04:55 2007
New Revision: 573413

URL: http://svn.apache.org/viewvc?rev=573413&view=rev
Log:
HADOOP-1838.  The blocksize of files created with an earlier release is
set to the default block size.  (Dhruba Borthakur via dhruba)


Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=573413&r1=573412&r2=573413&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Sep  6 17:04:55 2007
@@ -19,6 +19,9 @@
     HADOOP-1656.  The blockSize of a file is stored persistently in the file
     inode. (Dhruba Borthakur via dhruba)
 
+    HADOOP-1838.  The blocksize of files created with an earlier release is
+    set to the default block size.  (Dhruba Borthakur via dhruba)
+
   NEW FEATURES
 
     HADOOP-1636.  Allow configuration of the number of jobs kept in

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java?rev=573413&r1=573412&r2=573413&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java Thu Sep  6 17:04:55
2007
@@ -156,7 +156,7 @@
   // Version is reflected in the data storage file.
   // Versions are negative.
   // Decrement LAYOUT_VERSION to define a new version.
-  public static final int LAYOUT_VERSION = -8;
+  public static final int LAYOUT_VERSION = -9;
   // Current version: 
-  // added blocksize to inode
+  // files with one block uses the default blocksize
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java?rev=573413&r1=573412&r2=573413&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java Thu Sep  6 17:04:55
2007
@@ -27,6 +27,7 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.lang.Math;
 
 import org.apache.hadoop.io.ArrayWritable;
 import org.apache.hadoop.io.UTF8;
@@ -282,15 +283,17 @@
 
             // Older versions of HDFS does not store the block size in inode.
             // If the file has more than one block, use the size of the
-            // first block as the blocksize. Otherwise leave the blockSize as 0
-            // to indicate that we do not really know the "true" blocksize of 
-            // this file.
-            if (-7 <= logVersion) {
-              assert blockSize == 0;
+            // first block as the blocksize. Otherwise use the default
+            // block size.
+            if (-8 <= logVersion && blockSize == 0) {
               if (blocks.length > 1) {
                 blockSize = blocks[0].getNumBytes();
+              } else {
+                long first = ((blocks.length == 1)? blocks[0].getNumBytes(): 0);
+                blockSize = Math.max(fsNamesys.getDefaultBlockSize(), first);
               }
             }
+
             // add to the file tree
             fsDir.unprotectedAddFile(name.toString(), blocks, replication, 
                                      mtime, blockSize);

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java?rev=573413&r1=573412&r2=573413&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java Thu Sep  6 17:04:55 2007
@@ -35,6 +35,7 @@
 import java.util.Map;
 import java.util.Properties;
 import java.util.Random;
+import java.lang.Math;
 
 import org.apache.hadoop.dfs.FSConstants.StartupOption;
 import org.apache.hadoop.dfs.FSConstants.NodeType;
@@ -693,13 +694,14 @@
         }
         // Older versions of HDFS does not store the block size in inode.
         // If the file has more than one block, use the size of the 
-        // first block as the blocksize. Otherwise leave the blockSize as 0
-        // to indicate that we do not really know the "true" blocksize of this
-        // file.
-        if (-7 <= imgVersion) {
-          assert blockSize == 0;
+        // first block as the blocksize. Otherwise use the default block size.
+        //
+        if (-8 <= imgVersion && blockSize == 0) {
           if (numBlocks > 1) {
             blockSize = blocks[0].getNumBytes();
+          } else {
+            long first = ((numBlocks == 1) ? blocks[0].getNumBytes(): 0);
+            blockSize = Math.max(fsNamesys.getDefaultBlockSize(), first);
           }
         }
         fsDir.unprotectedAddFile(name.toString(), blocks, replication,

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?rev=573413&r1=573412&r2=573413&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Thu Sep  6 17:04:55
2007
@@ -182,6 +182,8 @@
   private long replicationRecheckInterval;
   //decommissionRecheckInterval is how often namenode checks if a node has finished decommission
   private long decommissionRecheckInterval;
+  // default block size of a file
+  private long defaultBlockSize = 0;
   static int replIndex = 0; // last datanode used for replication work
   static int REPL_WORK_PER_ITERATION = 32; // max percent datanodes per iteration
 
@@ -200,7 +202,7 @@
   private Daemon dnthread = null;
 
   /**
-   * dirs is a list oif directories where the filesystem directory state 
+   * dirs is a list of directories where the filesystem directory state 
    * is stored
    */
   public FSNamesystem(String hostname,
@@ -306,6 +308,7 @@
     this.decommissionRecheckInterval = conf.getInt(
                                                    "dfs.namenode.decommission.interval",
                                                    5 * 60 * 1000);    
+    this.defaultBlockSize = conf.getLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
   }
 
   /** Return the FSNamesystem object
@@ -314,7 +317,7 @@
   public static FSNamesystem getFSNamesystem() {
     return fsNamesystemObject;
   } 
-    
+
   NamespaceInfo getNamespaceInfo() {
     return new NamespaceInfo(dir.fsImage.getNamespaceID(),
                              dir.fsImage.getCTime(),
@@ -401,6 +404,10 @@
 
     out.flush();
     out.close();
+  }
+
+  long getDefaultBlockSize() {
+    return defaultBlockSize;
   }
     
   /* get replication factor of a block */



Mime
View raw message