hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject svn commit: r639044 - in /hadoop/core/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/BlockCrcUpgrade.java src/java/org/apache/hadoop/dfs/FSDataset.java
Date Wed, 19 Mar 2008 22:40:01 GMT
Author: shv
Date: Wed Mar 19 15:39:54 2008
New Revision: 639044

URL: http://svn.apache.org/viewvc?rev=639044&view=rev
Log:
HADOOP-2148. Eliminate redundant data-node blockMap lookups. Contributed by Konstantin Shvachko.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/BlockCrcUpgrade.java
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=639044&r1=639043&r2=639044&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Mar 19 15:39:54 2008
@@ -126,9 +126,11 @@
     HADOOP-2423.  Code optimization in FSNamesystem.mkdirs.
     (Tsz Wo (Nicholas), SZE via dhruba)
 
-		HADOOP-2606. ReplicationMonitor selects data-nodes to replicate directly
+    HADOOP-2606. ReplicationMonitor selects data-nodes to replicate directly
 		from needed replication blocks instead of looking up for the blocks for 
 		each live data-node. (shv)
+
+    HADOOP-2148. Eliminate redundant data-node blockMap lookups. (shv)
 
   BUG FIXES
 

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/BlockCrcUpgrade.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/BlockCrcUpgrade.java?rev=639044&r1=639043&r2=639044&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/BlockCrcUpgrade.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/BlockCrcUpgrade.java Wed Mar 19 15:39:54
2008
@@ -441,11 +441,6 @@
     Block block = blockInfo.block;
     FSDataset data = (FSDataset) blockInfo.dataNode.data;
     File blockFile = data.getBlockFile( block );
-    if ( blockFile == null || !blockFile.exists() ) {
-      throw new IOException("Block file "  + 
-                            ((blockFile != null) ? blockFile.getAbsolutePath()
-                              : "NULL") + " does not exist.");
-    }
     
     byte[] blockBuf = new byte[bytesPerChecksum];
     FileInputStream in = null;
@@ -785,9 +780,6 @@
     Block block = blockInfo.block;
     FSDataset data = (FSDataset) blockInfo.dataNode.data;
     File blockFile = data.getBlockFile( block );
-    if (blockFile == null || !blockFile.exists()) {
-      throw new IOException("Could not local file for block");
-    }
     long blockLen = blockFile.length();
     if ( blockLen != blockFile.length()) {
       LOG.warn("Mismatch in length for block: local file size is " +
@@ -1462,11 +1454,6 @@
         LOG.warn("Could not find file location for " + b + 
                  ". It might already be deleted. Exception : " +
                  StringUtils.stringifyException(e));
-        errors++;
-        continue;
-      }
-      if (!blockFile.exists()) {
-        LOG.error("could not find block file " + blockFile);
         errors++;
         continue;
       }

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java?rev=639044&r1=639043&r2=639044&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java Wed Mar 19 15:39:54 2008
@@ -527,15 +527,16 @@
   
   protected File getMetaFile(Block b) throws IOException {
     File blockFile = getBlockFile( b );
-    return new File( blockFile.getAbsolutePath() + METADATA_EXTENSION ); 
+    return getMetaFile(blockFile); 
   }
+
   public boolean metaFileExists(Block b) throws IOException {
     return getMetaFile(b).exists();
   }
   
   public long getMetaDataLength(Block b) throws IOException {
     File checksumFile = getMetaFile( b );
-  return checksumFile.length();
+    return checksumFile.length();
   }
 
   public MetaDataInputStream getMetaDataInputStream(Block b)
@@ -566,7 +567,7 @@
     volumeMap = new HashMap<Block, DatanodeBlockInfo>();
     volumes.getVolumeMap(volumeMap);
     blockWriteTimeout = Math.max(
-         conf.getInt("dfs.datanode.block.write.timeout.sec", 3600), 1) * 1000;
+         conf.getInt("dfs.datanode.block.write.timeout.sec", 3600), 1) * 1000L;
     registerMBean(storage.getStorageID());
   }
 
@@ -595,10 +596,10 @@
    * Find the block's on-disk length
    */
   public long getLength(Block b) throws IOException {
-    if (!isValidBlock(b)) {
+    File f = validateBlockFile(b);
+    if(f == null) {
       throw new IOException("Block " + b + " is not valid.");
     }
-    File f = getFile(b);
     return f.length();
   }
 
@@ -606,10 +607,11 @@
    * Get File name for a given block.
    */
   protected synchronized File getBlockFile(Block b) throws IOException {
-    if (!isValidBlock(b)) {
+    File f = validateBlockFile(b);
+    if(f == null) {
       throw new IOException("Block " + b + " is not valid.");
     }
-    return getFile(b);
+    return f;
   }
   
   public synchronized InputStream getBlockInputStream(Block b) throws IOException {
@@ -838,9 +840,18 @@
    * Check whether the given block is a valid one.
    */
   public boolean isValidBlock(Block b) {
+    return validateBlockFile(b) != null;
+  }
+
+  /**
+   * Find the file corresponding to the block and return it if it exists.
+   */
+  File validateBlockFile(Block b) {
     //Should we check for metadata file too?
     File f = getFile(b);
-    return (f!= null && f.exists());
+    if(f != null && f.exists())
+      return f;
+    return null;
   }
 
   /**
@@ -973,6 +984,10 @@
     return toString();
   }
   
+  /**
+   *  A duplicate of {@link #getLength()}
+   */
+  @Deprecated
   public long getBlockSize(Block b) {
     return getFile(b).length();
   }



Mime
View raw message