hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1132839 - in /hadoop/hdfs/branches/yahoo-merge: ./ src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/ src/java/org/apache/hadoop/hdfs/ src/test/hdfs/ src/test/hdfs/org/apache/hadoop/hdfs/ src/test/hdfs/org/apache/hadoop/hdfs/server/namenode...
Date Tue, 07 Jun 2011 00:08:02 GMT
Author: suresh
Date: Tue Jun  7 00:08:02 2011
New Revision: 1132839

URL: http://svn.apache.org/viewvc?rev=1132839&view=rev
Log:
Merging r1125145 for HDFS-1371 from trunk

Added:
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/TestClientReportBadBlock.java
      - copied unchanged from r1125145, hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestClientReportBadBlock.java
Modified:
    hadoop/hdfs/branches/yahoo-merge/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
    hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DFSInputStream.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
    hadoop/hdfs/branches/yahoo-merge/src/webapps/datanode/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/secondary/   (props changed)

Propchange: hadoop/hdfs/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jun  7 00:08:02 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265:796829-820463
 /hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1036738,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829
+/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1036738,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125145,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829

Modified: hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/CHANGES.txt?rev=1132839&r1=1132838&r2=1132839&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/hdfs/branches/yahoo-merge/CHANGES.txt Tue Jun  7 00:08:02 2011
@@ -476,6 +476,9 @@ Trunk (unreleased changes)
     HDFS-2020. Fix TestDFSUpgradeFromImage by removing the use of DataNode
     as a singleton. (suresh via todd)
 
+    HDFS-1371. One bad node can incorrectly flag many files as corrupt.
+    (Tanping Wang via jitendra)
+
     HDFS-1891. Disable IPV6 for junit tests to fix TestBackupNode failure.
     (suresh)
 

Propchange: hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jun  7 00:08:02 2011
@@ -1,4 +1,4 @@
 /hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
 /hadoop/core/trunk/src/c++/libhdfs:776175-784663
 /hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
-/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829
+/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125145,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829

Propchange: hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jun  7 00:08:02 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125145,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jun  7 00:08:02 2011
@@ -4,4 +4,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/java:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829
+/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125145,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DFSInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DFSInputStream.java?rev=1132839&r1=1132838&r2=1132839&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DFSInputStream.java (original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DFSInputStream.java Tue
Jun  7 00:08:02 2011
@@ -22,8 +22,13 @@ import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.util.AbstractMap;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -62,7 +67,7 @@ public class DFSInputStream extends FSIn
   private LocatedBlocks locatedBlocks = null;
   private long lastBlockBeingWrittenLength = 0;
   private DatanodeInfo currentNode = null;
-  private ExtendedBlock currentBlock = null;
+  private LocatedBlock currentLocatedBlock = null;
   private long pos = 0;
   private long blockEnd = -1;
 
@@ -204,8 +209,11 @@ public class DFSInputStream extends FSIn
   /**
    * Returns the block containing the target position. 
    */
-  public ExtendedBlock getCurrentBlock() {
-    return currentBlock;
+  synchronized public ExtendedBlock getCurrentBlock() {
+    if (currentLocatedBlock == null){
+      return null;
+    }
+    return currentLocatedBlock.getBlock();
   }
 
   /**
@@ -261,7 +269,7 @@ public class DFSInputStream extends FSIn
     if (updatePosition) {
       pos = offset;
       blockEnd = blk.getStartOffset() + blk.getBlockSize() - 1;
-      currentBlock = blk.getBlock();
+      currentLocatedBlock = blk;
     }
     return blk;
   }
@@ -459,8 +467,9 @@ public class DFSInputStream extends FSIn
    * name readBuffer() is chosen to imply similarity to readBuffer() in
    * ChecksuFileSystem
    */ 
-  private synchronized int readBuffer(byte buf[], int off, int len) 
-                                                  throws IOException {
+  private synchronized int readBuffer(byte buf[], int off, int len,
+      Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
+      throws IOException {
     IOException ioe;
     
     /* we retry current node only once. So this is set to true only here.
@@ -476,16 +485,19 @@ public class DFSInputStream extends FSIn
       try {
         return blockReader.read(buf, off, len);
       } catch ( ChecksumException ce ) {
-        DFSClient.LOG.warn("Found Checksum error for " + currentBlock + " from " +
-                 currentNode.getName() + " at " + ce.getPos());          
-        dfsClient.reportChecksumFailure(src, currentBlock, currentNode);
+        DFSClient.LOG.warn("Found Checksum error for "
+            + getCurrentBlock() + " from " + currentNode.getName()
+            + " at " + ce.getPos());        
         ioe = ce;
         retryCurrentNode = false;
+        // we want to remember which block replicas we have tried
+        addIntoCorruptedBlockMap(getCurrentBlock(), currentNode,
+            corruptedBlockMap);
       } catch ( IOException e ) {
         if (!retryCurrentNode) {
-          DFSClient.LOG.warn("Exception while reading from " + currentBlock +
-                   " of " + src + " from " + currentNode + ": " +
-                   StringUtils.stringifyException(e));
+          DFSClient.LOG.warn("Exception while reading from "
+              + getCurrentBlock() + " of " + src + " from "
+              + currentNode + ": " + StringUtils.stringifyException(e));
         }
         ioe = e;
       }
@@ -516,6 +528,8 @@ public class DFSInputStream extends FSIn
     if (closed) {
       throw new IOException("Stream closed");
     }
+    Map<ExtendedBlock,Set<DatanodeInfo>> corruptedBlockMap 
+      = new HashMap<ExtendedBlock, Set<DatanodeInfo>>();
     failures = 0;
     if (pos < getFileLength()) {
       int retries = 2;
@@ -525,7 +539,7 @@ public class DFSInputStream extends FSIn
             currentNode = blockSeekTo(pos);
           }
           int realLen = (int) Math.min((long) len, (blockEnd - pos + 1L));
-          int result = readBuffer(buf, off, realLen);
+          int result = readBuffer(buf, off, realLen, corruptedBlockMap);
           
           if (result >= 0) {
             pos += result;
@@ -548,12 +562,34 @@ public class DFSInputStream extends FSIn
           if (--retries == 0) {
             throw e;
           }
+        } finally {
+          // Check if need to report block replicas corruption either read
+          // was successful or ChecksumException occured.
+          reportCheckSumFailure(corruptedBlockMap, 
+              currentLocatedBlock.getLocations().length);
         }
       }
     }
     return -1;
   }
 
+  /**
+   * Add corrupted block replica into map.
+   * @param corruptedBlockMap 
+   */
+  private void addIntoCorruptedBlockMap(ExtendedBlock blk, DatanodeInfo node, 
+      Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap) {
+    Set<DatanodeInfo> dnSet = null;
+    if((corruptedBlockMap.containsKey(blk))) {
+      dnSet = corruptedBlockMap.get(blk);
+    }else {
+      dnSet = new HashSet<DatanodeInfo>();
+    }
+    if (!dnSet.contains(node)) {
+      dnSet.add(node);
+      corruptedBlockMap.put(blk, dnSet);
+    }
+  }
       
   private DNAddrPair chooseDataNode(LocatedBlock block)
     throws IOException {
@@ -602,8 +638,10 @@ public class DFSInputStream extends FSIn
     }
   } 
       
-  private void fetchBlockByteRange(LocatedBlock block, long start,
-                                   long end, byte[] buf, int offset) throws IOException {
+  private void fetchBlockByteRange(LocatedBlock block, long start, long end,
+      byte[] buf, int offset,
+      Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
+      throws IOException {
     //
     // Connect to best DataNode for desired Block, with potential offset
     //
@@ -643,7 +681,8 @@ public class DFSInputStream extends FSIn
         DFSClient.LOG.warn("fetchBlockByteRange(). Got a checksum exception for " +
                  src + " at " + block.getBlock() + ":" + 
                  e.getPos() + " from " + chosenNode.getName());
-        dfsClient.reportChecksumFailure(src, block.getBlock(), chosenNode);
+        // we want to remember what we have tried
+        addIntoCorruptedBlockMap(block.getBlock(), chosenNode, corruptedBlockMap);
       } catch (IOException e) {
         if (e instanceof InvalidBlockTokenException && refetchToken > 0) {
           DFSClient.LOG.info("Will get a new access token and retry, "
@@ -698,11 +737,21 @@ public class DFSInputStream extends FSIn
     // corresponding to position and realLen
     List<LocatedBlock> blockRange = getBlockRange(position, realLen);
     int remaining = realLen;
+    Map<ExtendedBlock,Set<DatanodeInfo>> corruptedBlockMap 
+      = new HashMap<ExtendedBlock, Set<DatanodeInfo>>();
     for (LocatedBlock blk : blockRange) {
       long targetStart = position - blk.getStartOffset();
       long bytesToRead = Math.min(remaining, blk.getBlockSize() - targetStart);
-      fetchBlockByteRange(blk, targetStart, 
-                          targetStart + bytesToRead - 1, buffer, offset);
+      try {
+        fetchBlockByteRange(blk, targetStart, 
+            targetStart + bytesToRead - 1, buffer, offset, corruptedBlockMap);
+      } finally {
+        // Check and report if any block replicas are corrupted.
+        // BlockMissingException may be caught if all block replicas are
+        // corrupted.
+        reportCheckSumFailure(corruptedBlockMap, blk.getLocations().length);
+      }
+
       remaining -= bytesToRead;
       position += bytesToRead;
       offset += bytesToRead;
@@ -713,7 +762,43 @@ public class DFSInputStream extends FSIn
     }
     return realLen;
   }
-   
+  
+  /**
+   * DFSInputStream reports checksum failure.
+   * Case I : client has tried multiple data nodes and at least one of the
+   * attempts has succeeded. We report the other failures as corrupted block to
+   * namenode. 
+   * Case II: client has tried out all data nodes, but all failed. We
+   * only report if the total number of replica is 1. We do not
+   * report otherwise since this maybe due to the client is a handicapped client
+   * (who can not read).
+   * @param corruptedBlockMap, map of corrupted blocks
+   * @param dataNodeCount, number of data nodes who contains the block replicas
+   */
+  private void reportCheckSumFailure(
+      Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap, 
+      int dataNodeCount) {
+    if (corruptedBlockMap.isEmpty()) {
+      return;
+    }
+    Iterator<Entry<ExtendedBlock, Set<DatanodeInfo>>> it = corruptedBlockMap
+        .entrySet().iterator();
+    Entry<ExtendedBlock, Set<DatanodeInfo>> entry = it.next();
+    ExtendedBlock blk = entry.getKey();
+    Set<DatanodeInfo> dnSet = entry.getValue();
+    if (((dnSet.size() < dataNodeCount) && (dnSet.size() > 0))
+        || ((dataNodeCount == 1) && (dnSet.size() == dataNodeCount))) {
+      DatanodeInfo[] locs = new DatanodeInfo[dnSet.size()];
+      int i = 0;
+      for (DatanodeInfo dn:dnSet) {
+        locs[i++] = dn;
+      }
+      LocatedBlock [] lblocks = { new LocatedBlock(blk, locs) };
+      dfsClient.reportChecksumFailure(src, lblocks);
+    }
+    corruptedBlockMap.clear();
+  }
+
   @Override
   public long skip(long n) throws IOException {
     if ( n > 0 ) {
@@ -752,9 +837,10 @@ public class DFSInputStream extends FSIn
           }
         } catch (IOException e) {//make following read to retry
           if(DFSClient.LOG.isDebugEnabled()) {
-            DFSClient.LOG.debug("Exception while seek to " + targetPos +
-                " from " + currentBlock +" of " + src + " from " +
-                currentNode + ": " + StringUtils.stringifyException(e));
+            DFSClient.LOG.debug("Exception while seek to " + targetPos
+                + " from " + getCurrentBlock() + " of " + src
+                + " from " + currentNode + ": "
+                + StringUtils.stringifyException(e));
           }
         }
       }

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1132839&r1=1132838&r2=1132839&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
Tue Jun  7 00:08:02 2011
@@ -680,6 +680,9 @@ public class DistributedFileSystem exten
    * is corrupt but we will report both to the namenode.  In the future,
    * we can consider figuring out exactly which block is corrupt.
    */
+  // We do not see a need for user to report block checksum errors and do not  
+  // want to rely on user to report block corruptions.
+  @Deprecated
   public boolean reportChecksumFailure(Path f, 
     FSDataInputStream in, long inPos, 
     FSDataInputStream sums, long sumsPos) {

Propchange: hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jun  7 00:08:02 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
-/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829
+/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125145,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829

Modified: hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=1132839&r1=1132838&r2=1132839&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Tue Jun  7 00:08:02 2011
@@ -341,16 +341,17 @@ public class TestFsck extends TestCase {
     int replicaCount = 0;
     Random random = new Random();
     String outStr = null;
+    short factor = 1;
 
     MiniDFSCluster cluster = null;
     try {
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
     cluster.waitActive();
     fs = cluster.getFileSystem();
     Path file1 = new Path("/testCorruptBlock");
-    DFSTestUtil.createFile(fs, file1, 1024, (short)3, 0);
+    DFSTestUtil.createFile(fs, file1, 1024, factor, 0);
     // Wait until file replication has completed
-    DFSTestUtil.waitReplication(fs, file1, (short)3);
+    DFSTestUtil.waitReplication(fs, file1, factor);
     ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);
 
     // Make sure filesystem is in healthy state
@@ -358,18 +359,16 @@ public class TestFsck extends TestCase {
     System.out.println(outStr);
     assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
     
-    // corrupt replicas 
-    for (int i=0; i < 3; i++) {
-      File blockFile = MiniDFSCluster.getBlockFile(i, block);
-      if (blockFile != null && blockFile.exists()) {
-        RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw");
-        FileChannel channel = raFile.getChannel();
-        String badString = "BADBAD";
-        int rand = random.nextInt((int)channel.size()/2);
-        raFile.seek(rand);
-        raFile.write(badString.getBytes());
-        raFile.close();
-      }
+    // corrupt replicas
+    File blockFile = MiniDFSCluster.getBlockFile(0, block);
+    if (blockFile != null && blockFile.exists()) {
+      RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw");
+      FileChannel channel = raFile.getChannel();
+      String badString = "BADBAD";
+      int rand = random.nextInt((int) channel.size()/2);
+      raFile.seek(rand);
+      raFile.write(badString.getBytes());
+      raFile.close();
     }
     // Read the file to trigger reportBadBlocks
     try {
@@ -384,7 +383,7 @@ public class TestFsck extends TestCase {
     blocks = dfsClient.getNamenode().
                getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
     replicaCount = blocks.get(0).getLocations().length;
-    while (replicaCount != 3) {
+    while (replicaCount != factor) {
       try {
         Thread.sleep(100);
       } catch (InterruptedException ignore) {

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jun  7 00:08:02 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
-/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829
+/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125145,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jun  7 00:08:02 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
-/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829
+/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125145,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jun  7 00:08:02 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
-/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829
+/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125145,1125189,1127712,1127759,1128534,1129942,1130368,1130855,1130870,1132829



Mime
View raw message