hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject svn commit: r1134157 - in /hadoop/hdfs/branches/yahoo-merge: ./ CHANGES.txt src/java/ src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
Date Fri, 10 Jun 2011 00:55:32 GMT
Author: mattf
Date: Fri Jun 10 00:55:32 2011
New Revision: 1134157

URL: http://svn.apache.org/viewvc?rev=1134157&view=rev
Log:
HDFS-900. Merge to yahoo-merge.

Modified:
    hadoop/hdfs/branches/yahoo-merge/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
    hadoop/hdfs/branches/yahoo-merge/src/java/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java

Propchange: hadoop/hdfs/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jun 10 00:55:32 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265:796829-820463
 /hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1036738,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125028,1125145,1125189,1127712,1127759,1128534,1129942,1130262,1130339,1130367-1130368,1130381,1130667,1130855,1130870,1131124,1131264,1131331,1132698,1132829,1133181,1134138
+/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1036738,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067079,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125028,1125145,1125189,1127712,1127759,1128534,1129942,1130262,1130339,1130367-1130368,1130381,1130667,1130855,1130870,1131124,1131264,1131331,1132698,1132829,1133181,1134138

Modified: hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/CHANGES.txt?rev=1134157&r1=1134156&r2=1134157&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/hdfs/branches/yahoo-merge/CHANGES.txt Fri Jun 10 00:55:32 2011
@@ -887,6 +887,8 @@ Release 0.22.0 - Unreleased
 
     HDFS-1503. TestSaveNamespace fails. (Todd Lipcon via cos)
 
+    HDFS-900. Corrupt replicas are not processed correctly in block report (shv)
+
 Release 0.21.1 - Unreleased
 
     HDFS-1411. Correct backup node startup command in hdfs user guide.

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jun 10 00:55:32 2011
@@ -4,4 +4,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/java:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125028,1125145,1125189,1127712,1127759,1128534,1129942,1130262,1130339,1130367-1130368,1130381,1130667,1130855,1130870,1131124,1131264,1131331,1132698,1132829,1133181,1134138
+/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067079,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125028,1125145,1125189,1127712,1127759,1128534,1129942,1130262,1130339,1130367-1130368,1130381,1130667,1130855,1130870,1131124,1131264,1131331,1132698,1132829,1133181,1134138

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java?rev=1134157&r1=1134156&r2=1134157&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
Fri Jun 10 00:55:32 2011
@@ -399,8 +399,7 @@ public class BlockManager {
       blocksMap.nodeIterator(block); it.hasNext();) {
       String storageID = it.next().getStorageID();
       // filter invalidate replicas
-      Collection<Block> blocks = recentInvalidateSets.get(storageID);
-      if(blocks==null || !blocks.contains(block)) {
+      if( ! belongsToInvalidates(storageID, block)) {
         machineSet.add(storageID);
       }
     }
@@ -498,16 +497,21 @@ public class BlockManager {
                             minReplication);
   }
 
-  void removeFromInvalidates(String datanodeId, Block block) {
-    Collection<Block> v = recentInvalidateSets.get(datanodeId);
+  void removeFromInvalidates(String storageID, Block block) {
+    Collection<Block> v = recentInvalidateSets.get(storageID);
     if (v != null && v.remove(block)) {
       pendingDeletionBlocksCount--;
       if (v.isEmpty()) {
-        recentInvalidateSets.remove(datanodeId);
+        recentInvalidateSets.remove(storageID);
       }
     }
   }
 
+  boolean belongsToInvalidates(String storageID, Block block) {
+    Collection<Block> invalidateSet = recentInvalidateSets.get(storageID);
+    return invalidateSet != null && invalidateSet.contains(block);
+  }
+
   /**
    * Adds block to list of blocks which will be invalidated on specified
    * datanode

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java?rev=1134157&r1=1134156&r2=1134157&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
Fri Jun 10 00:55:32 2011
@@ -498,6 +498,13 @@ public class DatanodeDescriptor extends 
           storedBlock.getBlockUCState());
     }
 
+    // Ignore replicas already scheduled to be removed from the DN
+    if(blockManager.belongsToInvalidates(getStorageID(), block)) {
+      assert storedBlock.findDatanode(this) < 0 : "Block " + block 
+        + " in recentInvalidatesSet should not appear in DN " + this;
+      return storedBlock;
+    }
+
     // Block is on the DN
     boolean isCorrupt = false;
     switch(rState) {



Mime
View raw message