hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1126890 - in /hadoop/hdfs/branches/yahoo-merge: ./ src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/ src/java/org/apache/hadoop/hdfs/server/datanode/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/ src/test/hdfs/org/apache/...
Date Tue, 24 May 2011 06:31:05 GMT
Author: suresh
Date: Tue May 24 06:31:05 2011
New Revision: 1126890

URL: http://svn.apache.org/viewvc?rev=1126890&view=rev
Log:
Merging r1053214 for HDFS-1509 from trunk

Modified:
    hadoop/hdfs/branches/yahoo-merge/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
    hadoop/hdfs/branches/yahoo-merge/build.xml   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
  (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
    hadoop/hdfs/branches/yahoo-merge/src/webapps/datanode/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/secondary/   (props changed)

Propchange: hadoop/hdfs/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 06:31:05 2011
@@ -2,4 +2,4 @@
 /hadoop/hdfs/branches/HDFS-1052:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265:796829-820463
 /hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036738,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036738,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Modified: hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/CHANGES.txt?rev=1126890&r1=1126889&r2=1126890&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/hdfs/branches/yahoo-merge/CHANGES.txt Tue May 24 06:31:05 2011
@@ -426,6 +426,9 @@ Trunk (unreleased changes)
 
     HDFS-1503. TestSaveNamespace fails. (Todd Lipcon via cos)
 
+    HDFS-1509. A savenamespace command writes the fsimage and edits into
+    all configured directories. (dhruba)
+
 Release 0.22.0 - Unreleased
 
   NEW FEATURES

Propchange: hadoop/hdfs/branches/yahoo-merge/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 06:31:05 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/build.xml:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
 /hadoop/hdfs/branches/branch-0.21/build.xml:820487
-/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 06:31:05 2011
@@ -1,4 +1,4 @@
 /hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
 /hadoop/core/trunk/src/c++/libhdfs:776175-784663
 /hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
-/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 06:31:05 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 06:31:05 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/java:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 06:31:05 2011
@@ -5,4 +5,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1126890&r1=1126889&r2=1126890&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Tue May 24 06:31:05 2011
@@ -841,8 +841,6 @@ public class FSEditLog {
       return; // nothing to do, edits.new exists!
 
     // check if any of failed storage is now available and put it back
-    // but don't save the current image to it 
-    // (it will be put there as a part of checkpointing)
     fsimage.attemptRestoreRemovedStorage(false);
 
     divertFileStreams(

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1126890&r1=1126889&r2=1126890&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Tue May 24 06:31:05 2011
@@ -888,7 +888,7 @@ public class FSImage extends Storage {
               sd1.unlock(); // unlock before removing (in case it will be
                             // restored)
             } catch (Exception e) {
-              // nothing
+              LOG.info("Unable to unlock bad storage directory : " +  sd.getRoot().getPath());
             }
             removedStorageDirs.add(sd1);
             it.remove();
@@ -1241,7 +1241,11 @@ public class FSImage extends Storage {
    * in which case the journal will be lost.
    */
   void saveNamespace(boolean renewCheckpointTime) throws IOException {
+ 
+    // try to restore all failed edit logs here
     assert editLog != null : "editLog must be initialized";
+    attemptRestoreRemovedStorage(true);
+
     editLog.close();
     if(renewCheckpointTime)
       this.checkpointTime = now();
@@ -1285,6 +1289,11 @@ public class FSImage extends Storage {
     for (Iterator<StorageDirectory> it = dirIterator(NameNodeDirType.EDITS);
                                                               it.hasNext();) {
       final StorageDirectory sd = it.next();
+      // if this directory already stores the image and edits, then it was
+      // already processed in the earlier loop.
+      if (sd.getStorageDirType() == NameNodeDirType.IMAGE_AND_EDITS) {
+        continue;
+      }
       FSImageSaver saver = new FSImageSaver(sd, errorSDs);
       Thread saveThread = new Thread(saver, saver.toString());
       saveThreads.add(saveThread);
@@ -1730,10 +1739,11 @@ public class FSImage extends Storage {
   }
 
   /**
-   * See if any of removed storages iw "writable" again, and can be returned 
-   * into service
+   * See if any of removed storages is "writable" again, and can be returned 
+   * into service. If saveNamespace is set, then this methdod is being 
+   * called form saveNamespace.
    */
-  synchronized void attemptRestoreRemovedStorage(boolean saveCurrentImage) {   
+  synchronized void attemptRestoreRemovedStorage(boolean saveNamespace) {   
     // if directory is "alive" - copy the images there...
     if(!restoreFailedStorage || removedStorageDirs.size() == 0) 
       return; //nothing to restore
@@ -1748,7 +1758,15 @@ public class FSImage extends Storage {
       try {
         
         if(root.exists() && root.canWrite()) { 
-          format(sd, saveCurrentImage);
+          /** If this call is being made from savenamespace command, then no
+           * need to format, the savenamespace command will format and write
+           * the new image to this directory anyways.
+           */
+          if (saveNamespace) {
+            sd.clearDirectory();
+          } else {
+            format(sd);
+          }
           LOG.info("restoring dir " + sd.getRoot().getAbsolutePath());
           if(sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) {
             File eFile = getEditFile(sd);

Propchange: hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 06:31:05 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
-/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Modified: hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java?rev=1126890&r1=1126889&r2=1126890&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
Tue May 24 06:31:05 2011
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.na
 
 import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.anyObject;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doThrow;
@@ -60,6 +61,17 @@ public class TestSaveNamespace {
 
   private static class FaultySaveImage implements Answer<Void> {
     int count = 0;
+    boolean exceptionType = true;
+
+    // generate a RuntimeException
+    public FaultySaveImage() {
+      this.exceptionType = true;
+    }
+
+    // generate either a RuntimeException or IOException
+    public FaultySaveImage(boolean etype) {
+      this.exceptionType = etype;
+    }
 
     public Void answer(InvocationOnMock invocation) throws Throwable {
       Object[] args = invocation.getArguments();
@@ -67,7 +79,11 @@ public class TestSaveNamespace {
 
       if (count++ == 1) {
         LOG.info("Injecting fault for file: " + f);
-        throw new RuntimeException("Injected fault: saveFSImage second time");
+        if (exceptionType) {
+          throw new RuntimeException("Injected fault: saveFSImage second time");
+        } else {
+          throw new IOException("Injected fault: saveFSImage second time");
+        }
       }
       LOG.info("Not injecting fault for file: " + f);
       return (Void)invocation.callRealMethod();
@@ -142,6 +158,82 @@ public class TestSaveNamespace {
     }
   }
 
+  /**
+   * Verify that a saveNamespace command brings faulty directories
+   * in fs.name.dir and fs.edit.dir back online.
+   */
+  @Test
+  public void testReinsertnamedirsInSavenamespace() throws Exception {
+    // create a configuration with the key to restore error
+    // directories in fs.name.dir
+    Configuration conf = getConf();
+    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_KEY, true);
+
+    NameNode.initMetrics(conf, NamenodeRole.ACTIVE);
+    GenericTestUtils.formatNamenode(conf);
+    FSNamesystem fsn = new FSNamesystem(conf);
+
+    // Replace the FSImage with a spy
+    FSImage originalImage = fsn.dir.fsImage;
+    FSImage spyImage = spy(originalImage);
+    spyImage.setStorageDirectories(
+        FSNamesystem.getNamespaceDirs(conf), 
+        FSNamesystem.getNamespaceEditsDirs(conf));
+    fsn.dir.fsImage = spyImage;
+
+    // inject fault
+    // The spy throws a IOException when writing to the second directory
+    doAnswer(new FaultySaveImage(false)).
+      when(spyImage).saveFSImage((File)anyObject());
+
+    try {
+      doAnEdit(fsn, 1);
+      fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+
+      // Save namespace - this  injects a fault and marks one
+      // directory as faulty.
+      LOG.info("Doing the first savenamespace.");
+      fsn.saveNamespace();
+      LOG.warn("First savenamespace sucessful.");
+      assertTrue("Savenamespace should have marked one directory as bad." +
+                 " But found " + spyImage.getRemovedStorageDirs().size() +
+                 " bad directories.", 
+                   spyImage.getRemovedStorageDirs().size() == 1);
+
+      // The next call to savenamespace should try inserting the
+      // erroneous directory back to fs.name.dir. This command should
+      // be successful.
+      LOG.info("Doing the second savenamespace.");
+      fsn.saveNamespace();
+      LOG.warn("Second savenamespace sucessful.");
+      assertTrue("Savenamespace should have been successful in removing " +
+                 " bad directories from Image."  +
+                 " But found " + originalImage.getRemovedStorageDirs().size() +
+                 " bad directories.", 
+                 originalImage.getRemovedStorageDirs().size() == 0);
+
+      // Now shut down and restart the namesystem
+      LOG.info("Shutting down fsimage.");
+      originalImage.close();
+      fsn.close();      
+      fsn = null;
+
+      // Start a new namesystem, which should be able to recover
+      // the namespace from the previous incarnation.
+      LOG.info("Loading new FSmage from disk.");
+      fsn = new FSNamesystem(conf);
+
+      // Make sure the image loaded including our edit.
+      LOG.info("Checking reloaded image.");
+      checkEditExists(fsn, 1);
+      LOG.info("Reloaded image is good.");
+    } finally {
+      if (fsn != null) {
+        fsn.close();
+      }
+    }
+  }
+
   @Test
   public void testCrashWhileSavingSecondImage() throws Exception {
     saveNamespaceWithInjectedFault(Fault.SAVE_FSIMAGE);

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 06:31:05 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
-/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 06:31:05 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
-/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 06:31:05 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
-/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576



Mime
View raw message