hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1126941 [1/3] - in /hadoop/hdfs/branches/yahoo-merge: ./ src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/ src/java/org/apache/hadoop/hdfs/server/common/ src/java/org/apache/hadoop/hdfs/server/datanode/ src/java/org/apache/hadoop/hdfs/serv...
Date Tue, 24 May 2011 09:03:15 GMT
Author: suresh
Date: Tue May 24 09:03:13 2011
New Revision: 1126941

URL: http://svn.apache.org/viewvc?rev=1126941&view=rev
Log:
Merging change r1079607 for HDFS-1557 from HDFS-1052 branch

Added:
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
      - copied unchanged from r1079607, hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
      - copied unchanged from r1079607, hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
Removed:
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
Modified:
    hadoop/hdfs/branches/yahoo-merge/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
    hadoop/hdfs/branches/yahoo-merge/build.xml   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
  (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/UpgradeManagerNamenode.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestParallelImageWrite.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
    hadoop/hdfs/branches/yahoo-merge/src/webapps/datanode/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/secondary/   (props changed)

Propchange: hadoop/hdfs/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 09:03:13 2011
@@ -1,5 +1,5 @@
 /hadoop/core/branches/branch-0.19/hdfs:713112
-/hadoop/hdfs/branches/HDFS-1052:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265:796829-820463
 /hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1036738,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1036738,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Modified: hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/CHANGES.txt?rev=1126941&r1=1126940&r2=1126941&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/hdfs/branches/yahoo-merge/CHANGES.txt Tue May 24 09:03:13 2011
@@ -258,6 +258,9 @@ Trunk (unreleased changes)
     HDFS-1914. Federation: namenode storage directories must be configurable
     specific to name service. (suresh)
 
+    HDFS-1735. Federation: merge FSImage change in federation to
+    FSImage+NNStorage refactoring in trunk (suresh)
+
   IMPROVEMENTS
 
     HDFS-1510. Added test-patch.properties required by test-patch.sh (nigel)
@@ -329,6 +332,8 @@ Trunk (unreleased changes)
 
     HDFS-1506. Refactor fsimage loading code. (hairong)
 
+    HDFS-1557. Separate Storage from FSImage. (Ivan Kelly via jitendra)
+
   OPTIMIZATIONS
 
     HDFS-1458. Improve checkpoint performance by avoiding unnecessary image

Propchange: hadoop/hdfs/branches/yahoo-merge/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 09:03:13 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/build.xml:713112
 /hadoop/core/trunk/build.xml:779102
-/hadoop/hdfs/branches/HDFS-1052/build.xml:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052/build.xml:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
 /hadoop/hdfs/branches/branch-0.21/build.xml:820487
-/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 09:03:13 2011
@@ -1,4 +1,4 @@
 /hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
 /hadoop/core/trunk/src/c++/libhdfs:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
-/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 09:03:13 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/contrib/hdfsproxy:713112
 /hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 09:03:13 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/java:713112
 /hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
-/hadoop/hdfs/branches/HDFS-1052/src/java:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052/src/java:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1126941&r1=1126940&r2=1126941&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
Tue May 24 09:03:13 2011
@@ -794,7 +794,13 @@ public abstract class Storage extends St
                             + from.getCanonicalPath() + " to " + to.getCanonicalPath());
   }
 
-  protected static void deleteDir(File dir) throws IOException {
+  /**
+   * Recursively delete all the content of the directory first and then 
+   * the directory itself from the local filesystem.
+   * @param dir The directory to delete
+   * @throws IOException
+   */
+  public static void deleteDir(File dir) throws IOException {
     if (!FileUtil.fullyDelete(dir))
       throw new IOException("Failed to delete " + dir.getCanonicalPath());
   }

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 09:03:13 2011
@@ -2,7 +2,7 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:713112
 /hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DatanodeBlockInfo.java:776175-785643,785929-786278
 /hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
-/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java?rev=1126941&r1=1126940&r2=1126941&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
Tue May 24 09:03:13 2011
@@ -128,7 +128,7 @@ public class BackupNode extends NameNode
 
   @Override // NameNode
   protected void loadNamesystem(Configuration conf) throws IOException {
-    BackupStorage bnImage = new BackupStorage();
+    BackupImage bnImage = new BackupImage();
     this.namesystem = new FSNamesystem(conf, bnImage);
     bnImage.recoverCreateRead(FSNamesystem.getNamespaceDirs(conf),
                               FSNamesystem.getNamespaceEditsDirs(conf));
@@ -224,7 +224,7 @@ public class BackupNode extends NameNode
     if(!nnRpcAddress.equals(nnReg.getAddress()))
       throw new IOException("Journal request from unexpected name-node: "
           + nnReg.getAddress() + " expecting " + nnRpcAddress);
-    BackupStorage bnImage = (BackupStorage)getFSImage();
+    BackupImage bnImage = (BackupImage)getFSImage();
     switch(jAction) {
       case (int)JA_IS_ALIVE:
         return;
@@ -246,8 +246,8 @@ public class BackupNode extends NameNode
   boolean shouldCheckpointAtStartup() {
     FSImage fsImage = getFSImage();
     if(isRole(NamenodeRole.CHECKPOINT)) {
-      assert fsImage.getNumStorageDirs() > 0;
-      return ! fsImage.getStorageDir(0).getVersionFile().exists();
+      assert fsImage.getStorage().getNumStorageDirs() > 0;
+      return ! fsImage.getStorage().getStorageDir(0).getVersionFile().exists();
     }
     if(namesystem == null || namesystem.dir == null || getFSImage() == null)
       return true;
@@ -309,14 +309,14 @@ public class BackupNode extends NameNode
    * @throws IOException
    */
   private void registerWith(NamespaceInfo nsInfo) throws IOException {
-    BackupStorage bnImage = (BackupStorage)getFSImage();
+    BackupImage bnImage = (BackupImage)getFSImage();
     // verify namespaceID
-    if(bnImage.getNamespaceID() == 0) // new backup storage
-      bnImage.setStorageInfo(nsInfo);
-    else if(bnImage.getNamespaceID() != nsInfo.getNamespaceID())
+    if(bnImage.getStorage().getNamespaceID() == 0) // new backup storage
+      bnImage.getStorage().setStorageInfo(nsInfo);
+    else if(bnImage.getStorage().getNamespaceID() != nsInfo.getNamespaceID())
       throw new IOException("Incompatible namespaceIDs"
           + ": active node namespaceID = " + nsInfo.getNamespaceID() 
-          + "; backup node namespaceID = " + bnImage.getNamespaceID());
+          + "; backup node namespaceID = " + bnImage.getStorage().getNamespaceID());
 
     setRegistration();
     NamenodeRegistration nnReg = null;
@@ -351,7 +351,7 @@ public class BackupNode extends NameNode
    * @throws IOException
    */
   void resetNamespace() throws IOException {
-    ((BackupStorage)getFSImage()).reset();
+    ((BackupImage)getFSImage()).reset();
   }
 
   /**

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java?rev=1126941&r1=1126940&r2=1126941&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
Tue May 24 09:03:13 2011
@@ -43,12 +43,12 @@ public class CheckpointSignature extends
   public CheckpointSignature() {}
 
   CheckpointSignature(FSImage fsImage) {
-    super(fsImage);
+    super(fsImage.getStorage());
     blockpoolID = fsImage.getBlockPoolID();
     editsTime = fsImage.getEditLog().getFsEditTime();
-    checkpointTime = fsImage.getCheckpointTime();
-    imageDigest = fsImage.imageDigest;
-    checkpointTime = fsImage.getCheckpointTime();
+    checkpointTime = fsImage.getStorage().getCheckpointTime();
+    imageDigest = fsImage.getStorage().getImageDigest();
+    checkpointTime = fsImage.getStorage().getCheckpointTime();
   }
 
   CheckpointSignature(String str) {
@@ -109,12 +109,13 @@ public class CheckpointSignature extends
   }
 
   void validateStorageInfo(FSImage si) throws IOException {
-    if(layoutVersion != si.layoutVersion
-        || namespaceID != si.namespaceID || cTime != si.cTime
-        || checkpointTime != si.checkpointTime 
-        || !imageDigest.equals(si.imageDigest)
-        || !clusterID.equals(si.clusterID)
-        || !blockpoolID.equals(si.blockpoolID)) {
+    if(layoutVersion != si.getLayoutVersion()
+        || namespaceID != si.getNamespaceID() 
+        || cTime != si.getStorage().cTime
+        || checkpointTime != si.getStorage().getCheckpointTime() 
+        || !imageDigest.equals(si.getStorage().imageDigest)
+        || !clusterID.equals(si.getClusterID())
+        || !blockpoolID.equals(si.getBlockPoolID())) {
       // checkpointTime can change when the image is saved - do not compare
       throw new IOException("Inconsistent checkpoint fields.\n"
           + "LV = " + layoutVersion + " namespaceID = " + namespaceID
@@ -123,10 +124,12 @@ public class CheckpointSignature extends
           + " ; clusterId = " + clusterID
           + " ; blockpoolId = " + blockpoolID
           + ".\nExpecting respectively: "
-          + si.layoutVersion + "; " + si.namespaceID + "; " + si.cTime
-          + "; " + si.checkpointTime + "; " + si.imageDigest
-          + "; " + si.clusterID + "; " + si.blockpoolID + "."
-          );
+          + si.getLayoutVersion() + "; " 
+          + si.getNamespaceID() + "; " + si.getStorage().cTime
+          + "; " + si.getStorage().getCheckpointTime() + "; " 
+          + si.getStorage().imageDigest
+          + "; " + si.getClusterID() + "; " 
+          + si.getBlockPoolID() + ".");
     }
   }
 

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java?rev=1126941&r1=1126940&r2=1126941&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
Tue May 24 09:03:13 2011
@@ -29,8 +29,8 @@ import org.apache.hadoop.hdfs.protocol.F
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole;
 import static org.apache.hadoop.hdfs.server.common.Util.now;
 import org.apache.hadoop.hdfs.server.namenode.FSImage.CheckpointStates;
-import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeDirType;
-import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeFile;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
 import org.apache.hadoop.hdfs.server.protocol.CheckpointCommand;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
@@ -63,8 +63,8 @@ class Checkpointer extends Daemon {
 
   private String infoBindAddress;
 
-  private BackupStorage getFSImage() {
-    return (BackupStorage)backupNode.getFSImage();
+  private BackupImage getFSImage() {
+    return (BackupImage)backupNode.getFSImage();
   }
 
   private NamenodeProtocol getNamenode(){
@@ -182,8 +182,8 @@ class Checkpointer extends Daemon {
   private void downloadCheckpoint(CheckpointSignature sig) throws IOException {
     // Retrieve image file
     String fileid = "getimage=1";
-    Collection<File> list = getFSImage().getFiles(NameNodeFile.IMAGE,
-        NameNodeDirType.IMAGE);
+    Collection<File> list = getFSImage()
+      .getStorage().getFiles(NameNodeFile.IMAGE, NameNodeDirType.IMAGE);
     File[] files = list.toArray(new File[list.size()]);
     assert files.length > 0 : "No checkpoint targets.";
     String nnHttpAddr = backupNode.nnHttpAddress;
@@ -193,7 +193,8 @@ class Checkpointer extends Daemon {
 
     // Retrieve edits file
     fileid = "getedit=1";
-    list = getFSImage().getFiles(NameNodeFile.EDITS, NameNodeDirType.EDITS);
+    list = getFSImage()
+      .getStorage().getFiles(NameNodeFile.EDITS, NameNodeDirType.EDITS);
     files = list.toArray(new File[list.size()]);
     assert files.length > 0 : "No checkpoint targets.";
     TransferFsImage.getFileClient(nnHttpAddr, fileid, files, false);
@@ -211,7 +212,7 @@ class Checkpointer extends Daemon {
     String fileid = "putimage=1&port=" + httpPort +
       "&machine=" + infoBindAddress +
       "&token=" + sig.toString() +
-      "&newChecksum=" + getFSImage().imageDigest.toString();
+      "&newChecksum=" + getFSImage().getStorage().getImageDigest().toString();
     LOG.info("Posted URL " + backupNode.nnHttpAddress + fileid);
     TransferFsImage.getFileClient(backupNode.nnHttpAddress, 
         fileid, (File[])null, false);
@@ -250,9 +251,9 @@ class Checkpointer extends Daemon {
       downloadCheckpoint(sig);
     }
 
-    BackupStorage bnImage = getFSImage();
-    bnImage.blockpoolID = backupNode.getBlockPoolId();
-    bnImage.clusterID = backupNode.getClusterId();
+    BackupImage bnImage = getFSImage();
+    bnImage.getStorage().setClusterID(backupNode.getBlockPoolId());
+    bnImage.getStorage().setClusterID(backupNode.getClusterId());
     bnImage.loadCheckpoint(sig);
     sig.validateStorageInfo(bnImage);
     bnImage.saveCheckpoint();
@@ -268,6 +269,6 @@ class Checkpointer extends Daemon {
         getFSImage().getEditLog().close();
     LOG.info("Checkpoint completed in "
         + (now() - startTime)/1000 + " seconds."
-        + " New Image Size: " + bnImage.getFsImageName().length());
+        + " New Image Size: " + bnImage.getStorage().getFsImageName().length());
   }
 }

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1126941&r1=1126940&r2=1126941&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Tue May 24 09:03:13 2011
@@ -153,8 +153,8 @@ class FSDirectory implements Closeable {
       throws IOException {
     // format before starting up if requested
     if (startOpt == StartupOption.FORMAT) {
-      fsImage.setStorageDirectories(dataDirs, editsDirs);
-      fsImage.format(fsImage.determineClusterId()); // reuse current id
+      fsImage.getStorage().setStorageDirectories(dataDirs, editsDirs);
+      fsImage.getStorage().format(fsImage.getStorage().determineClusterId()); // reuse current
id
       startOpt = StartupOption.REGULAR;
     }
     try {

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1126941&r1=1126940&r2=1126941&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Tue May 24 09:03:13 2011
@@ -36,11 +36,11 @@ import org.apache.hadoop.hdfs.server.com
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import static org.apache.hadoop.hdfs.server.common.Util.now;
-import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeDirType;
-import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeFile;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage.NNStorageListener;
 import org.apache.hadoop.hdfs.server.namenode.JournalStream.JournalType;
 import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
-import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
 import org.apache.hadoop.io.ArrayWritable;
 import org.apache.hadoop.io.BytesWritable;
@@ -56,7 +56,7 @@ import static org.apache.hadoop.hdfs.ser
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class FSEditLog {
+public class FSEditLog implements NNStorageListener {
 
   static final String NO_JOURNAL_STREAMS_WARNING = "!!! WARNING !!!" +
       " File system changes are not persistent. No journal streams.";
@@ -66,7 +66,6 @@ public class FSEditLog {
   private volatile int sizeOutputFlushBuffer = 512*1024;
 
   private ArrayList<EditLogOutputStream> editStreams = null;
-  private FSImage fsimage = null;
 
   // a monotonically increasing counter that represents transactionIds.
   private long txid = 0;
@@ -89,6 +88,8 @@ public class FSEditLog {
   private long totalTimeTransactions;  // total time for all transactions
   private NameNodeMetrics metrics;
 
+  private NNStorage storage;
+
   private static class TransactionId {
     public long txid;
 
@@ -104,23 +105,24 @@ public class FSEditLog {
     }
   };
 
-  FSEditLog(FSImage image) {
-    fsimage = image;
+  FSEditLog(NNStorage storage) {
     isSyncRunning = false;
+    this.storage = storage;
+    this.storage.registerListener(this);
     metrics = NameNode.getNameNodeMetrics();
     lastPrintTime = now();
   }
   
   private File getEditFile(StorageDirectory sd) {
-    return fsimage.getEditFile(sd);
+    return storage.getEditFile(sd);
   }
   
   private File getEditNewFile(StorageDirectory sd) {
-    return fsimage.getEditNewFile(sd);
+    return storage.getEditNewFile(sd);
   }
   
   private int getNumEditsDirs() {
-   return fsimage.getNumStorageDirs(NameNodeDirType.EDITS);
+   return storage.getNumStorageDirs(NameNodeDirType.EDITS);
   }
 
   synchronized int getNumEditStreams() {
@@ -151,8 +153,8 @@ public class FSEditLog {
       editStreams = new ArrayList<EditLogOutputStream>();
     
     ArrayList<StorageDirectory> al = null;
-    for (Iterator<StorageDirectory> it = 
-           fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
+    for (Iterator<StorageDirectory> it 
+         = storage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
       StorageDirectory sd = it.next();
       File eFile = getEditFile(sd);
       try {
@@ -166,7 +168,7 @@ public class FSEditLog {
       }
     }
     
-    if(al != null) fsimage.processIOError(al, false);
+    if(al != null) storage.reportErrorsOnDirectories(al);
   }
   
   
@@ -210,7 +212,7 @@ public class FSEditLog {
         errorStreams.add(eStream);
       }
     }
-    processIOError(errorStreams, true);
+    disableAndReportErrorOnStreams(errorStreams);
     editStreams.clear();
   }
 
@@ -228,64 +230,26 @@ public class FSEditLog {
 
   /**
    * The specified streams have IO errors. Close and remove them.
-   * If propagate is true - close related StorageDirectories.
-   * (is called with propagate value true from everywhere
-   *  except fsimage.processIOError)
-   */
-  synchronized void processIOError(
-      List<EditLogOutputStream> errorStreams,
-      boolean propagate) {
-    
+   */
+  synchronized
+  void disableAndReportErrorOnStreams(List<EditLogOutputStream> errorStreams) {
     if (errorStreams == null || errorStreams.size() == 0) {
       return;                       // nothing to do
     }
-
-    String lsd = fsimage.listStorageDirectories();
-    LOG.info("current list of storage dirs:" + lsd);
-
-    ArrayList<StorageDirectory> al = null;
-    for (EditLogOutputStream eStream : errorStreams) {
-      LOG.error("Unable to log edits to " + eStream.getName()
-          + "; removing it");     
-
-      StorageDirectory storageDir;
-      if(propagate && eStream.getType() == JournalType.FILE && //find SD
-          (storageDir = getStorage(eStream)) != null) {
-        LOG.info("about to remove corresponding storage:" 
-            + storageDir.getRoot().getAbsolutePath());
-        // remove corresponding storage dir
-        if(al == null) al = new ArrayList<StorageDirectory>(1);
-        al.add(storageDir);
+    ArrayList<StorageDirectory> errorDirs = new ArrayList<StorageDirectory>();
+    for (EditLogOutputStream e : errorStreams) {
+      if (e.getType() == JournalType.FILE) {
+        errorDirs.add(getStorageDirectoryForStream(e));
+      } else {
+        disableStream(e);
       }
-      Iterator<EditLogOutputStream> ies = editStreams.iterator();
-      while (ies.hasNext()) {
-        EditLogOutputStream es = ies.next();
-        if (es == eStream) {  
-          try { eStream.close(); } catch (IOException e) {
-            // nothing to do.
-            LOG.warn("Failed to close eStream " + eStream.getName()
-                + " before removing it (might be ok)");
-          }
-          ies.remove();
-          break;
-        }
-      } 
-    }
-    
-    if (editStreams == null || editStreams.size() <= 0) {
-      String msg = "Fatal Error: All storage directories are inaccessible.";
-      LOG.fatal(msg, new IOException(msg)); 
-      Runtime.getRuntime().exit(-1);
     }
 
-    // removed failed SDs
-    if(propagate && al != null) fsimage.processIOError(al, false);
-    
-    //for the rest of the streams
-    if(propagate) incrementCheckpointTime();
-    
-    lsd = fsimage.listStorageDirectories();
-    LOG.info("at the end current list of storage dirs:" + lsd);
+    try {
+      storage.reportErrorsOnDirectories(errorDirs);
+    } catch (IOException ioe) {
+      LOG.error("Problem erroring streams " + ioe);
+    }
   }
 
 
@@ -298,7 +262,7 @@ public class FSEditLog {
     String parentStorageDir = ((EditLogFileOutputStream)es).getFile()
     .getParentFile().getParentFile().getAbsolutePath();
 
-    Iterator<StorageDirectory> it = fsimage.dirIterator(); 
+    Iterator<StorageDirectory> it = storage.dirIterator(); 
     while (it.hasNext()) {
       StorageDirectory sd = it.next();
       LOG.info("comparing: " + parentStorageDir + " and " + sd.getRoot().getAbsolutePath());

@@ -355,7 +319,7 @@ public class FSEditLog {
           errorStreams.add(eStream);
         }
       }
-      processIOError(errorStreams, true);
+      disableAndReportErrorOnStreams(errorStreams);
       recordTransaction(start);
       
       // check if it is time to schedule an automatic sync
@@ -547,7 +511,7 @@ public class FSEditLog {
         }
       }
       long elapsed = now() - start;
-      processIOError(errorStreams, true);
+      disableAndReportErrorOnStreams(errorStreams);
   
       if (metrics != null) // Metrics non-null only when used inside name node
         metrics.addSync(elapsed);
@@ -814,7 +778,7 @@ public class FSEditLog {
         al.add(es);
       }
     }
-    if(al!=null) processIOError(al, true);
+    if(al!=null) disableAndReportErrorOnStreams(al);
     return size;
   }
   
@@ -823,7 +787,7 @@ public class FSEditLog {
    */
   synchronized void rollEditLog() throws IOException {
     waitForSyncToFinish();
-    Iterator<StorageDirectory> it = fsimage.dirIterator(NameNodeDirType.EDITS);
+    Iterator<StorageDirectory> it = storage.dirIterator(NameNodeDirType.EDITS);
     if(!it.hasNext()) 
       return;
     //
@@ -841,7 +805,7 @@ public class FSEditLog {
       return; // nothing to do, edits.new exists!
 
     // check if any of failed storage is now available and put it back
-    fsimage.attemptRestoreRemovedStorage(false);
+    storage.attemptRestoreRemovedStorage(false);
 
     divertFileStreams(
         Storage.STORAGE_DIR_CURRENT + "/" + NameNodeFile.EDITS_NEW.getName());
@@ -863,7 +827,7 @@ public class FSEditLog {
     EditStreamIterator itE = 
       (EditStreamIterator)getOutputStreamIterator(JournalType.FILE);
     Iterator<StorageDirectory> itD = 
-      fsimage.dirIterator(NameNodeDirType.EDITS);
+      storage.dirIterator(NameNodeDirType.EDITS);
     while(itE.hasNext() && itD.hasNext()) {
       EditLogOutputStream eStream = itE.next();
       StorageDirectory sd = itD.next();
@@ -885,7 +849,7 @@ public class FSEditLog {
         errorStreams.add(eStream);
       }
     }
-    processIOError(errorStreams, true);
+    disableAndReportErrorOnStreams(errorStreams);
   }
 
   /**
@@ -929,7 +893,7 @@ public class FSEditLog {
     EditStreamIterator itE = 
       (EditStreamIterator)getOutputStreamIterator(JournalType.FILE);
     Iterator<StorageDirectory> itD = 
-      fsimage.dirIterator(NameNodeDirType.EDITS);
+      storage.dirIterator(NameNodeDirType.EDITS);
     while(itE.hasNext() && itD.hasNext()) {
       EditLogOutputStream eStream = itE.next();
       StorageDirectory sd = itD.next();
@@ -964,7 +928,7 @@ public class FSEditLog {
         errorStreams.add(eStream);
       }
     }
-    processIOError(errorStreams, true);
+    disableAndReportErrorOnStreams(errorStreams);
   }
 
   /**
@@ -973,7 +937,7 @@ public class FSEditLog {
   synchronized File getFsEditName() {
     StorageDirectory sd = null;   
     for (Iterator<StorageDirectory> it = 
-      fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
+      storage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
       sd = it.next();   
       if(sd.getRoot().canRead())
         return getEditFile(sd);
@@ -985,7 +949,7 @@ public class FSEditLog {
    * Returns the timestamp of the edit log
    */
   synchronized long getFsEditTime() {
-    Iterator<StorageDirectory> it = fsimage.dirIterator(NameNodeDirType.EDITS);
+    Iterator<StorageDirectory> it = storage.dirIterator(NameNodeDirType.EDITS);
     if(it.hasNext())
       return getEditFile(it.next()).lastModified();
     return 0;
@@ -1052,7 +1016,7 @@ public class FSEditLog {
         errorStreams.add(eStream);
       }
     }
-    processIOError(errorStreams, true);
+    disableAndReportErrorOnStreams(errorStreams);
     recordTransaction(start);
   }
 
@@ -1126,8 +1090,8 @@ public class FSEditLog {
   }
 
   void incrementCheckpointTime() {
-    fsimage.incrementCheckpointTime();
-    Writable[] args = {new LongWritable(fsimage.getCheckpointTime())};
+    storage.incrementCheckpointTime();
+    Writable[] args = {new LongWritable(storage.getCheckpointTime())};
     logEdit(OP_CHECKPOINT_TIME, args);
   }
 
@@ -1148,7 +1112,7 @@ public class FSEditLog {
     }
     assert backupNode == null || backupNode.isRole(NamenodeRole.BACKUP) :
       "Not a backup node corresponds to a backup stream";
-    processIOError(errorStreams, true);
+    disableAndReportErrorOnStreams(errorStreams);
   }
 
   synchronized boolean checkBackupRegistration(
@@ -1175,7 +1139,7 @@ public class FSEditLog {
     }
     assert backupNode == null || backupNode.isRole(NamenodeRole.BACKUP) :
       "Not a backup node corresponds to a backup stream";
-    processIOError(errorStreams, true);
+    disableAndReportErrorOnStreams(errorStreams);
     return regAllowed;
   }
   
@@ -1186,4 +1150,80 @@ public class FSEditLog {
     }
     return new BytesWritable(bytes);
   }
+
+  /**
+   * Get the StorageDirectory for a stream
+   * @param es Stream whose StorageDirectory we wish to know
+   * @return the matching StorageDirectory
+   */
+  StorageDirectory getStorageDirectoryForStream(EditLogOutputStream es) {
+    String parentStorageDir = ((EditLogFileOutputStream)es).getFile().getParentFile().getParentFile().getAbsolutePath();
+
+    for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext();) {
+      StorageDirectory sd = it.next();
+      FSNamesystem.LOG.info("comparing: " + parentStorageDir 
+                            + " and " + sd.getRoot().getAbsolutePath()); 
+      if (parentStorageDir.equals(sd.getRoot().getAbsolutePath()))
+        return sd;
+    }
+    return null;
+  }
+
+  private synchronized void disableStream(EditLogOutputStream stream) {
+    try { stream.close(); } catch (IOException e) {
+      // nothing to do.
+      LOG.warn("Failed to close eStream " + stream.getName()
+               + " before removing it (might be ok)");
+    }
+    editStreams.remove(stream);
+
+    if (editStreams.size() <= 0) {
+      String msg = "Fatal Error: All storage directories are inaccessible.";
+      LOG.fatal(msg, new IOException(msg));
+      Runtime.getRuntime().exit(-1);
+    }
+  }
+
+  /**
+   * Error Handling on a storageDirectory
+   *
+   */
+  // NNStorageListener Interface
+  @Override // NNStorageListener
+  public synchronized void errorOccurred(StorageDirectory sd)
+      throws IOException {
+    ArrayList<EditLogOutputStream> errorStreams
+      = new ArrayList<EditLogOutputStream>();
+
+    for (EditLogOutputStream eStream : editStreams) {
+      LOG.error("Unable to log edits to " + eStream.getName()
+                + "; removing it");
+
+      StorageDirectory streamStorageDir = getStorageDirectoryForStream(eStream);
+      if (sd == streamStorageDir) {
+        errorStreams.add(eStream);
+      }
+    }
+
+    for (EditLogOutputStream eStream : errorStreams) {
+      disableStream(eStream);
+    }
+  }
+
+  @Override // NNStorageListener
+  public synchronized void formatOccurred(StorageDirectory sd)
+      throws IOException {
+    if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) {
+      createEditLogFile(NNStorage.getStorageFile(sd, NameNodeFile.EDITS));
+    }
+  };
+
+  @Override // NNStorageListener
+  public synchronized void directoryAvailable(StorageDirectory sd)
+      throws IOException {
+    if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) {
+      File eFile = getEditFile(sd);
+      addNewEditLogStream(eFile);
+    }
+  }
 }



Mime
View raw message