hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1126902 - in /hadoop/hdfs/branches/yahoo-merge: ./ src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/ src/java/org/apache/hadoop/hdfs/server/datanode/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/ src/webapps/datanode/ src...
Date Tue, 24 May 2011 07:25:14 GMT
Author: suresh
Date: Tue May 24 07:25:13 2011
New Revision: 1126902

URL: http://svn.apache.org/viewvc?rev=1126902&view=rev
Log:
Merge change r1038001 for HDFS-1481 from trunk

Modified:
    hadoop/hdfs/branches/yahoo-merge/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
    hadoop/hdfs/branches/yahoo-merge/build.xml   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
  (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/datanode/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/secondary/   (props changed)

Propchange: hadoop/hdfs/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:25:13 2011
@@ -2,4 +2,4 @@
 /hadoop/hdfs/branches/HDFS-1052:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265:796829-820463
 /hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1036738,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1036738,1038001,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Modified: hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/CHANGES.txt?rev=1126902&r1=1126901&r2=1126902&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/hdfs/branches/yahoo-merge/CHANGES.txt Tue May 24 07:25:13 2011
@@ -621,6 +621,8 @@ Release 0.22.0 - Unreleased
     HDFS-259. Remove intentionally corrupt 0.13 directory layout creation.
     (Todd Lipcon via eli).
 
+    HDFS-1481. NameNode should validate fsimage before rolling. (hairong)
+
   OPTIMIZATIONS
 
     HDFS-1140. Speedup INode.getPathComponents. (Dmytro Molkov via shv)

Propchange: hadoop/hdfs/branches/yahoo-merge/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:25:13 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/build.xml:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
 /hadoop/hdfs/branches/branch-0.21/build.xml:820487
-/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:25:13 2011
@@ -1,4 +1,4 @@
 /hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
 /hadoop/core/trunk/src/c++/libhdfs:776175-784663
 /hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
-/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:25:13 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:25:13 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/java:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:25:13 2011
@@ -5,4 +5,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java?rev=1126902&r1=1126901&r2=1126902&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
Tue May 24 07:25:13 2011
@@ -187,7 +187,7 @@ class Checkpointer extends Daemon {
     File[] files = list.toArray(new File[list.size()]);
     assert files.length > 0 : "No checkpoint targets.";
     String nnHttpAddr = backupNode.nnHttpAddress;
-    TransferFsImage.getFileClient(nnHttpAddr, fileid, files);
+    TransferFsImage.getFileClient(nnHttpAddr, fileid, files, false);
     LOG.info("Downloaded file " + files[0].getName() + " size " +
              files[0].length() + " bytes.");
 
@@ -196,7 +196,7 @@ class Checkpointer extends Daemon {
     list = getFSImage().getFiles(NameNodeFile.EDITS, NameNodeDirType.EDITS);
     files = list.toArray(new File[list.size()]);
     assert files.length > 0 : "No checkpoint targets.";
-    TransferFsImage.getFileClient(nnHttpAddr, fileid, files);
+    TransferFsImage.getFileClient(nnHttpAddr, fileid, files, false);
     LOG.info("Downloaded file " + files[0].getName() + " size " +
         files[0].length() + " bytes.");
   }
@@ -213,7 +213,8 @@ class Checkpointer extends Daemon {
       "&token=" + sig.toString() +
       "&newChecksum=" + getFSImage().imageDigest.toString();
     LOG.info("Posted URL " + backupNode.nnHttpAddress + fileid);
-    TransferFsImage.getFileClient(backupNode.nnHttpAddress, fileid, (File[])null);
+    TransferFsImage.getFileClient(backupNode.nnHttpAddress, 
+        fileid, (File[])null, false);
   }
 
   /**

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java?rev=1126902&r1=1126901&r2=1126902&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
Tue May 24 07:25:13 2011
@@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.server.common.JspHelper;
 import org.apache.hadoop.hdfs.util.DataTransferThrottler;
+import org.apache.hadoop.io.MD5Hash;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.StringUtils;
 
@@ -91,14 +92,20 @@ public class GetImageServlet extends Htt
             // issue a HTTP get request to download the new fsimage 
             nnImage.validateCheckpointUpload(ff.getToken());
             nnImage.newImageDigest = ff.getNewChecksum();
-            reloginIfNecessary().doAs(new PrivilegedExceptionAction<Void>() {
+            MD5Hash downloadImageDigest = reloginIfNecessary().doAs(
+                new PrivilegedExceptionAction<MD5Hash>() {
                 @Override
-                public Void run() throws Exception {
-                  TransferFsImage.getFileClient(ff.getInfoServer(), "getimage=1", 
-                      nnImage.getFsImageNameCheckpoint());
-                  return null;
+                public MD5Hash run() throws Exception {
+                  return TransferFsImage.getFileClient(
+                      ff.getInfoServer(), "getimage=1", 
+                      nnImage.getFsImageNameCheckpoint(), true);
                 }
             });
+            if (!nnImage.newImageDigest.equals(downloadImageDigest)) {
+              throw new IOException("The downloaded image is corrupt," +
+                  " expecting a checksum " + nnImage.newImageDigest +
+                  " but received a checksum " + downloadImageDigest);
+            }
            nnImage.checkpointUploadDone();
           }
           return null;

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1126902&r1=1126901&r2=1126902&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
Tue May 24 07:25:13 2011
@@ -348,7 +348,7 @@ public class SecondaryNameNode implement
                 NameNodeDirType.IMAGE);
             File[] srcNames = list.toArray(new File[list.size()]);
             assert srcNames.length > 0 : "No checkpoint targets.";
-            TransferFsImage.getFileClient(fsName, fileid, srcNames);
+            TransferFsImage.getFileClient(fsName, fileid, srcNames, false);
             LOG.info("Downloaded file " + srcNames[0].getName() + " size " +
                      srcNames[0].length() + " bytes.");
         
@@ -357,7 +357,7 @@ public class SecondaryNameNode implement
             list = getFSImage().getFiles(NameNodeFile.EDITS, NameNodeDirType.EDITS);
             srcNames = list.toArray(new File[list.size()]);;
             assert srcNames.length > 0 : "No checkpoint targets.";
-            TransferFsImage.getFileClient(fsName, fileid, srcNames);
+            TransferFsImage.getFileClient(fsName, fileid, srcNames, false);
             LOG.info("Downloaded file " + srcNames[0].getName() + " size " +
                 srcNames[0].length() + " bytes.");
         
@@ -383,7 +383,7 @@ public class SecondaryNameNode implement
       "&token=" + sig.toString() +
       "&newChecksum=" + checkpointImage.imageDigest;
     LOG.info("Posted URL " + fsName + fileid);
-    TransferFsImage.getFileClient(fsName, fileid, (File[])null);
+    TransferFsImage.getFileClient(fsName, fileid, (File[])null, false);
   }
 
   /**

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java?rev=1126902&r1=1126901&r2=1126902&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
Tue May 24 07:25:13 2011
@@ -19,6 +19,8 @@ package org.apache.hadoop.hdfs.server.na
 
 import java.io.*;
 import java.net.*;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
 import java.util.Iterator;
 import java.util.Map;
 import java.lang.Math;
@@ -169,8 +171,11 @@ class TransferFsImage implements FSConst
   /**
    * Client-side Method to fetch file from a server
    * Copies the response from the URL to a list of local files.
+   * 
+   * @Return a digest of the received file if getChecksum is true
    */
-  static void getFileClient(String fsName, String id, File[] localPath)
+  static MD5Hash getFileClient(String fsName, String id, File[] localPath,
+      boolean getChecksum)
     throws IOException {
     byte[] buf = new byte[BUFFER_SIZE];
     String proto = UserGroupInformation.isSecurityEnabled() ? "https://" : "http://";
@@ -195,6 +200,11 @@ class TransferFsImage implements FSConst
     }
     long received = 0;
     InputStream stream = connection.getInputStream();
+    MessageDigest digester = null;
+    if (getChecksum) {
+      digester = MD5Hash.getDigester();
+      stream = new DigestInputStream(stream, digester);
+    }
     FileOutputStream[] output = null;
 
     try {
@@ -230,5 +240,6 @@ class TransferFsImage implements FSConst
                               advertisedSize);
       }
     }
+    return digester==null ? null : new MD5Hash(digester.digest());
   }
 }

Propchange: hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:25:13 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
-/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:25:13 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
-/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:25:13 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
-/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:25:13 2011
@@ -3,4 +3,4 @@
 /hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
-/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576



Mime
View raw message