Return-Path: Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: (qmail 74331 invoked from network); 7 Mar 2011 23:21:35 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 7 Mar 2011 23:21:35 -0000 Received: (qmail 87462 invoked by uid 500); 7 Mar 2011 23:21:35 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 87434 invoked by uid 500); 7 Mar 2011 23:21:35 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 87425 invoked by uid 99); 7 Mar 2011 23:21:35 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 07 Mar 2011 23:21:35 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 07 Mar 2011 23:21:26 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id A752A2388962; Mon, 7 Mar 2011 23:21:03 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1079020 - in /hadoop/hdfs/branches/HDFS-1052: ./ src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/ src/java/org/apache/hadoop/hdfs/ src/java/org/apache/hadoop/hdfs/server/datanode/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/... Date: Mon, 07 Mar 2011 23:21:03 -0000 To: hdfs-commits@hadoop.apache.org From: suresh@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110307232103.A752A2388962@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: suresh Date: Mon Mar 7 23:21:02 2011 New Revision: 1079020 URL: http://svn.apache.org/viewvc?rev=1079020&view=rev Log: Merging revisions r1052169:r1062010 from trunk to federation Added: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestWriteConfigurationToDFS.java - copied unchanged from r1060618, hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestWriteConfigurationToDFS.java Modified: hadoop/hdfs/branches/HDFS-1052/ (props changed) hadoop/hdfs/branches/HDFS-1052/CHANGES.txt hadoop/hdfs/branches/HDFS-1052/build.xml (contents, props changed) hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs/ (props changed) hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/ (props changed) hadoop/hdfs/branches/HDFS-1052/src/java/ (props changed) hadoop/hdfs/branches/HDFS-1052/src/java/hdfs-default.xml hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java (props changed) hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/UnderReplicatedBlocks.java hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/ (props changed) hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode/ (props changed) hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs/ (props changed) hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary/ (props changed) Propchange: hadoop/hdfs/branches/HDFS-1052/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Mar 7 23:21:02 2011 @@ -1,4 +1,4 @@ /hadoop/core/branches/branch-0.19/hdfs:713112 /hadoop/hdfs/branches/HDFS-265:796829-820463 /hadoop/hdfs/branches/branch-0.21:820487 -/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036738,1038222-1038859,1039957,1040005,1040027-1052169,1052823,1060619,1061067,1062020 +/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036738,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020 Modified: hadoop/hdfs/branches/HDFS-1052/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/CHANGES.txt?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/CHANGES.txt (original) +++ hadoop/hdfs/branches/HDFS-1052/CHANGES.txt Mon Mar 7 23:21:02 2011 @@ -7,6 +7,8 @@ Trunk (unreleased changes) HDFS-1526. Dfs client name for a map/reduce task should be unique among threads. (hairong) + HDFS-1536. Improve HDFS WebUI. (hairong) + NEW FEATURES HDFS-1365. Federation: propose ClusterID and BlockPoolID format @@ -244,6 +246,9 @@ Trunk (unreleased changes) name node is in safe mode. (Patrick Kling via hairong) HDFS-1534. Fix some incorrect logs in FSDirectory. (eli) + + HDFS-1539. A config option for the datanode to fsycn a block file + when block is completely written. (dhruba) OPTIMIZATIONS @@ -274,6 +279,14 @@ Trunk (unreleased changes) HDFS-1551. Fix pom templates dependency list (gkesavan) + HDFS-1509. A savenamespace command writes the fsimage and edits into + all configured directories. (dhruba) + + HDFS-1540. Make Datanode handle errors from RPC calls to namenode + more elegantly. (dhruba) + + HDFS-1463. Accesstime of a file is not updated in safeMode. (dhruba) + Release 0.22.0 - Unreleased NEW FEATURES @@ -692,6 +705,19 @@ Release 0.22.0 - Unreleased HDFS-1511. 98 Release Audit warnings on trunk and branch-0.22. (jghoman) + HDFS-1560. dfs.data.dir permissions should default to 700. + (Todd Lipcon via eli) + + HDFS-1550. NPE when listing a file with no location. (hairong) + + HDFS-1542. Add test for HADOOP-7082, a deadlock writing Configuration to + HDFS. (todd) + + HDFS-1504. FSImageSaver should catch all exceptions, not just IOE. (todd) + + HDFS-884. DataNode throws IOException if all data directories are + unavailable. (Steve Loughran and shv) + Release 0.21.1 - Unreleased HDFS-1411. Correct backup node startup command in hdfs user guide. @@ -733,7 +759,9 @@ Release 0.21.1 - Unreleased HDFS-1548. Fault-injection tests are executed multiple times if invoked with run-test-hdfs-fault-inject target (cos) -Release 0.21.0 - 2010-08-13 + HDFS-1552. Remove java5 dependencies from build. (cos) + + HDFS-996. JUnit tests should never depend on anything in conf (cos) INCOMPATIBLE CHANGES Modified: hadoop/hdfs/branches/HDFS-1052/build.xml URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/build.xml?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/build.xml (original) +++ hadoop/hdfs/branches/HDFS-1052/build.xml Mon Mar 7 23:21:02 2011 @@ -94,6 +94,7 @@ + @@ -229,7 +230,9 @@ - + + + @@ -289,6 +292,12 @@ + + + + + + Propchange: hadoop/hdfs/branches/HDFS-1052/build.xml ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Mar 7 23:21:02 2011 @@ -2,4 +2,4 @@ /hadoop/core/trunk/build.xml:779102 /hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463 /hadoop/hdfs/branches/branch-0.21/build.xml:820487 -/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1052169,1052823,1060619,1061067,1062020 +/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020 Propchange: hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Mar 7 23:21:02 2011 @@ -1,3 +1,3 @@ /hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112 /hadoop/core/trunk/src/c++/libhdfs:776175-784663 -/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1052169,1052823,1060619,1061067,1062020 +/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020 Propchange: hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Mar 7 23:21:02 2011 @@ -2,4 +2,4 @@ /hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663 /hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463 /hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487 -/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1052169,1052823,1060619,1061067,1062020 +/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020 Propchange: hadoop/hdfs/branches/HDFS-1052/src/java/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Mar 7 23:21:02 2011 @@ -2,4 +2,4 @@ /hadoop/core/trunk/src/hdfs:776175-785643,785929-786278 /hadoop/hdfs/branches/HDFS-265/src/java:796829-820463 /hadoop/hdfs/branches/branch-0.21/src/java:820487 -/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1052169,1052823,1060619,1061067,1062020 +/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020 Modified: hadoop/hdfs/branches/HDFS-1052/src/java/hdfs-default.xml URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/hdfs-default.xml?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/java/hdfs-default.xml (original) +++ hadoop/hdfs/branches/HDFS-1052/src/java/hdfs-default.xml Mon Mar 7 23:21:02 2011 @@ -274,7 +274,7 @@ creations/deletions), or "all". dfs.datanode.data.dir.perm - 755 + 700 Permissions for the directories on on the local filesystem where the DFS data node store its blocks. The permissions can either be octal or symbolic. Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Mon Mar 7 23:21:02 2011 @@ -106,6 +106,8 @@ public class DFSConfigKeys extends Commo public static final int DFS_LIST_LIMIT_DEFAULT = 1000; public static final String DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY = "dfs.datanode.failed.volumes.tolerated"; public static final int DFS_DATANODE_FAILED_VOLUMES_TOLERATED_DEFAULT = 0; + public static final String DFS_DATANODE_SYNCONCLOSE_KEY = "dfs.datanode.synconclose"; + public static final boolean DFS_DATANODE_SYNCONCLOSE_DEFAULT = false; //Delegation token related keys public static final String DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY = "dfs.namenode.delegation.key.update-interval"; @@ -146,7 +148,7 @@ public class DFSConfigKeys extends Commo public static final String DFS_DATANODE_ADDRESS_KEY = "dfs.datanode.address"; public static final String DFS_DATANODE_ADDRESS_DEFAULT = "0.0.0.0:50010"; public static final String DFS_DATANODE_DATA_DIR_PERMISSION_KEY = "dfs.datanode.data.dir.perm"; - public static final String DFS_DATANODE_DATA_DIR_PERMISSION_DEFAULT = "755"; + public static final String DFS_DATANODE_DATA_DIR_PERMISSION_DEFAULT = "700"; public static final String DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY = "dfs.datanode.directoryscan.interval"; public static final int DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT = 21600; public static final String DFS_DATANODE_DIRECTORYSCAN_THREADS_KEY = "dfs.datanode.directoryscan.threads"; Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java Mon Mar 7 23:21:02 2011 @@ -229,6 +229,9 @@ public class DFSUtil { } int nrBlocks = blocks.locatedBlockCount(); BlockLocation[] blkLocations = new BlockLocation[nrBlocks]; + if (nrBlocks == 0) { + return blkLocations; + } int idx = 0; for (LocatedBlock blk : blocks.getLocatedBlocks()) { assert idx < nrBlocks : "Incorrect index"; Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java Mon Mar 7 23:21:02 2011 @@ -25,6 +25,7 @@ import java.io.BufferedOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.EOFException; +import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; @@ -63,6 +64,7 @@ class BlockReceiver implements java.io.C private DataInputStream in = null; // from where data are read private DataChecksum checksum; // from where chunks of a block can be read private OutputStream out = null; // to block file at local disk + private OutputStream cout = null; // output stream for cehcksum file private DataOutputStream checksumOut = null; // to crc file at local disk private int bytesPerChecksum; private int checksumSize; @@ -142,10 +144,10 @@ class BlockReceiver implements java.io.C this.bytesPerChecksum, this.checksumSize); if (streams != null) { this.out = streams.dataOut; + this.cout = streams.checksumOut; this.checksumOut = new DataOutputStream(new BufferedOutputStream( - streams.checksumOut, + streams.checksumOut, SMALL_BUFFER_SIZE)); - // write data chunk header if creating a new replica if (isCreate) { BlockMetadataHeader.writeHeader(checksumOut, checksum); @@ -186,6 +188,9 @@ class BlockReceiver implements java.io.C try { if (checksumOut != null) { checksumOut.flush(); + if (datanode.syncOnClose && (cout instanceof FileOutputStream)) { + ((FileOutputStream)cout).getChannel().force(true); + } checksumOut.close(); checksumOut = null; } @@ -196,6 +201,9 @@ class BlockReceiver implements java.io.C try { if (out != null) { out.flush(); + if (datanode.syncOnClose && (out instanceof FileOutputStream)) { + ((FileOutputStream)out).getChannel().force(true); + } out.close(); out = null; } Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Mon Mar 7 23:21:02 2011 @@ -350,6 +350,7 @@ public class DataNode extends Configured int writePacketSize = 0; boolean isBlockTokenEnabled; BlockPoolTokenSecretManager blockPoolTokenSecretManager; + boolean syncOnClose; public DataBlockScanner blockScanner = null; private DirectoryScanner directoryScanner = null; @@ -440,6 +441,10 @@ public class DataNode extends Configured "dfs.blockreport.intervalMsec." + " Setting initial delay to 0 msec:"); } this.heartBeatInterval = conf.getLong("dfs.heartbeat.interval", HEARTBEAT_INTERVAL) * 1000L; + + // do we need to sync block file contents to disk when blockfile is closed? + this.syncOnClose = conf.getBoolean(DFSConfigKeys.DFS_DATANODE_SYNCONCLOSE_KEY, + DFSConfigKeys.DFS_DATANODE_SYNCONCLOSE_DEFAULT); } private void startInfoServer(Configuration conf) throws IOException { @@ -2086,21 +2091,19 @@ public class DataNode extends Configured DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_DEFAULT)); ArrayList dirs = getDataDirsFromURIs(dataDirs, localFS, permission); - if (dirs.size() > 0) { - return new DataNode(conf, dirs, resources); - } - LOG.error("All directories in " - + DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY + " are invalid."); - return null; + assert dirs.size() > 0 : "number of data directories should be > 0"; + return new DataNode(conf, dirs, resources); } // DataNode ctor expects AbstractList instead of List or Collection... static ArrayList getDataDirsFromURIs(Collection dataDirs, - LocalFileSystem localFS, FsPermission permission) { + LocalFileSystem localFS, FsPermission permission) throws IOException { ArrayList dirs = new ArrayList(); + StringBuilder invalidDirs = new StringBuilder(); for (URI dirURI : dataDirs) { if (!"file".equalsIgnoreCase(dirURI.getScheme())) { LOG.warn("Unsupported URI schema in " + dirURI + ". Ignoring ..."); + invalidDirs.append("\"").append(dirURI).append("\" "); continue; } // drop any (illegal) authority in the URI for backwards compatibility @@ -2110,10 +2113,14 @@ public class DataNode extends Configured dirs.add(data); } catch (IOException e) { LOG.warn("Invalid directory in: " - + DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY + ": " - + e.getMessage()); + + DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY + ": ", e); + invalidDirs.append("\"").append(data.getCanonicalPath()).append("\" "); } } + if (dirs.size() == 0) + throw new IOException("All directories in " + + DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY + " are invalid: " + + invalidDirs); return dirs; } Propchange: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Mar 7 23:21:02 2011 @@ -4,4 +4,4 @@ /hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278 /hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463 /hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487 -/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1052169,1052823,1060619,1061067,1062020 +/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020 Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java Mon Mar 7 23:21:02 2011 @@ -118,8 +118,6 @@ public class BlockManager { * Last block index used for replication work. */ private int replIndex = 0; - private long missingBlocksInCurIter = 0; - private long missingBlocksInPrevIter = 0; Random r = new Random(); // for block replicas placement @@ -668,6 +666,11 @@ public class BlockManager { corruptReplicaBlocksCount = corruptReplicas.size(); } + /** Return number of under-replicated but not missing blocks */ + int getUnderReplicatedNotMissingBlocks() { + return neededReplications.getUnderReplicatedBlockCount(); + } + /** * Schedule blocks for deletion at datanodes * @param nodesToProcess number of datanodes to schedule deletion work @@ -749,8 +752,6 @@ public class BlockManager { try { synchronized (neededReplications) { if (neededReplications.size() == 0) { - missingBlocksInCurIter = 0; - missingBlocksInPrevIter = 0; return blocksToReplicate; } @@ -769,8 +770,6 @@ public class BlockManager { if (!neededReplicationsIterator.hasNext()) { // start from the beginning replIndex = 0; - missingBlocksInPrevIter = missingBlocksInCurIter; - missingBlocksInCurIter = 0; blocksToProcess = Math.min(blocksToProcess, neededReplications .size()); if (blkCnt >= blocksToProcess) @@ -827,10 +826,6 @@ public class BlockManager { containingNodes = new ArrayList(); NumberReplicas numReplicas = new NumberReplicas(); srcNode = chooseSourceDatanode(block, containingNodes, numReplicas); - if ((numReplicas.liveReplicas() + numReplicas.decommissionedReplicas()) - <= 0) { - missingBlocksInCurIter++; - } if(srcNode == null) // block can not be replicated from any node return false; @@ -1724,7 +1719,7 @@ public class BlockManager { long getMissingBlocksCount() { // not locking - return Math.max(missingBlocksInPrevIter, missingBlocksInCurIter); + return this.neededReplications.getCorruptBlockSize(); } BlockInfo addINode(BlockInfo block, INodeFile iNode) { Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Mon Mar 7 23:21:02 2011 @@ -874,7 +874,7 @@ public class FSEditLog { return; // nothing to do, edits.new exists! // check if any of failed storage is now available and put it back - fsimage.attemptRestoreRemovedStorage(); + fsimage.attemptRestoreRemovedStorage(false); divertFileStreams( Storage.STORAGE_DIR_CURRENT + "/" + NameNodeFile.EDITS_NEW.getName()); Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Mon Mar 7 23:21:02 2011 @@ -886,7 +886,7 @@ public class FSImage extends Storage { sd1.unlock(); // unlock before removing (in case it will be // restored) } catch (Exception e) { - // nothing + LOG.info("Unable to unlock bad storage directory : " + sd.getRoot().getPath()); } removedStorageDirs.add(sd1); it.remove(); @@ -1197,9 +1197,9 @@ public class FSImage extends Storage { public void run() { try { saveCurrent(sd); - } catch (IOException ie) { - LOG.error("Unable to save image for " + sd.getRoot(), ie); - errorSDs.add(sd); + } catch (Throwable t) { + LOG.error("Unable to save image for " + sd.getRoot(), t); + errorSDs.add(sd); } } @@ -1237,7 +1237,11 @@ public class FSImage extends Storage { * in which case the journal will be lost. */ void saveNamespace(boolean renewCheckpointTime) throws IOException { + + // try to restore all failed edit logs here assert editLog != null : "editLog must be initialized"; + attemptRestoreRemovedStorage(true); + editLog.close(); if(renewCheckpointTime) this.checkpointTime = now(); @@ -1281,6 +1285,11 @@ public class FSImage extends Storage { for (Iterator it = dirIterator(NameNodeDirType.EDITS); it.hasNext();) { final StorageDirectory sd = it.next(); + // if this directory already stores the image and edits, then it was + // already processed in the earlier loop. + if (sd.getStorageDirType() == NameNodeDirType.IMAGE_AND_EDITS) { + continue; + } FSImageSaver saver = new FSImageSaver(sd, errorSDs); Thread saveThread = new Thread(saver, saver.toString()); saveThreads.add(saveThread); @@ -1715,10 +1724,11 @@ public class FSImage extends Storage { } /** - * See if any of removed storages iw "writable" again, and can be returned - * into service + * See if any of removed storages is "writable" again, and can be returned + * into service. If saveNamespace is set, then this methdod is being + * called form saveNamespace. */ - synchronized void attemptRestoreRemovedStorage() { + synchronized void attemptRestoreRemovedStorage(boolean saveNamespace) { // if directory is "alive" - copy the images there... if(!restoreFailedStorage || removedStorageDirs.size() == 0) return; //nothing to restore @@ -1733,7 +1743,15 @@ public class FSImage extends Storage { try { if(root.exists() && root.canWrite()) { - format(sd); + /** If this call is being made from savenamespace command, then no + * need to format, the savenamespace command will format and write + * the new image to this directory anyways. + */ + if (saveNamespace) { + sd.clearDirectory(); + } else { + format(sd); + } LOG.info("restoring dir " + sd.getRoot().getAbsolutePath()); if(sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) { File eFile = getEditFile(sd); Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Mon Mar 7 23:21:02 2011 @@ -849,6 +849,12 @@ public class FSNamesystem implements FSC } else { // second attempt is with write lock writeLock(); // writelock is needed to set accesstime } + + // if the namenode is in safemode, then do not update access time + if (isInSafeMode()) { + doAccessTime = false; + } + try { long now = now(); INodeFile inode = dir.getFileINode(src); @@ -4534,6 +4540,11 @@ public class FSNamesystem implements FSC return blockManager.underReplicatedBlocksCount; } + /** Return number of under-replicated but not missing blocks */ + public long getUnderReplicatedNotMissingBlocks() { + return blockManager.getUnderReplicatedNotMissingBlocks(); + } + /** Returns number of blocks with corrupt replicas */ public long getCorruptReplicaBlocks() { return blockManager.corruptReplicaBlocksCount; Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java Mon Mar 7 23:21:02 2011 @@ -145,7 +145,7 @@ class NamenodeJspHelper { // Ideally this should be displayed in RED long missingBlocks = fsn.getMissingBlocksCount(); if (missingBlocks > 0) { - return "
WARNING :" + " There are about " + missingBlocks + return "
WARNING :" + " There are " + missingBlocks + " missing blocks. Please check the log or run fsck.

"; } return ""; @@ -167,6 +167,10 @@ class NamenodeJspHelper { return " "; } + private String colTxt(String title) { + return " "; + } + private void counterReset() { colNum = 0; rowNum = 0; @@ -321,9 +325,9 @@ class NamenodeJspHelper { + "" + "Decommissioning Nodes " + colTxt() + ":" + colTxt() + decommissioning.size() - + rowTxt() + colTxt() + + rowTxt() + colTxt("Excludes missing blocks.") + "Number of Under-Replicated Blocks" + colTxt() + ":" + colTxt() - + fsn.getUnderReplicatedBlocks() + + fsn.getUnderReplicatedNotMissingBlocks() + "
\n"); if (live.isEmpty() && dead.isEmpty()) { Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/UnderReplicatedBlocks.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/UnderReplicatedBlocks.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/UnderReplicatedBlocks.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/UnderReplicatedBlocks.java Mon Mar 7 23:21:02 2011 @@ -26,8 +26,8 @@ import org.apache.hadoop.hdfs.protocol.B * Blocks have only one replicas has the highest */ class UnderReplicatedBlocks implements Iterable { - static final int LEVEL = 4; - static public final int QUEUE_WITH_CORRUPT_BLOCKS = 2; + static final int LEVEL = 5; + static public final int QUEUE_WITH_CORRUPT_BLOCKS = 4; private List> priorityQueues = new ArrayList>(); /* constructor */ @@ -55,6 +55,20 @@ class UnderReplicatedBlocks implements I return size; } + /* Return the number of under replication blocks excluding corrupt blocks */ + synchronized int getUnderReplicatedBlockCount() { + int size = 0; + for (int i=0; i set:priorityQueues) { Propchange: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Mar 7 23:21:02 2011 @@ -2,4 +2,4 @@ /hadoop/core/trunk/src/test/hdfs:776175-785643 /hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463 /hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487 -/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1052169,1052823,1060619,1061067,1062020 +/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020 Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java Mon Mar 7 23:21:02 2011 @@ -76,7 +76,14 @@ public class TestDatanodeConfig { // 1. Test unsupported schema. Only "file:" is supported. String dnDir = makeURI("shv", null, fileAsURI(dataDir).getPath()); conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dnDir); - DataNode dn = DataNode.createDataNode(new String[]{}, conf); + DataNode dn = null; + try { + dn = DataNode.createDataNode(new String[]{}, conf); + } catch(IOException e) { + // expecting exception here + } + if(dn != null) + dn.shutdown(); assertNull("Data-node startup should have failed.", dn); // 2. Test "file:" schema and no schema (path-only). Both should work. Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java Mon Mar 7 23:21:02 2011 @@ -718,6 +718,39 @@ public class TestFileCreation extends ju } /** + * Test creating a file whose data gets sync when closed + */ + public void testFileCreationSyncOnClose() throws IOException { + Configuration conf = new HdfsConfiguration(); + conf.setBoolean(DFSConfigKeys.DFS_DATANODE_SYNCONCLOSE_KEY, true); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + + try { + FileSystem fs = cluster.getFileSystem(); + + Path[] p = {new Path("/foo"), new Path("/bar")}; + + //write 2 files at the same time + FSDataOutputStream[] out = {fs.create(p[0]), fs.create(p[1])}; + int i = 0; + for(; i < 100; i++) { + out[0].write(i); + out[1].write(i); + } + out[0].close(); + for(; i < 200; i++) {out[1].write(i);} + out[1].close(); + + //verify + FSDataInputStream[] in = {fs.open(p[0]), fs.open(p[1])}; + for(i = 0; i < 100; i++) {assertEquals(i, in[0].read());} + for(i = 0; i < 200; i++) {assertEquals(i, in[1].read());} + } finally { + if (cluster != null) {cluster.shutdown();} + } + } + + /** * Create a file, write something, hflush but not close. * Then change lease period and wait for lease recovery. * Finally, read the block directly from each Datanode and verify the content. Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java Mon Mar 7 23:21:02 2011 @@ -50,6 +50,7 @@ public class TestMissingBlocksAlert exte //minimize test delay conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 0); int fileLen = 10*1024; + conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, fileLen/2); //start a cluster with single datanode cluster = new MiniDFSCluster.Builder(conf).build(); @@ -84,13 +85,16 @@ public class TestMissingBlocksAlert exte Thread.sleep(100); } assertTrue(dfs.getMissingBlocksCount() == 1); + assertEquals(4, dfs.getUnderReplicatedBlocksCount()); + assertEquals(3, + cluster.getNamesystem().getUnderReplicatedNotMissingBlocks()); // Now verify that it shows up on webui URL url = new URL("http://" + conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY) + "/dfshealth.jsp"); String dfsFrontPage = DFSTestUtil.urlGet(url); - String warnStr = "WARNING : There are about "; + String warnStr = "WARNING : There are "; assertTrue("HDFS Front page does not contain expected warning", dfsFrontPage.contains(warnStr + "1 missing blocks")); @@ -104,6 +108,10 @@ public class TestMissingBlocksAlert exte Thread.sleep(100); } + assertEquals(2, dfs.getUnderReplicatedBlocksCount()); + assertEquals(2, + cluster.getNamesystem().getUnderReplicatedNotMissingBlocks()); + // and make sure WARNING disappears // Now verify that it shows up on webui dfsFrontPage = DFSTestUtil.urlGet(url); Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java Mon Mar 7 23:21:02 2011 @@ -179,9 +179,9 @@ public class TestDiskError { @Test public void testLocalDirs() throws Exception { Configuration conf = new Configuration(); - final String permStr = "755"; + final String permStr = conf.get( + DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_KEY); FsPermission expected = new FsPermission(permStr); - conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_KEY, permStr); // Check permissions on directories in 'dfs.data.dir' FileSystem localFS = FileSystem.getLocal(conf); Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java Mon Mar 7 23:21:02 2011 @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.na import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; @@ -60,6 +61,17 @@ public class TestSaveNamespace { private static class FaultySaveImage implements Answer { int count = 0; + boolean exceptionType = true; + + // generate a RuntimeException + public FaultySaveImage() { + this.exceptionType = true; + } + + // generate either a RuntimeException or IOException + public FaultySaveImage(boolean etype) { + this.exceptionType = etype; + } public Void answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); @@ -67,7 +79,11 @@ public class TestSaveNamespace { if (count++ == 1) { LOG.info("Injecting fault for file: " + f); - throw new RuntimeException("Injected fault: saveFSImage second time"); + if (exceptionType) { + throw new RuntimeException("Injected fault: saveFSImage second time"); + } else { + throw new IOException("Injected fault: saveFSImage second time"); + } } LOG.info("Not injecting fault for file: " + f); return (Void)invocation.callRealMethod(); @@ -142,6 +158,82 @@ public class TestSaveNamespace { } } + /** + * Verify that a saveNamespace command brings faulty directories + * in fs.name.dir and fs.edit.dir back online. + */ + @Test + public void testReinsertnamedirsInSavenamespace() throws Exception { + // create a configuration with the key to restore error + // directories in fs.name.dir + Configuration conf = getConf(); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_KEY, true); + + NameNode.initMetrics(conf, NamenodeRole.ACTIVE); + NameNode.format(conf); + FSNamesystem fsn = new FSNamesystem(conf); + + // Replace the FSImage with a spy + FSImage originalImage = fsn.dir.fsImage; + FSImage spyImage = spy(originalImage); + spyImage.setStorageDirectories( + FSNamesystem.getNamespaceDirs(conf), + FSNamesystem.getNamespaceEditsDirs(conf)); + fsn.dir.fsImage = spyImage; + + // inject fault + // The spy throws a IOException when writing to the second directory + doAnswer(new FaultySaveImage(false)). + when(spyImage).saveFSImage((File)anyObject()); + + try { + doAnEdit(fsn, 1); + fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + + // Save namespace - this injects a fault and marks one + // directory as faulty. + LOG.info("Doing the first savenamespace."); + fsn.saveNamespace(); + LOG.warn("First savenamespace sucessful."); + assertTrue("Savenamespace should have marked one directory as bad." + + " But found " + spyImage.getRemovedStorageDirs().size() + + " bad directories.", + spyImage.getRemovedStorageDirs().size() == 1); + + // The next call to savenamespace should try inserting the + // erroneous directory back to fs.name.dir. This command should + // be successful. + LOG.info("Doing the second savenamespace."); + fsn.saveNamespace(); + LOG.warn("Second savenamespace sucessful."); + assertTrue("Savenamespace should have been successful in removing " + + " bad directories from Image." + + " But found " + originalImage.getRemovedStorageDirs().size() + + " bad directories.", + originalImage.getRemovedStorageDirs().size() == 0); + + // Now shut down and restart the namesystem + LOG.info("Shutting down fsimage."); + originalImage.close(); + fsn.close(); + fsn = null; + + // Start a new namesystem, which should be able to recover + // the namespace from the previous incarnation. + LOG.info("Loading new FSmage from disk."); + fsn = new FSNamesystem(conf); + + // Make sure the image loaded including our edit. + LOG.info("Checking reloaded image."); + checkEditExists(fsn, 1); + LOG.info("Reloaded image is good."); + } finally { + if (fsn != null) { + fsn.close(); + } + } + } + @Test public void testCrashWhileSavingSecondImage() throws Exception { saveNamespaceWithInjectedFault(Fault.SAVE_FSIMAGE); Modified: hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java?rev=1079020&r1=1079019&r2=1079020&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java (original) +++ hadoop/hdfs/branches/HDFS-1052/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java Mon Mar 7 23:21:02 2011 @@ -39,7 +39,7 @@ public class TestDataDirs { @Test public void testGetDataDirsFromURIs() throws Throwable { File localDir = make(stub(File.class).returning(true).from.exists()); when(localDir.mkdir()).thenReturn(true); - FsPermission normalPerm = new FsPermission("755"); + FsPermission normalPerm = new FsPermission("700"); FsPermission badPerm = new FsPermission("000"); FileStatus stat = make(stub(FileStatus.class) .returning(normalPerm, normalPerm, badPerm).from.getPermission()); Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Mar 7 23:21:02 2011 @@ -2,4 +2,4 @@ /hadoop/core/trunk/src/webapps/datanode:776175-784663 /hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463 /hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487 -/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1052169,1052823,1060619,1061067,1062020 +/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020 Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Mar 7 23:21:02 2011 @@ -2,4 +2,4 @@ /hadoop/core/trunk/src/webapps/hdfs:776175-784663 /hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463 /hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487 -/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1052169,1052823,1060619,1061067,1062020 +/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020 Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Mar 7 23:21:02 2011 @@ -2,4 +2,4 @@ /hadoop/core/trunk/src/webapps/secondary:776175-784663 /hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463 /hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487 -/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1052169,1052823,1060619,1061067,1062020 +/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020