Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4E41110B2A for ; Fri, 31 Jan 2014 05:39:10 +0000 (UTC) Received: (qmail 57484 invoked by uid 500); 31 Jan 2014 05:39:08 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 57417 invoked by uid 500); 31 Jan 2014 05:39:07 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 57409 invoked by uid 99); 31 Jan 2014 05:39:07 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 31 Jan 2014 05:39:07 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 31 Jan 2014 05:39:04 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id DA3F323888D7; Fri, 31 Jan 2014 05:38:42 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1563065 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java Date: Fri, 31 Jan 2014 05:38:42 -0000 To: hdfs-commits@hadoop.apache.org From: suresh@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140131053842.DA3F323888D7@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: suresh Date: Fri Jan 31 05:38:42 2014 New Revision: 1563065 URL: http://svn.apache.org/r1563065 Log: HDFS-5856. Merge 1563064 from trunk. Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1563065&r1=1563064&r2=1563065&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Jan 31 05:38:42 2014 @@ -24,6 +24,9 @@ Release 2.4.0 - UNRELEASED HDFS-5843. DFSClient.getFileChecksum() throws IOException if checksum is disabled. (Laurent Goujon via jing9) + HDFS-5856. DataNode.checkDiskError might throw NPE. + (Josh Elser via suresh) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1563065&r1=1563064&r2=1563065&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Fri Jan 31 05:38:42 2014 @@ -36,6 +36,7 @@ import java.net.SocketTimeoutException; import java.net.URI; import java.net.UnknownHostException; import java.nio.channels.ClosedByInterruptException; +import java.nio.channels.ClosedChannelException; import java.nio.channels.SocketChannel; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; @@ -51,7 +52,6 @@ import java.util.concurrent.atomic.Atomi import javax.management.ObjectName; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -1325,12 +1325,7 @@ public class DataNode extends Configured protected void checkDiskError(Exception e ) throws IOException { LOG.warn("checkDiskError: exception: ", e); - if (e instanceof SocketException || e instanceof SocketTimeoutException - || e instanceof ClosedByInterruptException - || e.getMessage().startsWith("An established connection was aborted") - || e.getMessage().startsWith("Broken pipe") - || e.getMessage().startsWith("Connection reset") - || e.getMessage().contains("java.nio.channels.SocketChannel")) { + if (isNetworkRelatedException(e)) { LOG.info("Not checking disk as checkDiskError was called on a network" + " related exception"); return; @@ -1344,6 +1339,28 @@ public class DataNode extends Configured } /** + * Check if the provided exception looks like it's from a network error + * @param e the exception from a checkDiskError call + * @return true if this exception is network related, false otherwise + */ + protected boolean isNetworkRelatedException(Exception e) { + if (e instanceof SocketException + || e instanceof SocketTimeoutException + || e instanceof ClosedChannelException + || e instanceof ClosedByInterruptException) { + return true; + } + + String msg = e.getMessage(); + + return null != msg + && (msg.startsWith("An established connection was aborted") + || msg.startsWith("Broken pipe") + || msg.startsWith("Connection reset") + || msg.contains("java.nio.channels.SocketChannel")); + } + + /** * Check if there is a disk failure and if so, handle the error */ public void checkDiskError() { Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java?rev=1563065&r1=1563064&r2=1563065&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java Fri Jan 31 05:38:42 2014 @@ -18,12 +18,16 @@ package org.apache.hadoop.hdfs.server.datanode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.DataOutputStream; import java.io.File; import java.net.InetSocketAddress; import java.net.Socket; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.nio.channels.ClosedChannelException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -196,4 +200,16 @@ public class TestDiskError { } } } + + @Test + public void testNetworkErrorsIgnored() { + DataNode dn = cluster.getDataNodes().iterator().next(); + + assertTrue(dn.isNetworkRelatedException(new SocketException())); + assertTrue(dn.isNetworkRelatedException(new SocketTimeoutException())); + assertTrue(dn.isNetworkRelatedException(new ClosedChannelException())); + assertTrue(dn.isNetworkRelatedException(new Exception("Broken pipe foo bar"))); + assertFalse(dn.isNetworkRelatedException(new Exception())); + assertFalse(dn.isNetworkRelatedException(new Exception("random problem"))); + } }