Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: subru@apache.org
To: common-commits@hadoop.apache.org
Date: Tue, 19 Jul 2016 22:02:06 -0000
Message-Id: <ae936cdec0a14863af208514c0fabaa2@git.apache.org>
In-Reply-To: <34f611f4ac964d3fb5185d84fa54c7b3@git.apache.org>
References: <34f611f4ac964d3fb5185d84fa54c7b3@git.apache.org>
Subject: [26/50] hadoop git commit: HADOOP-13351. TestDFSClientSocketSize
 buffer size tests are flaky. Contributed by Aaron Fabbri and Mingliang Liu.
archived-at: Tue, 19 Jul 2016 22:01:51 -0000

HADOOP-13351. TestDFSClientSocketSize buffer size tests are flaky. Contributed by Aaron Fabbri and Mingliang Liu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5537c6b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5537c6b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5537c6b2

Branch: refs/heads/YARN-2915
Commit: 5537c6b23430285ebee33c6d9b69d3ec1e9b17b1
Parents: 6cf0175
Author: Akira Ajisaka <aajisaka@apache.org>
Authored: Thu Jul 14 14:40:58 2016 -0700
Committer: Akira Ajisaka <aajisaka@apache.org>
Committed: Thu Jul 14 14:40:58 2016 -0700

----------------------------------------------------------------------
 .../hadoop/hdfs/TestDFSClientSocketSize.java    | 80 +++++++++++---------
 1 file changed, 44 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5537c6b2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientSocketSize.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientSocketSize.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientSocketSize.java
index 2376576..a2a7afc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientSocketSize.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientSocketSize.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
 
-import org.junit.After;
 import org.junit.Test;
 
 import org.slf4j.Logger;
@@ -33,7 +32,6 @@ import java.net.Socket;
 
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY;
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 public class TestDFSClientSocketSize {
@@ -43,55 +41,65 @@ public class TestDFSClientSocketSize {
     GenericTestUtils.setLogLevel(DataStreamer.LOG, Level.ALL);
   }
 
-  private final Configuration conf = new Configuration();
-  private MiniDFSCluster cluster;
-  private Socket socket;
-
+  /**
+   * The setting of socket send buffer size in
+   * {@link java.net.Socket#setSendBufferSize(int)} is only a hint.  Actual
+   * value may differ.  We just sanity check that it is somewhere close.
+   */
   @Test
   public void testDefaultSendBufferSize() throws IOException {
-    socket = createSocket();
-    assertEquals("Send buffer size should be the default value.",
-        DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT,
-        socket.getSendBufferSize());
+    assertTrue("Send buffer size should be somewhere near default.",
+        getSendBufferSize(new Configuration()) >=
+            DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT / 2);
   }
 
+  /**
+   * Note that {@link java.net.Socket#setSendBufferSize(int)} is only a hint.
+   * If this test is flaky it should be ignored.  See HADOOP-13351.
+   */
   @Test
   public void testSpecifiedSendBufferSize() throws IOException {
-    final int mySendBufferSize = 64 * 1024;  // 64 KB
-    conf.setInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY, mySendBufferSize);
-    socket = createSocket();
-    assertEquals("Send buffer size should be the customized value.",
-        mySendBufferSize, socket.getSendBufferSize());
+    final Configuration conf1 = new Configuration();
+    conf1.setInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY, 256 * 1024); // 256 KB
+    final int sendBufferSize1 = getSendBufferSize(conf1);
+
+    final Configuration conf2 = new Configuration();
+    conf2.setInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY, 1024);       // 1 KB
+    final int sendBufferSize2 = getSendBufferSize(conf2);
+
+    LOG.info("Large buf size is {}, small is {}",
+        sendBufferSize1, sendBufferSize2);
+    assertTrue("Larger specified send buffer should have effect",
+        sendBufferSize1 > sendBufferSize2);
   }
 
   @Test
   public void testAutoTuningSendBufferSize() throws IOException {
+    final Configuration conf = new Configuration();
     conf.setInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY, 0);
-    socket = createSocket();
-    LOG.info("The auto tuned send buffer size is: {}",
-        socket.getSendBufferSize());
+    final int sendBufferSize = getSendBufferSize(conf);
+    LOG.info("The auto tuned send buffer size is: {}", sendBufferSize);
     assertTrue("Send buffer size should be non-negative value which is " +
-        "determined by system (kernel).", socket.getSendBufferSize() > 0);
+          "determined by system (kernel).", sendBufferSize > 0);
   }
 
-  @After
-  public void tearDown() throws Exception {
-    if (socket != null) {
-      LOG.info("Closing the DFSClient socket.");
-    }
-    if (cluster != null) {
-      LOG.info("Shutting down MiniDFSCluster.");
-      cluster.shutdown();
-      cluster = null;
+  private int getSendBufferSize(Configuration conf) throws IOException {
+    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(1)
+        .build();
+    try {
+      cluster.waitActive();
+      LOG.info("MiniDFSCluster started.");
+      try (Socket socket = DataStreamer.createSocketForPipeline(
+          new DatanodeInfo(cluster.dataNodes.get(0).datanode.getDatanodeId()),
+          1, cluster.getFileSystem().getClient())) {
+        return socket.getSendBufferSize();
+      }
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
     }
   }
 
-  private Socket createSocket() throws IOException {
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
-    cluster.waitActive();
-    LOG.info("MiniDFSCluster started.");
-    return DataStreamer.createSocketForPipeline(
-        new DatanodeInfo(cluster.dataNodes.get(0).datanode.getDatanodeId()),
-        1, cluster.getFileSystem().getClient());
-  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org