hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From x...@apache.org
Subject hadoop git commit: HDFS-11195. Return error when appending files by webhdfs rest api fails. Contributed by Yuanbo Liu.
Date Tue, 20 Dec 2016 20:27:19 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 523411d69 -> 5b7acdd20


HDFS-11195. Return error when appending files by webhdfs rest api fails. Contributed by Yuanbo
Liu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b7acdd2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b7acdd2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b7acdd2

Branch: refs/heads/trunk
Commit: 5b7acdd206f5a7d1b7af29b68adaa7587d7d8c43
Parents: 523411d
Author: Xiao Chen <xiao@apache.org>
Authored: Tue Dec 20 12:24:00 2016 -0800
Committer: Xiao Chen <xiao@apache.org>
Committed: Tue Dec 20 12:24:00 2016 -0800

----------------------------------------------------------------------
 .../server/datanode/web/webhdfs/HdfsWriter.java | 19 ++++++++--
 .../org/apache/hadoop/hdfs/MiniDFSCluster.java  | 15 ++++++--
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 39 ++++++++++++++++++++
 3 files changed, 65 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7acdd2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java
index 99924e5..8de4bb2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java
@@ -55,9 +55,13 @@ class HdfsWriter extends SimpleChannelInboundHandler<HttpContent>
{
     throws IOException {
     chunk.content().readBytes(out, chunk.content().readableBytes());
     if (chunk instanceof LastHttpContent) {
-      response.headers().set(CONNECTION, CLOSE);
-      ctx.write(response).addListener(ChannelFutureListener.CLOSE);
-      releaseDfsResources();
+      try {
+        releaseDfsResourcesAndThrow();
+        response.headers().set(CONNECTION, CLOSE);
+        ctx.write(response).addListener(ChannelFutureListener.CLOSE);
+      } catch (Exception cause) {
+        exceptionCaught(ctx, cause);
+      }
     }
   }
 
@@ -71,7 +75,10 @@ class HdfsWriter extends SimpleChannelInboundHandler<HttpContent>
{
     releaseDfsResources();
     DefaultHttpResponse resp = ExceptionHandler.exceptionCaught(cause);
     resp.headers().set(CONNECTION, CLOSE);
-    ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE);
+    ctx.writeAndFlush(resp).addListener(ChannelFutureListener.CLOSE);
+    if (LOG != null && LOG.isDebugEnabled()) {
+      LOG.debug("Exception in channel handler ", cause);
+    }
   }
 
   private void releaseDfsResources() {
@@ -79,4 +86,8 @@ class HdfsWriter extends SimpleChannelInboundHandler<HttpContent>
{
     IOUtils.cleanup(LOG, client);
   }
 
+  private void releaseDfsResourcesAndThrow() throws Exception {
+    out.close();
+    client.close();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7acdd2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index cf02a8d..51dca41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -1980,14 +1980,21 @@ public class MiniDFSCluster implements AutoCloseable {
    */
   public void shutdownDataNodes() {
     for (int i = dataNodes.size()-1; i >= 0; i--) {
-      LOG.info("Shutting down DataNode " + i);
-      DataNode dn = dataNodes.remove(i).datanode;
-      dn.shutdown();
-      numDataNodes--;
+      shutdownDataNode(i);
     }
   }
 
   /**
+   * Shutdown the datanode at a given index.
+   */
+  public void shutdownDataNode(int dnIndex) {
+    LOG.info("Shutting down DataNode " + dnIndex);
+    DataNode dn = dataNodes.remove(dnIndex).datanode;
+    dn.shutdown();
+    numDataNodes--;
+  }
+
+  /**
    * Shutdown all the namenodes.
    */
   public synchronized void shutdownNameNodes() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7acdd2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 259353c..d4495dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -1181,4 +1181,43 @@ public class TestWebHDFS {
       cluster.shutdown();
     }
   }
+
+  @Test
+  public void testWebHdfsAppend() throws Exception {
+    MiniDFSCluster cluster = null;
+    final Configuration conf = WebHdfsTestUtil.createConf();
+    final int dnNumber = 3;
+    try {
+
+      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNumber).build();
+
+      final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem(
+          conf, WebHdfsConstants.WEBHDFS_SCHEME);
+
+      final DistributedFileSystem fs = cluster.getFileSystem();
+
+      final Path appendFile = new Path("/testAppend.txt");
+      final String content = "hello world";
+      DFSTestUtil.writeFile(fs, appendFile, content);
+
+      for (int index = 0; index < dnNumber - 1; index++){
+        cluster.shutdownDataNode(index);
+      }
+      cluster.restartNameNodes();
+      cluster.waitActive();
+
+      try {
+        DFSTestUtil.appendFile(webFS, appendFile, content);
+        fail("Should fail to append file since "
+            + "datanode number is 1 and replication is 3");
+      } catch (IOException ignored) {
+        String resultContent = DFSTestUtil.readFile(fs, appendFile);
+        assertTrue(resultContent.equals(content));
+      }
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown(true);
+      }
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message