hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From umamah...@apache.org
Subject hadoop git commit: HDFS-9261. Erasure Coding: Skip encoding the data cells if all the parity data streamers are failed for the current block group. (Rakesh R via umamahesh)
Date Thu, 29 Oct 2015 06:35:22 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 588baab16 -> 07ecdb877


HDFS-9261. Erasure Coding: Skip encoding the data cells if all the parity data streamers are
failed for the current block group. (Rakesh R via umamahesh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07ecdb87
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07ecdb87
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07ecdb87

Branch: refs/heads/trunk
Commit: 07ecdb877dee0aa76a3269e37ac4783a55ab6bf6
Parents: 588baab
Author: Uma Mahesh <umamahesh@apache.org>
Authored: Wed Oct 28 23:33:11 2015 -0700
Committer: Uma Mahesh <umamahesh@apache.org>
Committed: Wed Oct 28 23:33:11 2015 -0700

----------------------------------------------------------------------
 .../hadoop/hdfs/DFSStripedOutputStream.java       | 18 ++++++++++++++++++
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt       |  3 +++
 2 files changed, 21 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/07ecdb87/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 884fa42..c595026 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -844,6 +844,11 @@ public class DFSStripedOutputStream extends DFSOutputStream {
 
   void writeParityCells() throws IOException {
     final ByteBuffer[] buffers = cellBuffers.getBuffers();
+    // Skips encoding and writing parity cells if there are no healthy parity
+    // data streamers
+    if (!checkAnyParityStreamerIsHealthy()) {
+      return;
+    }
     //encode the data cells
     encode(encoder, numDataBlocks, buffers);
     for (int i = numDataBlocks; i < numAllBlocks; i++) {
@@ -852,6 +857,19 @@ public class DFSStripedOutputStream extends DFSOutputStream {
     cellBuffers.clear();
   }
 
+  private boolean checkAnyParityStreamerIsHealthy() {
+    for (int i = numDataBlocks; i < numAllBlocks; i++) {
+      if (streamers.get(i).isHealthy()) {
+        return true;
+      }
+    }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Skips encoding and writing parity cells as there are "
+          + "no healthy parity data streamers: " + streamers);
+    }
+    return false;
+  }
+
   void writeParity(int index, ByteBuffer buffer, byte[] checksumBuf)
       throws IOException {
     final StripedDataStreamer current = setCurrentStreamer(index);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/07ecdb87/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index aa26088..a87fe76 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -184,6 +184,9 @@ Trunk (Unreleased)
     HDFS-9070. Allow fsck display pending replica location information for
     being-written blocks. (GAO Rui via jing9)
 
+    HDFS-9261. Erasure Coding: Skip encoding the data cells if all the parity data 
+    streamers are failed for the current block group. (Rakesh R via umamahesh)
+
   OPTIMIZATIONS
 
   BUG FIXES


Mime
View raw message