hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [11/50] [abbrv] hadoop git commit: HDFS-11541. Call RawErasureEncoder and RawErasureDecoder release() methods. Contributed by SammiChen.
Date Tue, 04 Apr 2017 19:56:35 GMT
HDFS-11541. Call RawErasureEncoder and RawErasureDecoder release() methods. Contributed by
SammiChen.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84d787b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84d787b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84d787b9

Branch: refs/heads/HDFS-7240
Commit: 84d787b9d51196010495d51dc5ebf66c01c340ab
Parents: 0e6f8e4
Author: Rakesh Radhakrishnan <rakeshr@apache.org>
Authored: Wed Mar 29 11:41:48 2017 +0530
Committer: Rakesh Radhakrishnan <rakeshr@apache.org>
Committed: Wed Mar 29 11:41:48 2017 +0530

----------------------------------------------------------------------
 .../hadoop/hdfs/DFSStripedInputStream.java      | 20 ++++----
 .../hadoop/hdfs/DFSStripedOutputStream.java     |  1 +
 .../StripedBlockChecksumReconstructor.java      | 48 +++++++++++---------
 .../erasurecode/StripedBlockReconstructor.java  |  1 +
 .../erasurecode/StripedReconstructor.java       |  6 +++
 5 files changed, 46 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 922f74e..07e6609 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -177,14 +177,18 @@ public class DFSStripedInputStream extends DFSInputStream {
 
   @Override
   public synchronized void close() throws IOException {
-    super.close();
-    if (curStripeBuf != null) {
-      BUFFER_POOL.putBuffer(curStripeBuf);
-      curStripeBuf = null;
-    }
-    if (parityBuf != null) {
-      BUFFER_POOL.putBuffer(parityBuf);
-      parityBuf = null;
+    try {
+      super.close();
+    } finally {
+      if (curStripeBuf != null) {
+        BUFFER_POOL.putBuffer(curStripeBuf);
+        curStripeBuf = null;
+      }
+      if (parityBuf != null) {
+        BUFFER_POOL.putBuffer(parityBuf);
+        parityBuf = null;
+      }
+      decoder.release();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 52fc5eb..22b30e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -1033,6 +1033,7 @@ public class DFSStripedOutputStream extends DFSOutputStream {
       setClosed();
       // shutdown executor of flushAll tasks
       flushAllExecutor.shutdownNow();
+      encoder.release();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
index 9555618..6917317 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
@@ -75,29 +75,33 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor
{
   public void reconstruct() throws IOException {
     MessageDigest digester = MD5Hash.getDigester();
     long maxTargetLength = getMaxTargetLength();
-    while (requestedLen > 0 && getPositionInBlock() < maxTargetLength) {
-      long remaining = maxTargetLength - getPositionInBlock();
-      final int toReconstructLen = (int) Math
-          .min(getStripedReader().getBufferSize(), remaining);
-      // step1: read from minimum source DNs required for reconstruction.
-      // The returned success list is the source DNs we do real read from
-      getStripedReader().readMinimumSources(toReconstructLen);
-
-      // step2: decode to reconstruct targets
-      reconstructTargets(toReconstructLen);
-
-      // step3: calculate checksum
-      checksumDataLen += checksumWithTargetOutput(targetBuffer.array(),
-          toReconstructLen, digester);
-
-      updatePositionInBlock(toReconstructLen);
-      requestedLen -= toReconstructLen;
-      clearBuffers();
-    }
+    try {
+      while (requestedLen > 0 && getPositionInBlock() < maxTargetLength) {
+        long remaining = maxTargetLength - getPositionInBlock();
+        final int toReconstructLen = (int) Math
+            .min(getStripedReader().getBufferSize(), remaining);
+        // step1: read from minimum source DNs required for reconstruction.
+        // The returned success list is the source DNs we do real read from
+        getStripedReader().readMinimumSources(toReconstructLen);
+
+        // step2: decode to reconstruct targets
+        reconstructTargets(toReconstructLen);
+
+        // step3: calculate checksum
+        checksumDataLen += checksumWithTargetOutput(targetBuffer.array(),
+            toReconstructLen, digester);
+
+        updatePositionInBlock(toReconstructLen);
+        requestedLen -= toReconstructLen;
+        clearBuffers();
+      }
 
-    byte[] digest = digester.digest();
-    md5 = new MD5Hash(digest);
-    md5.write(checksumWriter);
+      byte[] digest = digester.digest();
+      md5 = new MD5Hash(digest);
+      md5.write(checksumWriter);
+    } finally {
+      cleanup();
+    }
   }
 
   private long checksumWithTargetOutput(byte[] outputData, int toReconstructLen,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
index a1da536..1119bbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
@@ -74,6 +74,7 @@ class StripedBlockReconstructor extends StripedReconstructor
       metrics.incrECReconstructionBytesWritten(getBytesWritten());
       getStripedReader().close();
       stripedWriter.close();
+      cleanup();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
index cd17864..b8433c7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
@@ -253,6 +253,12 @@ abstract class StripedReconstructor {
     return decoder;
   }
 
+  void cleanup() {
+    if (decoder != null) {
+      decoder.release();
+    }
+  }
+
   StripedReader getStripedReader() {
     return stripedReader;
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message