hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vinayakum...@apache.org
Subject [32/50] hadoop git commit: HDFS-8744. Erasure Coding: the number of chunks in packet is not updated when writing parity data. Contributed by Li Bo
Date Fri, 14 Aug 2015 10:54:46 GMT
HDFS-8744. Erasure Coding: the number of chunks in packet is not updated when writing parity
data. Contributed by Li Bo


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/436454a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/436454a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/436454a8

Branch: refs/heads/HDFS-7285-REBASE
Commit: 436454a8927f14bc1fcf7231eb4cb9ce01c2c395
Parents: 61b228a
Author: boli2 <bo.b.li@intel.com>
Authored: Mon Jul 13 00:41:36 2015 -0400
Committer: Vinayakumar B <vinayakumarb@apache.org>
Committed: Thu Aug 13 17:19:54 2015 +0530

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt       | 3 +++
 .../src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java  | 2 +-
 .../src/main/java/org/apache/hadoop/hdfs/DFSPacket.java        | 6 +++---
 .../java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java    | 4 +++-
 4 files changed, 10 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/436454a8/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 90f6732..7b6d165 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -341,3 +341,6 @@
 
     HDFS-8484. Erasure coding: Two contiguous blocks occupy IDs belong to same
     striped group. (Walter Su via jing9)
+
+    HDFS-8744. Erasure Coding: the number of chunks in packet is not updated
+    when writing parity data. (Li Bo)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/436454a8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 088f6dd..0f02049 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -418,7 +418,7 @@ public class DFSOutputStream extends FSOutputSummer
 
     currentPacket.writeChecksum(checksum, ckoff, cklen);
     currentPacket.writeData(b, offset, len);
-    currentPacket.incNumChunks();
+    currentPacket.incNumChunks(1);
     getStreamer().incBytesCurBlock(len);
 
     // If packet is full, enqueue it for transmission

http://git-wip-us.apache.org/repos/asf/hadoop/blob/436454a8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
index a26e35e..2698de3 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
@@ -259,10 +259,10 @@ public class DFSPacket {
   }
 
   /**
-   * increase the number of chunks by one
+   * increase the number of chunks by n
    */
-  synchronized void incNumChunks() {
-    numChunks++;
+  synchronized void incNumChunks(int n) {
+    numChunks += n;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/436454a8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 5e80b46..5ad1db8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -389,11 +389,13 @@ public class DFSStripedOutputStream extends DFSOutputStream {
       int maxBytesToPacket = p.getMaxChunks() * bytesPerChecksum;
       int toWrite = byteBuffer.remaining() > maxBytesToPacket ?
           maxBytesToPacket: byteBuffer.remaining();
-      int ckLen = ((toWrite - 1) / bytesPerChecksum + 1) * getChecksumSize();
+      int chunks = (toWrite - 1) / bytesPerChecksum + 1;
+      int ckLen = chunks * getChecksumSize();
       p.writeChecksum(checksumBuf, ckOff, ckLen);
       ckOff += ckLen;
       p.writeData(byteBuffer, toWrite);
       getCurrentStreamer().incBytesCurBlock(toWrite);
+      p.incNumChunks(chunks);
       packets.add(p);
     }
     return packets;


Mime
View raw message