hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sammic...@apache.org
Subject [1/2] hadoop git commit: HDFS-11915. Sync rbw dir on the first hsync() to avoid file lost on power failure. Contributed by Vinayakumar B.
Date Tue, 10 Apr 2018 08:47:12 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.9.1 91bb336d2 -> a7de3cfa7


HDFS-11915. Sync rbw dir on the first hsync() to avoid file lost on power failure. Contributed
by Vinayakumar B.

(cherry picked from commit 2273499aef18ac2c7ffc435a61db8cea591e8b1f)
(cherry picked from commit f24d3b69b403f3a2c5af6b9c74a643fb9f4492e5)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b42f02ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b42f02ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b42f02ca

Branch: refs/heads/branch-2.9.1
Commit: b42f02ca0c011f5998a12dbbc22e26888874a22d
Parents: 91bb336
Author: Wei-Chiu Chuang <weichiu@apache.org>
Authored: Fri Jan 12 10:00:00 2018 -0800
Committer: Sammi Chen <sammi.chen@intel.com>
Committed: Tue Apr 10 11:41:48 2018 +0800

----------------------------------------------------------------------
 .../hdfs/server/datanode/BlockReceiver.java       |  9 +++++++++
 .../hadoop/hdfs/server/datanode/DatanodeUtil.java | 18 ++++++++++++++++++
 .../datanode/fsdataset/impl/FsDatasetImpl.java    | 15 ++-------------
 3 files changed, 29 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b42f02ca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index c8a33ca..7f381b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -24,6 +24,7 @@ import java.io.Closeable;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.EOFException;
+import java.io.File;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
@@ -127,6 +128,7 @@ class BlockReceiver implements Closeable {
 
   private boolean syncOnClose;
   private volatile boolean dirSyncOnFinalize;
+  private boolean dirSyncOnHSyncDone = false;
   private long restartBudget;
   /** the reference of the volume where the block receiver writes to */
   private ReplicaHandler replicaHandler;
@@ -421,6 +423,13 @@ class BlockReceiver implements Closeable {
       }
       flushTotalNanos += flushEndNanos - flushStartNanos;
     }
+    if (isSync && !dirSyncOnHSyncDone && replicaInfo instanceof ReplicaInfo)
{
+      ReplicaInfo rInfo = (ReplicaInfo) replicaInfo;
+      File baseDir = rInfo.getBlockFile().getParentFile();
+      FileIoProvider fileIoProvider = datanode.getFileIoProvider();
+      DatanodeUtil.fsyncDirectory(fileIoProvider, rInfo.getVolume(), baseDir);
+      dirSyncOnHSyncDone = true;
+    }
     if (checksumOut != null || streams.getDataOut() != null) {
       datanode.metrics.addFlushNanos(flushTotalNanos);
       if (isSync) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b42f02ca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index c98ff54..e29a5ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -142,4 +142,22 @@ public class DatanodeUtil {
     }
     return (FileInputStream)lin.getWrappedStream();
   }
+
+  /**
+   * Call fsync on specified directories to sync metadata changes.
+   * @param fileIoProvider
+   * @param volume
+   * @param dirs
+   * @throws IOException
+   */
+  public static void fsyncDirectory(FileIoProvider fileIoProvider,
+      FsVolumeSpi volume, File... dirs) throws IOException {
+    for (File dir : dirs) {
+      try {
+        fileIoProvider.dirSync(volume, dir);
+      } catch (IOException e) {
+        throw new IOException("Failed to sync " + dir, e);
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b42f02ca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 2512f6d..4486b73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -929,18 +929,6 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
     return dstfile;
   }
 
-  private void fsyncDirectory(FsVolumeSpi volume, File... dirs)
-      throws IOException {
-    FileIoProvider fileIoProvider = datanode.getFileIoProvider();
-    for (File dir : dirs) {
-      try {
-        fileIoProvider.dirSync(volume, dir);
-      } catch (IOException e) {
-        throw new IOException("Failed to sync " + dir, e);
-      }
-    }
-  }
-
   /**
    * Copy the block and meta files for the given block to the given destination.
    * @return the new meta and block files.
@@ -1801,7 +1789,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
       FsVolumeSpi v = replicaInfo.getVolume();
       File f = replicaInfo.getBlockFile();
       File dest = finalizedReplicaInfo.getBlockFile();
-      fsyncDirectory(v, dest.getParentFile(), f.getParentFile());
+      DatanodeUtil.fsyncDirectory(datanode.getFileIoProvider(), v,
+          dest.getParentFile(), f.getParentFile());
     }
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message