hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bra...@apache.org
Subject [14/37] hadoop git commit: HDDS-794. Add configs to set StateMachineData write timeout in ContainerStateMachine. Contributed by Shashikant Banerjee.
Date Wed, 07 Nov 2018 02:09:12 GMT
HDDS-794. Add configs to set StateMachineData write timeout in ContainerStateMachine. Contributed
by Shashikant Banerjee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ddefdd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ddefdd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ddefdd5

Branch: refs/heads/HDFS-13891
Commit: 5ddefdd50751ed316f2eb9046f294bbdcdfb2428
Parents: 942693b
Author: Arpit Agarwal <arp@apache.org>
Authored: Mon Nov 5 10:10:10 2018 -0800
Committer: Arpit Agarwal <arp@apache.org>
Committed: Mon Nov 5 10:41:28 2018 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java |  6 ++++++
 .../org/apache/hadoop/ozone/OzoneConfigKeys.java  |  9 +++++++++
 .../common/src/main/resources/ozone-default.xml   |  7 +++++++
 .../server/ratis/ContainerStateMachine.java       | 18 ++++++++++++++++--
 .../server/ratis/XceiverServerRatis.java          | 14 ++++++++++++++
 .../container/keyvalue/helpers/ChunkUtils.java    |  2 ++
 .../container/keyvalue/impl/ChunkManagerImpl.java |  3 ++-
 7 files changed, 56 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 56692af..38eec61 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -79,6 +79,12 @@ public final class ScmConfigKeys {
       "dfs.container.ratis.segment.preallocated.size";
   public static final int
       DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = 128 * 1024 * 1024;
+  public static final String
+      DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
+      "dfs.container.ratis.statemachinedata.sync.timeout";
+  public static final TimeDuration
+      DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
+      TimeDuration.valueOf(10, TimeUnit.SECONDS);
   public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
       "dfs.ratis.client.request.timeout.duration";
   public static final TimeDuration

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 3b4f017..54b1cf8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -229,6 +229,15 @@ public final class OzoneConfigKeys {
       = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
   public static final int DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT
       = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
+
+  // config settings to enable stateMachineData write timeout
+  public static final String
+      DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
+      ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT;
+  public static final TimeDuration
+      DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
+      ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;
+
   public static final int DFS_CONTAINER_CHUNK_MAX_SIZE
       = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
   public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index eb68662..5ff60eb 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -53,6 +53,13 @@
     </description>
   </property>
   <property>
+    <name>dfs.container.ratis.statemachinedata.sync.timeout</name>
+    <value>10s</value>
+    <tag>OZONE, DEBUG, CONTAINER, RATIS</tag>
+    <description>Timeout for StateMachine data writes by Ratis.
+    </description>
+  </property>
+  <property>
     <name>dfs.container.ratis.datanode.storage.dir</name>
     <value/>
     <tag>OZONE, CONTAINER, STORAGE, MANAGEMENT, RATIS</tag>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index d5762bc..2a4a227 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -307,9 +307,18 @@ public class ContainerStateMachine extends BaseStateMachine {
           () -> runCommand(requestProto), chunkExecutor);
     }
     writeChunkFutureMap.put(entryIndex, writeChunkFuture);
+    LOG.debug("writeChunk writeStateMachineData : blockId " + write.getBlockID()
+        + " logIndex " + entryIndex + " chunkName " + write.getChunkData()
+        .getChunkName());
     // Remove the future once it finishes execution from the
     // writeChunkFutureMap.
-    writeChunkFuture.thenApply(r -> writeChunkFutureMap.remove(entryIndex));
+    writeChunkFuture.thenApply(r -> {
+      writeChunkFutureMap.remove(entryIndex);
+      LOG.debug("writeChunk writeStateMachineData  completed: blockId " + write
+          .getBlockID() + " logIndex " + entryIndex + " chunkName " + write
+          .getChunkData().getChunkName());
+      return r;
+    });
     return writeChunkFuture;
   }
 
@@ -531,7 +540,12 @@ public class ContainerStateMachine extends BaseStateMachine {
       if (cmdType == Type.CreateContainer) {
         long containerID = requestProto.getContainerID();
         future.thenApply(
-            r -> createContainerFutureMap.remove(containerID).complete(null));
+            r -> {
+              createContainerFutureMap.remove(containerID).complete(null);
+              LOG.info("create Container Transaction completed for container " +
+                  containerID + " log index " + index);
+              return r;
+            });
       }
 
       future.thenAccept(m -> {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 599f821..a679e5e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -186,6 +186,20 @@ public final class XceiverServerRatis implements XceiverServerSpi {
     RaftClientConfigKeys.Rpc
         .setRequestTimeout(properties, clientRequestTimeout);
 
+    // set the configs enable and set the stateMachineData sync timeout
+    RaftServerConfigKeys.Log.StateMachineData.setSync(properties, true);
+    timeUnit = OzoneConfigKeys.
+        DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT.getUnit();
+    duration = conf.getTimeDuration(
+        OzoneConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT,
+        OzoneConfigKeys.
+            DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT
+            .getDuration(), timeUnit);
+    final TimeDuration dataSyncTimeout =
+        TimeDuration.valueOf(duration, timeUnit);
+    RaftServerConfigKeys.Log.StateMachineData
+        .setSyncTimeout(properties, dataSyncTimeout);
+
     // Set the server Request timeout
     timeUnit = OzoneConfigKeys.DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_DEFAULT
         .getUnit();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
index 718f5de..8f9d589 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
@@ -138,6 +138,8 @@ public final class ChunkUtils {
         }
       }
     }
+    log.debug("Write Chunk completed for chunkFile: {}, size {}", chunkFile,
+        data.length);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
index c630e19..cdd19df 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
@@ -82,7 +82,8 @@ public class ChunkManagerImpl implements ChunkManager {
           chunkFile, info);
       File tmpChunkFile = getTmpChunkFile(chunkFile, info);
 
-      LOG.debug("writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file",
+      LOG.debug(
+          "writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file:{}",
           info.getChunkName(), stage, chunkFile, tmpChunkFile);
 
       switch (stage) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message