hadoop-ozone-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shashik...@apache.org
Subject [hadoop-ozone] 01/01: HDDS-2854. Add a config for the write chunk executor queue limit.
Date Wed, 08 Jan 2020 09:22:03 GMT
This is an automated email from the ASF dual-hosted git repository.

shashikant pushed a commit to branch HDDS-2854
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git

commit a2c854846041e4299ff29555de919ff946ac5006
Author: Shashikant Banerjee <shashikant@apache.org>
AuthorDate: Wed Jan 8 14:51:34 2020 +0530

    HDDS-2854. Add a config for the write chunk executor queue limit.
---
 .../src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java  | 6 ++++++
 .../src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java   | 8 ++++++++
 hadoop-hdds/common/src/main/resources/ozone-default.xml          | 9 +++++++++
 .../common/transport/server/ratis/XceiverServerRatis.java        | 7 ++++++-
 4 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 00d326e..8c11b05 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -48,6 +48,12 @@ public final class ScmConfigKeys {
       = "dfs.container.ratis.num.write.chunk.threads";
   public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT
       = 60;
+  public static final String
+          DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT
+          = "dfs.container.ratis.num.write.chunk.executor.queue.limit";
+  public static final int
+          DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT_DEFAULT
+          = 1024;
   public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
       = "dfs.container.ratis.replication.level";
   public static final ReplicationLevel
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 1700926..184124d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -240,6 +240,14 @@ public final class OzoneConfigKeys {
       = ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY;
   public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT
       = ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT;
+  public static final String
+          DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT
+          = ScmConfigKeys.
+          DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT;
+  public static final int
+          DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT_DEFAULT
+          = ScmConfigKeys.
+          DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT_DEFAULT;
   public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
       = ScmConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY;
   public static final ReplicationLevel
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index c83a5bf..a319f0c 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -204,6 +204,15 @@
     </description>
   </property>
   <property>
+    <name>dfs.container.ratis.num.write.chunk.executor.queue.limit</name>
+    <value>60</value>
+    <tag>OZONE, RATIS, PERFORMANCE</tag>
+    <description>Maximum limit of the number of tasks that can be pending
+      in the thread pool that Ratis will use for writing chunks
+      (1024 by default).
+    </description>
+  </property>
+  <property>
     <name>dfs.container.ratis.leader.num.pending.requests</name>
     <value>4096</value>
     <tag>OZONE, RATIS, PERFORMANCE</tag>
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index dd9327a..341e2af 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -127,10 +127,15 @@ public final class XceiverServerRatis implements XceiverServerSpi {
     final int numWriteChunkThreads = conf.getInt(
         OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY,
         OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT);
+    final int queueLimit = conf.getInt(
+            OzoneConfigKeys.
+                    DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT,
+            OzoneConfigKeys.
+            DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT_DEFAULT);
     chunkExecutor =
         new ThreadPoolExecutor(numWriteChunkThreads, numWriteChunkThreads,
             100, TimeUnit.SECONDS,
-            new ArrayBlockingQueue<>(1024),
+            new ArrayBlockingQueue<>(queueLimit),
             new ThreadPoolExecutor.CallerRunsPolicy());
     this.context = context;
     this.replicationLevel =


---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org


Mime
View raw message