accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmil...@apache.org
Subject [accumulo] branch master updated: Get defaults for BatchWriterConfig from ClientProperty (#777)
Date Thu, 22 Nov 2018 00:00:56 GMT
This is an automated email from the ASF dual-hosted git repository.

mmiller pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/master by this push:
     new 200f510  Get defaults for BatchWriterConfig from ClientProperty (#777)
200f510 is described below

commit 200f51033e96e861d0fb6d3c32edab70cbc0bd49
Author: Mike Miller <mmiller@apache.org>
AuthorDate: Wed Nov 21 19:00:52 2018 -0500

    Get defaults for BatchWriterConfig from ClientProperty (#777)
---
 .../accumulo/core/client/BatchWriterConfig.java    | 25 ++++++++++++++++++----
 .../apache/accumulo/core/conf/ClientProperty.java  |  9 ++++----
 .../core/client/BatchWriterConfigTest.java         | 10 +++++++++
 3 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java b/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
index b27732f..cd400b1 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
@@ -17,6 +17,11 @@
 package org.apache.accumulo.core.client;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.conf.ClientProperty.BATCH_WRITER_DURABILITY;
+import static org.apache.accumulo.core.conf.ClientProperty.BATCH_WRITER_MAX_LATENCY_SEC;
+import static org.apache.accumulo.core.conf.ClientProperty.BATCH_WRITER_MAX_MEMORY_BYTES;
+import static org.apache.accumulo.core.conf.ClientProperty.BATCH_WRITER_MAX_TIMEOUT_SEC;
+import static org.apache.accumulo.core.conf.ClientProperty.BATCH_WRITER_MAX_WRITE_THREADS;
 
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -26,6 +31,7 @@ import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.accumulo.core.clientImpl.DurabilityImpl;
+import org.apache.accumulo.core.conf.ClientProperty;
 import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.util.StringUtils;
@@ -37,21 +43,32 @@ import org.apache.hadoop.util.StringUtils;
  */
 public class BatchWriterConfig implements Writable {
 
-  private static final Long DEFAULT_MAX_MEMORY = 50 * 1024 * 1024L;
+  private static final Long DEFAULT_MAX_MEMORY = Long
+      .parseLong(BATCH_WRITER_MAX_MEMORY_BYTES.getDefaultValue());
   private Long maxMemory = null;
 
-  private static final Long DEFAULT_MAX_LATENCY = 2 * 60 * 1000L;
+  private static final Long DEFAULT_MAX_LATENCY = TimeUnit.MILLISECONDS
+      .convert(Long.parseLong(BATCH_WRITER_MAX_LATENCY_SEC.getDefaultValue()), TimeUnit.SECONDS);
   private Long maxLatency = null;
 
-  private static final Long DEFAULT_TIMEOUT = Long.MAX_VALUE;
+  private static final Long DEFAULT_TIMEOUT = getDefaultTimeout();
   private Long timeout = null;
 
-  private static final Integer DEFAULT_MAX_WRITE_THREADS = 3;
+  private static final Integer DEFAULT_MAX_WRITE_THREADS = Integer
+      .parseInt(BATCH_WRITER_MAX_WRITE_THREADS.getDefaultValue());
   private Integer maxWriteThreads = null;
 
   private Durability durability = Durability.DEFAULT;
   private boolean isDurabilitySet = false;
 
+  private static Long getDefaultTimeout() {
+    Long def = Long.parseLong(BATCH_WRITER_MAX_TIMEOUT_SEC.getDefaultValue());
+    if (def.equals(0L))
+      return Long.MAX_VALUE;
+    else
+      return TimeUnit.MILLISECONDS.convert(def, TimeUnit.SECONDS);
+  }
+
   /**
    * Sets the maximum memory to batch before writing. The smaller this value, the more frequently
    * the {@link BatchWriter} will write.<br>
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/ClientProperty.java b/core/src/main/java/org/apache/accumulo/core/conf/ClientProperty.java
index 8801507..a1ef2c5 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ClientProperty.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ClientProperty.java
@@ -56,13 +56,14 @@ public enum ClientProperty {
   BATCH_WRITER_MAX_LATENCY_SEC("batch.writer.max.latency.sec", "120",
       "Max amount of time (in seconds) to hold data in memory before flushing it"),
   BATCH_WRITER_MAX_TIMEOUT_SEC("batch.writer.max.timeout.sec", "0",
-      "Max amount" + " of time (in seconds) an unresponsive server will be re-tried. An"
+      "Max amount of time (in seconds) an unresponsive server will be re-tried. An"
           + " exception is thrown when this timeout is exceeded. Set to zero for no timeout."),
   BATCH_WRITER_MAX_WRITE_THREADS("batch.writer.max.write.threads", "3",
       "Maximum number of threads to use for writing data to tablet servers."),
-  BATCH_WRITER_DURABILITY("batch.writer.durability", "default",
-      "Change the" + " durability for the BatchWriter session. To use the table's durability"
-          + " setting. use \"default\" which is the table's durability setting."),
+  BATCH_WRITER_DURABILITY("batch.writer.durability", "default", Property.TABLE_DURABILITY
+      .getDescription() + " Setting this property will "
+      + "change the durability for the BatchWriter session. A value of \"default\" will use
the "
+      + "table's durability setting. "),
 
   // Scanner
   SCANNER_BATCH_SIZE("scanner.batch.size", "1000",
diff --git a/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
b/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
index daf40f3..e5c0051 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
@@ -25,8 +25,10 @@ import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.accumulo.core.conf.ClientProperty;
 import org.junit.Test;
 
 public class BatchWriterConfigTest {
@@ -235,4 +237,12 @@ public class BatchWriterConfigTest {
     assertEquals(bwConfig.getMaxWriteThreads(), createdConfig.getMaxWriteThreads());
   }
 
+  @Test
+  public void countClientProps() {
+    // count the number in case one gets added to in one place but not the other
+    ClientProperty[] bwProps = Arrays.stream(ClientProperty.values())
+        .filter(c -> c.name().startsWith("BATCH_WRITER")).toArray(ClientProperty[]::new);
+    assertEquals(5, bwProps.length);
+  }
+
 }


Mime
View raw message