accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [accumulo] branch master updated: Change max walogs from table to tserver property. (#1008)
Date Mon, 04 Mar 2019 21:03:49 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/master by this push:
     new 8b6aaa5  Change max walogs from table to tserver property. (#1008)
8b6aaa5 is described below

commit 8b6aaa57126ed1f1fe6e8485c7bd559ff64f9b54
Author: Keith Turner <kturner@apache.org>
AuthorDate: Mon Mar 4 16:03:44 2019 -0500

    Change max walogs from table to tserver property. (#1008)
    
    After the changes in #860 having a tablet property for max write ahead
    logs no longer makes sense.  This should be a tserver property because
    the behavior controls the max walogs per tserver.  This commit replaces
    the table property with a tablet server property.
---
 .../apache/accumulo/core/conf/AccumuloConfiguration.java    | 13 +++++++++++++
 .../main/java/org/apache/accumulo/core/conf/Property.java   |  9 +++++++--
 .../java/org/apache/accumulo/tserver/tablet/Tablet.java     |  4 +++-
 .../accumulo/test/performance/RollWALPerformanceIT.java     |  2 +-
 4 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
index 64a05a9..ec6cb0d 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
@@ -89,6 +89,19 @@ public abstract class AccumuloConfiguration implements Iterable<Entry<String,Str
   public abstract String get(Property property);
 
   /**
+   * Given a property and a deprecated property determine which one to use base on which
one is set.
+   */
+  public Property resolve(Property property, Property deprecatedProperty) {
+    if (isPropertySet(property, true)) {
+      return property;
+    } else if (isPropertySet(deprecatedProperty, true)) {
+      return deprecatedProperty;
+    } else {
+      return property;
+    }
+  }
+
+  /**
    * Returns property key/value pairs in this configuration. The pairs include those defined
in this
    * configuration which pass the given filter, and those supplied from the parent configuration
    * which are not included from here.
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 9e6029b..9529b1a 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -314,6 +314,10 @@ public enum Property {
           + " When there are more RFiles than this setting multiple passes must be"
           + " made, which is slower. However opening too many RFiles at once can cause"
           + " problems."),
+  TSERV_WALOG_MAX_REFERENCED("tserver.walog.max.referenced", "3", PropertyType.COUNT,
+      "When a tablet server has more than this many write ahead logs, any tablet referencing
older "
+          + "logs over this threshold is minor compacted.  Also any tablet referencing this
many "
+          + "logs or more will be compacted."),
   TSERV_WALOG_MAX_SIZE("tserver.walog.max.size", "1g", PropertyType.BYTES,
       "The maximum size for each write-ahead log. See comment for property"
           + " tserver.memory.maps.max"),
@@ -599,9 +603,10 @@ public enum Property {
       "A tablet is split when the combined size of RFiles exceeds this amount."),
   TABLE_MAX_END_ROW_SIZE("table.split.endrow.size.max", "10K", PropertyType.BYTES,
       "Maximum size of end row"),
+  @Deprecated
+  @ReplacedBy(property = Property.TSERV_WALOG_MAX_REFERENCED)
   TABLE_MINC_LOGS_MAX("table.compaction.minor.logs.threshold", "3", PropertyType.COUNT,
-      "When there are more than this many write-ahead logs against a tablet, it"
-          + " will be minor compacted. See comment for property" + " tserver.memory.maps.max"),
+      "This property is deprecated since 2.0.0."),
   TABLE_MINC_COMPACT_IDLETIME("table.compaction.minor.idle", "5m", PropertyType.TIMEDURATION,
       "After a tablet has been idle (no mutations) for this time period it may have its "
           + "in-memory map flushed to disk in a minor compaction. There is no guarantee an
idle "
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index d8b10e2..780f78c 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -2463,7 +2463,9 @@ public class Tablet {
   public void checkIfMinorCompactionNeededForLogs(List<DfsLogger> closedLogs) {
 
     // grab this outside of tablet lock.
-    int maxLogs = tableConfiguration.getCount(Property.TABLE_MINC_LOGS_MAX);
+    @SuppressWarnings("deprecation")
+    int maxLogs = tableConfiguration.getCount(tableConfiguration
+        .resolve(Property.TSERV_WALOG_MAX_REFERENCED, Property.TABLE_MINC_LOGS_MAX));
 
     String reason = null;
     synchronized (this) {
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
b/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
index ab78f28..f1981cb 100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
@@ -50,7 +50,7 @@ public class RollWALPerformanceIT extends ConfigurableMacBase {
   protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
     cfg.setProperty(Property.TSERV_WAL_REPLICATION, "1");
     cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "5M");
-    cfg.setProperty(Property.TABLE_MINC_LOGS_MAX, "100");
+    cfg.setProperty(Property.TSERV_WALOG_MAX_REFERENCED, "100");
     cfg.setProperty(Property.GC_CYCLE_START, "1s");
     cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
     cfg.useMiniDFS(true);


Mime
View raw message