accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edcole...@apache.org
Subject [accumulo] branch 1.9 updated: Fix #1345 - Allow gc initiated compaction as option. (#1352)
Date Thu, 12 Sep 2019 19:06:11 GMT
This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch 1.9
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/1.9 by this push:
     new 073873d  Fix #1345 - Allow gc initiated compaction as option. (#1352)
073873d is described below

commit 073873db494fc5467a0b8ededba53d277d41015a
Author: EdColeman <dev1@etcoleman.com>
AuthorDate: Thu Sep 12 15:06:06 2019 -0400

    Fix #1345 - Allow gc initiated compaction as option. (#1352)
    
    * Fix #1345 - Allow gc initiated compaction as option.
    
    Added a property - gc.use.full.compaction that defaults to current behaviour. When false,
    performs a blocking flush instead.
    
    * fix type in property description.
    
    * wip: use enum, compact, flush or none
    
    * fix test failure by adding PropertyTest
    
    * added timing log messsgae
    
    * update durations to use nanoseconds
---
 .../org/apache/accumulo/core/conf/Property.java    |  5 ++
 .../apache/accumulo/core/conf/PropertyType.java    |  3 ++
 .../accumulo/core/conf/PropertyTypeTest.java       |  6 +++
 .../apache/accumulo/gc/SimpleGarbageCollector.java | 60 ++++++++++++++++------
 4 files changed, 59 insertions(+), 15 deletions(-)

diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 494f6b2..5e0dd91 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -556,6 +556,11 @@ public enum Property {
       "Archive any files/directories instead of moving to the HDFS trash or deleting."),
   GC_TRACE_PERCENT("gc.trace.percent", "0.01", PropertyType.FRACTION,
       "Percent of gc cycles to trace"),
+  GC_USE_FULL_COMPACTION("gc.post.metadata.action", "compact", PropertyType.GC_POST_ACTION,
+      "When the gc runs it can make a lot of changes to the metadata, on completion, "
+          + " to force the changes to be written to disk, the metadata and root tables can
be flushed"
+          + " and possibly compacted. Legal values are: compact - which both flushes and
compacts the"
+          + " metadata; flush - which flushes only (compactions may be triggered if required);
or none"),
 
   // properties that are specific to the monitor server behavior
   MONITOR_PREFIX("monitor.", null, PropertyType.PREFIX,
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java b/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
index a3f0eba..13d403f 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
@@ -112,6 +112,9 @@ public enum PropertyType {
   DURABILITY("durability", in(true, null, "none", "log", "flush", "sync"),
       "One of 'none', 'log', 'flush' or 'sync'."),
 
+  GC_POST_ACTION("gc_post_action", in(true, null, "none", "flush", "compact"),
+      "One of 'none', 'flush', or 'compact'."),
+
   STRING("string", Predicates.<String>alwaysTrue(),
       "An arbitrary string of characters whose format is unspecified and"
           + " interpreted based on the context of the property to which it applies."),
diff --git a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java
index 3dcf0fd..8505afe 100644
--- a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java
@@ -160,6 +160,12 @@ public class PropertyTypeTest {
   }
 
   @Test
+  public void testTypeGC_POST_ACTION() {
+    valid(null, "none", "flush", "compact");
+    invalid("", "other");
+  }
+
+  @Test
   public void testTypeFRACTION() {
     valid(null, "1", "0", "1.0", "25%", "2.5%", "10.2E-3", "10.2E-3%", ".3");
     invalid("", "other", "20%%", "-0.3", "3.6a", "%25", "3%a");
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 24ce61c..59e0eed 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -44,6 +44,7 @@ import org.apache.accumulo.core.client.MutationsRejectedException;
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.conf.SiteConfiguration;
 import org.apache.accumulo.core.data.Key;
@@ -183,14 +184,19 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements
Ifa
     this.opts = opts;
     this.fs = fs;
 
-    long gcDelay = getConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY);
-    log.info("start delay: " + getStartDelay() + " milliseconds");
-    log.info("time delay: " + gcDelay + " milliseconds");
-    log.info("safemode: " + opts.safeMode);
-    log.info("verbose: " + opts.verbose);
-    log.info("memory threshold: " + CANDIDATE_MEMORY_PERCENTAGE + " of "
-        + Runtime.getRuntime().maxMemory() + " bytes");
-    log.info("delete threads: " + getNumDeleteThreads());
+    final AccumuloConfiguration conf = getConfiguration();
+
+    final long gcDelay = conf.getTimeInMillis(Property.GC_CYCLE_DELAY);
+    final String useFullCompaction = conf.get(Property.GC_USE_FULL_COMPACTION);
+
+    log.info("start delay: {} milliseconds", getStartDelay());
+    log.info("time delay: {} milliseconds", gcDelay);
+    log.info("safemode: {}", opts.safeMode);
+    log.info("verbose: {}", opts.verbose);
+    log.info("memory threshold: {} of {} bytes", CANDIDATE_MEMORY_PERCENTAGE,
+        Runtime.getRuntime().maxMemory());
+    log.info("delete threads: {}", getNumDeleteThreads());
+    log.info("gc post metadata action: {}", useFullCompaction);
   }
 
   /**
@@ -540,7 +546,6 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements
Ifa
   }
 
   private void run() {
-    long tStart, tStop;
 
     // Sleep for an initial period, giving the master time to start up and
     // old data files to be unused
@@ -586,7 +591,7 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements
Ifa
       Trace.on("gc", sampler);
 
       Span gcSpan = Trace.start("loop");
-      tStart = System.currentTimeMillis();
+      final long tStart = System.nanoTime();
       try {
         System.gc(); // make room
 
@@ -608,8 +613,9 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements
Ifa
         log.error("{}", e.getMessage(), e);
       }
 
-      tStop = System.currentTimeMillis();
-      log.info(String.format("Collect cycle took %.2f seconds", ((tStop - tStart) / 1000.0)));
+      final long tStop = System.nanoTime();
+      log.info(String.format("Collect cycle took %.2f seconds",
+          (TimeUnit.NANOSECONDS.toMillis(tStop - tStart) / 1000.0)));
 
       // We want to prune references to fully-replicated WALs from the replication table
which are
       // no longer referenced in the metadata table
@@ -638,11 +644,35 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements
Ifa
       }
       gcSpan.stop();
 
-      // we just made a lot of metadata changes: flush them out
+      // We just made a lot of metadata changes. Flush them out.
+      // Either with flush and full compaction, or flush only and allow automatic triggering
+      // of any necessary compactions.
       try {
         Connector connector = getConnector();
-        connector.tableOperations().compact(MetadataTable.NAME, null, null, true, true);
-        connector.tableOperations().compact(RootTable.NAME, null, null, true, true);
+
+        final long actionStart = System.nanoTime();
+
+        String action = getConfiguration().get(Property.GC_USE_FULL_COMPACTION);
+        log.debug("gc post action {} started", action);
+
+        switch (action) {
+          case "compact":
+            connector.tableOperations().compact(MetadataTable.NAME, null, null, true, true);
+            connector.tableOperations().compact(RootTable.NAME, null, null, true, true);
+            break;
+          case "flush":
+            connector.tableOperations().flush(MetadataTable.NAME, null, null, true);
+            connector.tableOperations().flush(RootTable.NAME, null, null, true);
+            break;
+          default:
+            log.trace("\'none - no action\' or invalid value provided: {}", action);
+        }
+
+        final long actionComplete = System.nanoTime();
+
+        log.info("gc post action {} completed in {} seconds", action, String.format("%.2f",
+            (TimeUnit.NANOSECONDS.toMillis(actionComplete - actionStart) / 1000.0)));
+
       } catch (Exception e) {
         log.warn("{}", e.getMessage(), e);
       }


Mime
View raw message