accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dlmar...@apache.org
Subject accumulo git commit: ACCUMULO-4004: Add new property for WALog max age, close log when age is reached.
Date Tue, 29 Mar 2016 20:04:29 GMT
Repository: accumulo
Updated Branches:
  refs/heads/ACCUMULO-4004 [created] f7d22c395


ACCUMULO-4004: Add new property for WALog max age, close log when age is reached.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f7d22c39
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f7d22c39
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f7d22c39

Branch: refs/heads/ACCUMULO-4004
Commit: f7d22c395a27ced9fa382b44162371d8b464988d
Parents: 1d49030
Author: Dave Marion <dlmarion@apache.org>
Authored: Tue Mar 29 16:02:38 2016 -0400
Committer: Dave Marion <dlmarion@apache.org>
Committed: Tue Mar 29 16:02:38 2016 -0400

----------------------------------------------------------------------
 .../java/org/apache/accumulo/core/conf/Property.java     |  1 +
 .../java/org/apache/accumulo/tserver/TabletServer.java   |  3 ++-
 .../apache/accumulo/tserver/log/TabletServerLogger.java  | 11 ++++++++---
 3 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7d22c39/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index b8311a9..e8537ac 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -254,6 +254,7 @@ public enum Property {
           + "must be made, which is slower. However opening too many files at once can cause
problems."),
   TSERV_WALOG_MAX_SIZE("tserver.walog.max.size", "1G", PropertyType.MEMORY,
       "The maximum size for each write-ahead log. See comment for property tserver.memory.maps.max"),
+  TSERV_WALOG_MAX_AGE("tserver.walog.max.age", "24h", PropertyType.TIMEDURATION, "The maximum
age for each write-ahead log."),
 
   TSERV_WALOG_TOLERATED_CREATION_FAILURES("tserver.walog.tolerated.creation.failures", "50",
PropertyType.COUNT,
       "The maximum number of failures tolerated when creating a new WAL file within the period
specified by tserver.walog.failures.period."

http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7d22c39/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index d19dfa9..be52d5b 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -354,6 +354,7 @@ public class TabletServer extends AccumuloServerContext implements Runnable
{
     }, 5000, 5000);
 
     final long walogMaxSize = aconf.getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE);
+    final long walogMaxAge = aconf.getTimeInMillis(Property.TSERV_WALOG_MAX_AGE);
     final long minBlockSize = CachedConfiguration.getInstance().getLong("dfs.namenode.fs-limits.min-block-size",
0);
     if (minBlockSize != 0 && minBlockSize > walogMaxSize)
       throw new RuntimeException("Unable to start TabletServer. Logger is set to use blocksize
" + walogMaxSize + " but hdfs minimum block size is "
@@ -367,7 +368,7 @@ public class TabletServer extends AccumuloServerContext implements Runnable
{
     final RetryFactory walCreationRetryFactory = new RetryFactory(toleratedWalCreationFailures,
walCreationFailureRetryIncrement,
         walCreationFailureRetryIncrement, walCreationFailureRetryMax);
 
-    logger = new TabletServerLogger(this, walogMaxSize, syncCounter, flushCounter, walCreationRetryFactory);
+    logger = new TabletServerLogger(this, walogMaxSize, syncCounter, flushCounter, walCreationRetryFactory,
walogMaxAge);
     this.resourceManager = new TabletServerResourceManager(this, fs);
     this.security = AuditedSecurityOperation.getInstance(this);
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7d22c39/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
index b76b9cd..7b52857 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
@@ -74,6 +74,7 @@ public class TabletServerLogger {
 
   private final AtomicLong logSizeEstimate = new AtomicLong();
   private final long maxSize;
+  private final long maxAge;
 
   private final TabletServer tserver;
 
@@ -102,6 +103,8 @@ public class TabletServerLogger {
   private final RetryFactory retryFactory;
   private Retry retry = null;
 
+  private long createTime = 0;
+
   static private abstract class TestCallWithWriteLock {
     abstract boolean test();
 
@@ -145,13 +148,14 @@ public class TabletServerLogger {
     }
   }
 
-  public TabletServerLogger(TabletServer tserver, long maxSize, AtomicLong syncCounter, AtomicLong
flushCounter, RetryFactory retryFactory) {
+  public TabletServerLogger(TabletServer tserver, long maxSize, AtomicLong syncCounter, AtomicLong
flushCounter, RetryFactory retryFactory, long maxAge) {
     this.tserver = tserver;
     this.maxSize = maxSize;
     this.syncCounter = syncCounter;
     this.flushCounter = flushCounter;
     this.retryFactory = retryFactory;
     this.retry = null;
+    this.maxAge = maxAge;
   }
 
   private DfsLogger initializeLoggers(final AtomicInteger logIdOut) throws IOException {
@@ -224,6 +228,7 @@ public class TabletServerLogger {
           retry = null;
         }
 
+        this.createTime = System.currentTimeMillis();
         return;
       } else {
         throw new RuntimeException("Error: unexpected type seen: " + next);
@@ -434,12 +439,12 @@ public class TabletServerLogger {
         });
       }
     }
-    // if the log gets too big, reset it .. grab the write lock first
+    // if the log gets too big or too old, reset it .. grab the write lock first
     logSizeEstimate.addAndGet(4 * 3); // event, tid, seq overhead
     testLockAndRun(logIdLock, new TestCallWithWriteLock() {
       @Override
       boolean test() {
-        return logSizeEstimate.get() > maxSize;
+        return (logSizeEstimate.get() > maxSize) || ((System.currentTimeMillis() - createTime)
> maxAge);
       }
 
       @Override


Mime
View raw message