accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dlmar...@apache.org
Subject [04/39] accumulo git commit: ACCUMULO-4004: Added new property to close walogs after some time period.
Date Tue, 12 Apr 2016 13:58:21 GMT
ACCUMULO-4004: Added new property to close walogs after some time period.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0206d784
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0206d784
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0206d784

Branch: refs/heads/ACCUMULO-4173
Commit: 0206d7845bf8bba08e18373f1dbb8e9aaf6d6d60
Parents: f6bfe90
Author: Dave Marion <dlmarion@apache.org>
Authored: Thu Mar 31 15:53:04 2016 -0400
Committer: Dave Marion <dlmarion@apache.org>
Committed: Thu Mar 31 15:53:04 2016 -0400

----------------------------------------------------------------------
 .../java/org/apache/accumulo/core/conf/Property.java     |  1 +
 .../java/org/apache/accumulo/tserver/TabletServer.java   |  3 ++-
 .../apache/accumulo/tserver/log/TabletServerLogger.java  | 11 ++++++++---
 3 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/0206d784/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 9243494..2149ad9 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -209,6 +209,7 @@ public enum Property {
           + "must be made, which is slower. However opening too many files at once can cause
problems."),
   TSERV_WALOG_MAX_SIZE("tserver.walog.max.size", "1G", PropertyType.MEMORY,
       "The maximum size for each write-ahead log. See comment for property tserver.memory.maps.max"),
+  TSERV_WALOG_MAX_AGE("tserver.walog.max.age", "24h", PropertyType.TIMEDURATION, "The maximum
age for each write-ahead log."),
   TSERV_MAJC_DELAY("tserver.compaction.major.delay", "30s", PropertyType.TIMEDURATION,
       "Time a tablet server will sleep between checking which tablets need compaction."),
   TSERV_MAJC_THREAD_MAXOPEN("tserver.compaction.major.thread.files.open.max", "10", PropertyType.COUNT,

http://git-wip-us.apache.org/repos/asf/accumulo/blob/0206d784/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index ac8f2ec..38bd8ac 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -3567,11 +3567,12 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
     security = AuditedSecurityOperation.getInstance();
     clientAddress = HostAndPort.fromParts(hostname, 0);
     long walogMaxSize = getSystemConfiguration().getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE);
+    long walogMaxAge = getSystemConfiguration().getTimeInMillis(Property.TSERV_WALOG_MAX_AGE);
     long minBlockSize = CachedConfiguration.getInstance().getLong("dfs.namenode.fs-limits.min-block-size",
0);
     if (minBlockSize != 0 && minBlockSize > walogMaxSize)
       throw new RuntimeException("Unable to start TabletServer. Logger is set to use blocksize
" + walogMaxSize + " but hdfs minimum block size is "
           + minBlockSize + ". Either increase the " + Property.TSERV_WALOG_MAX_SIZE + " or
decrease dfs.namenode.fs-limits.min-block-size in hdfs-site.xml.");
-    logger = new TabletServerLogger(this, walogMaxSize);
+    logger = new TabletServerLogger(this, walogMaxSize, walogMaxAge);
 
     try {
       AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig(new
Iterable<Entry<String,String>>() {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/0206d784/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
index c864c78..158fdbd 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
@@ -56,6 +56,7 @@ public class TabletServerLogger {
 
   private final AtomicLong logSizeEstimate = new AtomicLong();
   private final long maxSize;
+  private final long maxAge;
 
   private final TabletServer tserver;
 
@@ -76,6 +77,8 @@ public class TabletServerLogger {
 
   private final AtomicInteger seqGen = new AtomicInteger();
 
+  private long createTime = 0;
+
   private static boolean enabled(Tablet tablet) {
     return tablet.getTableConfiguration().getBoolean(Property.TABLE_WALOG_ENABLED);
   }
@@ -127,9 +130,10 @@ public class TabletServerLogger {
     }
   }
 
-  public TabletServerLogger(TabletServer tserver, long maxSize) {
+  public TabletServerLogger(TabletServer tserver, long maxSize, long maxAge) {
     this.tserver = tserver;
     this.maxSize = maxSize;
+    this.maxAge = maxAge;
   }
 
   private int initializeLoggers(final List<DfsLogger> copy) throws IOException {
@@ -185,6 +189,7 @@ public class TabletServerLogger {
       alog.open(tserver.getClientAddressString());
       loggers.add(alog);
       logSetId.incrementAndGet();
+      this.createTime = System.currentTimeMillis();
       return;
     } catch (Exception t) {
       throw new RuntimeException(t);
@@ -312,11 +317,11 @@ public class TabletServerLogger {
         });
       }
     }
-    // if the log gets too big, reset it .. grab the write lock first
+    // if the log gets too big or too old, reset it .. grab the write lock first
     logSizeEstimate.addAndGet(4 * 3); // event, tid, seq overhead
     testLockAndRun(logSetLock, new TestCallWithWriteLock() {
       boolean test() {
-        return logSizeEstimate.get() > maxSize;
+        return (logSizeEstimate.get() > maxSize) || ((System.currentTimeMillis() - createTime)
> maxAge);
       }
 
       void withWriteLock() throws IOException {


Mime
View raw message