accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject git commit: ACCUMULO-2922 unit test LargestFirstMemoryManager
Date Wed, 18 Jun 2014 21:06:03 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master 5f24240e4 -> ffd2626a8


ACCUMULO-2922 unit test LargestFirstMemoryManager


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ffd2626a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ffd2626a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ffd2626a

Branch: refs/heads/master
Commit: ffd2626a8cc4df476eef9bfcd613896dce165a4d
Parents: 5f24240
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Wed Jun 18 17:04:25 2014 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Wed Jun 18 17:05:57 2014 -0400

----------------------------------------------------------------------
 .../tabletserver/LargestFirstMemoryManager.java |  11 +-
 .../LargestFirstMemoryManagerTest.java          | 262 +++++++++++++++++++
 2 files changed, 270 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/ffd2626a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
index b891ad6..d06d61e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
@@ -40,7 +40,8 @@ public class LargestFirstMemoryManager implements MemoryManager {
   private static final long ZERO_TIME = System.currentTimeMillis();
   private static final int TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER = 2;
   private static final double MAX_FLUSH_AT_ONCE_PERCENT = 0.20;
-  
+
+
   private long maxMemory = -1;
   private int maxConcurrentMincs;
   private int numWaitingMultiplier;
@@ -111,7 +112,7 @@ public class LargestFirstMemoryManager implements MemoryManager {
     maxObserved = 0;
   }
   
-  private long getMinCIdleThreshold(KeyExtent extent) {
+  protected long getMinCIdleThreshold(KeyExtent extent) {
     Text tableId = extent.getTableId();
     if (!mincIdleThresholds.containsKey(tableId))
       mincIdleThresholds.put(tableId, config.getTableConfiguration(tableId.toString()).getTimeInMillis(Property.TABLE_MINC_COMPACT_IDLETIME));
@@ -131,7 +132,7 @@ public class LargestFirstMemoryManager implements MemoryManager {
     
     TreeMap<Long,TabletInfo> largestMemTablets = new LargestMap(maxMinCs);
     final TreeMap<Long,TabletInfo> largestIdleMemTablets = new LargestMap(maxConcurrentMincs);
-    final long now = System.currentTimeMillis();
+    final long now = currentTimeMillis();
     
     long ingestMemory = 0;
     long compactionMemory = 0;
@@ -222,6 +223,10 @@ public class LargestFirstMemoryManager implements MemoryManager {
     return result;
   }
   
+  protected long currentTimeMillis() {
+    return System.currentTimeMillis();
+  }
+
   @Override
   public void tabletClosed(KeyExtent extent) {}
   

http://git-wip-us.apache.org/repos/asf/accumulo/blob/ffd2626a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
new file mode 100644
index 0000000..a9c609b
--- /dev/null
+++ b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tabletserver;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class LargestFirstMemoryManagerTest {
+  
+  private static final long ZERO = System.currentTimeMillis();
+  private static final long LATER = ZERO + 20 * 60 * 1000;
+  private static final long ONE_GIG = 1024*1024*1024;
+  private static final long HALF_GIG = ONE_GIG/2;
+  private static final long QGIG = ONE_GIG/4;
+  private static final long ONE_MINUTE = 60 * 1000;
+
+  @Test
+  public void test() throws Exception {
+    LargestFirstMemoryManagerUnderTest mgr = new LargestFirstMemoryManagerUnderTest();
+    Instance instance = new MockInstance();
+    ServerConfiguration config = new ServerConfiguration(instance);
+    mgr.init(config);
+    MemoryManagementActions result;
+    // nothing to do
+    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1000, 0), t(k("y"), ZERO,
2000, 0)));
+    assertEquals(0, result.tabletsToMinorCompact.size());
+    // one tablet is really big
+    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, ONE_GIG, 0), t(k("y"),
ZERO, 2000, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
+    // one tablet is idle
+    mgr.currentTime = LATER;
+    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER,
2000, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
+    // one tablet is idle, but one is really big
+    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER,
ONE_GIG, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("y"), result.tabletsToMinorCompact.get(0));
+    // lots of work to do
+    mgr = new LargestFirstMemoryManagerUnderTest();
+    mgr.init(config);
+    result = mgr.getMemoryManagementActions(tablets(
+        t(k("a"), ZERO, HALF_GIG, 0),
+        t(k("b"), ZERO, HALF_GIG+1, 0),
+        t(k("c"), ZERO, HALF_GIG+2, 0),
+        t(k("d"), ZERO, HALF_GIG+3, 0),
+        t(k("e"), ZERO, HALF_GIG+4, 0),
+        t(k("f"), ZERO, HALF_GIG+5, 0),
+        t(k("g"), ZERO, HALF_GIG+6, 0),
+        t(k("h"), ZERO, HALF_GIG+7, 0),
+        t(k("i"), ZERO, HALF_GIG+8, 0))
+        );
+    assertEquals(2, result.tabletsToMinorCompact.size());
+    assertEquals(k("i"), result.tabletsToMinorCompact.get(0));
+    assertEquals(k("h"), result.tabletsToMinorCompact.get(1));
+    // one finished, one in progress, one filled up
+    mgr = new LargestFirstMemoryManagerUnderTest();
+    mgr.init(config);
+    result = mgr.getMemoryManagementActions(tablets(
+        t(k("a"), ZERO, HALF_GIG, 0),
+        t(k("b"), ZERO, HALF_GIG+1, 0),
+        t(k("c"), ZERO, HALF_GIG+2, 0),
+        t(k("d"), ZERO, HALF_GIG+3, 0),
+        t(k("e"), ZERO, HALF_GIG+4, 0),
+        t(k("f"), ZERO, HALF_GIG+5, 0),
+        t(k("g"), ZERO, ONE_GIG, 0),
+        t(k("h"), ZERO, 0, HALF_GIG+7),
+        t(k("i"), ZERO, 0, 0))
+        );
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
+    // memory is very full, lots of candidates
+    result = mgr.getMemoryManagementActions(tablets(
+        t(k("a"), ZERO, HALF_GIG, 0),
+        t(k("b"), ZERO, ONE_GIG+1, 0),
+        t(k("c"), ZERO, ONE_GIG+2, 0),
+        t(k("d"), ZERO, ONE_GIG+3, 0),
+        t(k("e"), ZERO, ONE_GIG+4, 0),
+        t(k("f"), ZERO, ONE_GIG+5, 0),
+        t(k("g"), ZERO, ONE_GIG+6, 0),
+        t(k("h"), ZERO, 0, 0),
+        t(k("i"), ZERO, 0, 0))
+        );
+    assertEquals(2, result.tabletsToMinorCompact.size());
+    assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
+    assertEquals(k("f"), result.tabletsToMinorCompact.get(1));
+    // only have two compactors, still busy
+    result = mgr.getMemoryManagementActions(tablets(
+        t(k("a"), ZERO, HALF_GIG, 0),
+        t(k("b"), ZERO, ONE_GIG+1, 0),
+        t(k("c"), ZERO, ONE_GIG+2, 0),
+        t(k("d"), ZERO, ONE_GIG+3, 0),
+        t(k("e"), ZERO, ONE_GIG+4, 0),
+        t(k("f"), ZERO, ONE_GIG, ONE_GIG+5),
+        t(k("g"), ZERO, ONE_GIG, ONE_GIG+6),
+        t(k("h"), ZERO, 0, 0),
+        t(k("i"), ZERO, 0, 0))
+        );
+    assertEquals(0, result.tabletsToMinorCompact.size());
+    // finished one 
+    result = mgr.getMemoryManagementActions(tablets(
+        t(k("a"), ZERO, HALF_GIG, 0),
+        t(k("b"), ZERO, ONE_GIG+1, 0),
+        t(k("c"), ZERO, ONE_GIG+2, 0),
+        t(k("d"), ZERO, ONE_GIG+3, 0),
+        t(k("e"), ZERO, ONE_GIG+4, 0),
+        t(k("f"), ZERO, ONE_GIG, ONE_GIG+5),
+        t(k("g"), ZERO, ONE_GIG, 0),
+        t(k("h"), ZERO, 0, 0),
+        t(k("i"), ZERO, 0, 0))
+        );
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("e"), result.tabletsToMinorCompact.get(0));
+
+    // many are running: do nothing
+    mgr = new LargestFirstMemoryManagerUnderTest();
+    mgr.init(config);
+    result = mgr.getMemoryManagementActions(tablets(
+        t(k("a"), ZERO, HALF_GIG, 0),
+        t(k("b"), ZERO, HALF_GIG+1, 0),
+        t(k("c"), ZERO, HALF_GIG+2, 0),
+        t(k("d"), ZERO, 0, HALF_GIG),
+        t(k("e"), ZERO, 0, HALF_GIG),
+        t(k("f"), ZERO, 0, HALF_GIG),
+        t(k("g"), ZERO, 0, HALF_GIG),
+        t(k("i"), ZERO, 0, HALF_GIG),
+        t(k("j"), ZERO, 0, HALF_GIG),
+        t(k("k"), ZERO, 0, HALF_GIG),
+        t(k("l"), ZERO, 0, HALF_GIG),
+        t(k("m"), ZERO, 0, HALF_GIG)
+        ));
+    assertEquals(0, result.tabletsToMinorCompact.size());
+    
+    // observe adjustment:
+    mgr = new LargestFirstMemoryManagerUnderTest();
+    mgr.init(config);
+    // compact the largest
+    result = mgr.getMemoryManagementActions(tablets(
+        t(k("a"), ZERO, QGIG, 0),
+        t(k("b"), ZERO, QGIG+1, 0),
+        t(k("c"), ZERO, QGIG+2, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("c"), result.tabletsToMinorCompact.get(0));
+    // show that it is compacting... do nothing
+    result = mgr.getMemoryManagementActions(tablets(
+        t(k("a"), ZERO, QGIG, 0),
+        t(k("b"), ZERO, QGIG+1, 0),
+        t(k("c"), ZERO, 0, QGIG+2)));
+    assertEquals(0, result.tabletsToMinorCompact.size());
+    // not going to bother compacting any more
+    mgr.currentTime += ONE_MINUTE;
+    result = mgr.getMemoryManagementActions(tablets(
+        t(k("a"), ZERO, QGIG, 0),
+        t(k("b"), ZERO, QGIG+1, 0),
+        t(k("c"), ZERO, 0, QGIG+2)));
+    assertEquals(0, result.tabletsToMinorCompact.size());
+    // now do nothing
+    mgr.currentTime += ONE_MINUTE;
+    result = mgr.getMemoryManagementActions(tablets(
+        t(k("a"), ZERO, QGIG, 0),
+        t(k("b"), ZERO, 0, 0),
+        t(k("c"), ZERO, 0, 0)));
+    assertEquals(0, result.tabletsToMinorCompact.size());
+    // on no! more data, this time we compact because we've adjusted
+    mgr.currentTime += ONE_MINUTE;
+    result = mgr.getMemoryManagementActions(tablets(
+        t(k("a"), ZERO, QGIG, 0),
+        t(k("b"), ZERO, QGIG+1, 0),
+        t(k("c"), ZERO, 0, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("b"), result.tabletsToMinorCompact.get(0));
+  }
+  
+  private static class LargestFirstMemoryManagerUnderTest extends LargestFirstMemoryManager
{
+
+    public long currentTime = ZERO;
+
+    @Override
+    protected long currentTimeMillis() {
+      return currentTime;
+    }
+
+    @Override
+    protected long getMinCIdleThreshold(KeyExtent extent) {
+      return 15 * 60 * 1000;
+    }
+    
+  }
+  
+  private static KeyExtent k(String endRow) {
+    return new KeyExtent(new Text("1"), new Text(endRow), null);
+  }
+  
+  private static class TestTabletState implements TabletState {
+    
+    private final KeyExtent extent;
+    private final long lastCommit;
+    private final long memSize;
+    private final long compactingSize;
+    
+    TestTabletState(KeyExtent extent, long commit, long memsize, long compactingTableSize)
{
+      this.extent = extent;
+      this.lastCommit = commit;
+      this.memSize = memsize;
+      this.compactingSize = compactingTableSize;
+    }
+
+    @Override
+    public KeyExtent getExtent() {
+      return extent;
+    }
+
+    @Override
+    public long getLastCommitTime() {
+      return lastCommit;
+    }
+
+    @Override
+    public long getMemTableSize() {
+      return memSize;
+    }
+
+    @Override
+    public long getMinorCompactingMemTableSize() {
+      return compactingSize;
+    }
+    
+  }
+
+  private TabletState t(KeyExtent ke, long lastCommit, long memSize, long compactingSize)
{
+    return new TestTabletState(ke, lastCommit, memSize, compactingSize);
+  }
+  
+  private static List<TabletState> tablets(TabletState ... states) {
+    return Arrays.asList(states);
+  }
+  
+}


Mime
View raw message