accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [5/9] accumulo git commit: ACCUMULO-3777 Add existence check to MemoryManager
Date Sun, 10 May 2015 21:06:23 GMT
ACCUMULO-3777 Add existence check to MemoryManager

Somehow we got into a situation where we were checking tablets
for a deleted table. Add a quick check for existence before
running the extent through the memory manager.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/719aac43
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/719aac43
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/719aac43

Branch: refs/heads/1.7
Commit: 719aac433e0428f969130820427d62404d5642e2
Parents: 757c1cb
Author: Josh Elser <elserj@apache.org>
Authored: Sun May 10 01:09:19 2015 -0400
Committer: Josh Elser <elserj@apache.org>
Committed: Sun May 10 01:09:19 2015 -0400

----------------------------------------------------------------------
 .../tabletserver/LargestFirstMemoryManager.java | 13 ++++
 .../LargestFirstMemoryManagerTest.java          | 67 ++++++++++++++++++++
 2 files changed, 80 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/719aac43/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
index 323e59d..a39c8b6 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
@@ -22,6 +22,8 @@ import java.util.List;
 import java.util.Map.Entry;
 import java.util.TreeMap;
 
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.Tables;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.impl.KeyExtent;
 import org.apache.accumulo.server.conf.ServerConfiguration;
@@ -143,11 +145,16 @@ public class LargestFirstMemoryManager implements MemoryManager {
     return mincIdleThresholds.get(tableId);
   }
 
+  boolean tableExists(Instance instance, String tableId) {
+    return Tables.exists(instance, tableId);
+  }
+
   @Override
   public MemoryManagementActions getMemoryManagementActions(List<TabletState> tablets)
{
     if (maxMemory < 0)
       throw new IllegalStateException("need to initialize " + LargestFirstMemoryManager.class.getName());
 
+    final Instance instance = config.getInstance();
     final int maxMinCs = maxConcurrentMincs * numWaitingMultiplier;
 
     mincIdleThresholds.clear();
@@ -164,6 +171,12 @@ public class LargestFirstMemoryManager implements MemoryManager {
 
     // find the largest and most idle tablets
     for (TabletState ts : tablets) {
+      // Make sure that the table still exists
+      if (!tableExists(instance, ts.getExtent().getTableId().toString())) {
+        log.info("Ignoring extent for deleted table: " + ts.getExtent());
+        continue;
+      }
+
       final long memTabletSize = ts.getMemTableSize();
       final long minorCompactingSize = ts.getMinorCompactingMemTableSize();
       final long idleTime = now - Math.max(ts.getLastCommitTime(), ZERO_TIME);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/719aac43/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
index 7c15cfc..b08b980 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
@@ -33,6 +33,8 @@ import org.apache.accumulo.server.conf.TableConfiguration;
 import org.apache.hadoop.io.Text;
 import org.junit.Test;
 
+import com.google.common.base.Function;
+
 public class LargestFirstMemoryManagerTest {
 
   private static final long ZERO = System.currentTimeMillis();
@@ -160,6 +162,52 @@ public class LargestFirstMemoryManagerTest {
     assertEquals(k("b"), result.tabletsToMinorCompact.get(0));
   }
 
+  @Test
+  public void testDeletedTable() throws Exception {
+    final String deletedTableId = "1";
+    Function<String,Boolean> existenceCheck = new Function<String,Boolean>()
{
+      public Boolean apply(String tableId) {
+        return !deletedTableId.equals(tableId);
+      }
+    };
+    LargestFirstMemoryManagerWithExistenceCheck mgr = new LargestFirstMemoryManagerWithExistenceCheck(existenceCheck);
+    ServerConfiguration config = new ServerConfiguration() {
+      ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance());
+
+      @Override
+      public AccumuloConfiguration getConfiguration() {
+        return DefaultConfiguration.getInstance();
+      }
+
+      @Override
+      public TableConfiguration getTableConfiguration(String tableId) {
+        return delegate.getTableConfiguration(tableId);
+      }
+
+      @Override
+      public TableConfiguration getTableConfiguration(KeyExtent extent) {
+        return delegate.getTableConfiguration(extent);
+      }
+
+      @Override
+      public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) {
+        return delegate.getNamespaceConfiguration(namespaceId);
+      }
+
+      @Override
+      public Instance getInstance() {
+        return delegate.getInstance();
+      }
+    };
+    mgr.init(config);
+    MemoryManagementActions result;
+    // one tablet is really big and the other is for a nonexistent table
+    KeyExtent extent = new KeyExtent(new Text("2"), new Text("j"), null);
+    result = mgr.getMemoryManagementActions(tablets(t(extent, ZERO, ONE_GIG, 0), t(k("j"),
ZERO, ONE_GIG, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(extent, result.tabletsToMinorCompact.get(0));
+  }
+
   private static class LargestFirstMemoryManagerUnderTest extends LargestFirstMemoryManager
{
 
     public long currentTime = ZERO;
@@ -174,6 +222,25 @@ public class LargestFirstMemoryManagerTest {
       return 15 * 60 * 1000;
     }
 
+    @Override
+    boolean tableExists(Instance instance, String tableId) {
+      return true;
+    }
+  }
+
+  private static class LargestFirstMemoryManagerWithExistenceCheck extends LargestFirstMemoryManagerUnderTest
{
+
+    Function<String,Boolean> existenceCheck;
+
+    public LargestFirstMemoryManagerWithExistenceCheck(Function<String,Boolean> existenceCheck)
{
+      super();
+      this.existenceCheck = existenceCheck;
+    }
+
+    @Override
+    boolean tableExists(Instance instance, String tableId) {
+      return existenceCheck.apply(tableId);
+    }
   }
 
   private static KeyExtent k(String endRow) {


Mime
View raw message