Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8DC481734E for ; Sun, 10 May 2015 21:06:19 +0000 (UTC) Received: (qmail 61276 invoked by uid 500); 10 May 2015 21:06:19 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 61159 invoked by uid 500); 10 May 2015 21:06:19 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 60955 invoked by uid 99); 10 May 2015 21:06:19 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 10 May 2015 21:06:19 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3E375E17C0; Sun, 10 May 2015 21:06:19 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@accumulo.apache.org Date: Sun, 10 May 2015 21:06:24 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [6/9] accumulo git commit: ACCUMULO-3777 Add existence check to MemoryManager ACCUMULO-3777 Add existence check to MemoryManager Somehow we got into a situation where we were checking tablets for a deleted table. Add a quick check for existence before running the extent through the memory manager. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/719aac43 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/719aac43 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/719aac43 Branch: refs/heads/master Commit: 719aac433e0428f969130820427d62404d5642e2 Parents: 757c1cb Author: Josh Elser Authored: Sun May 10 01:09:19 2015 -0400 Committer: Josh Elser Committed: Sun May 10 01:09:19 2015 -0400 ---------------------------------------------------------------------- .../tabletserver/LargestFirstMemoryManager.java | 13 ++++ .../LargestFirstMemoryManagerTest.java | 67 ++++++++++++++++++++ 2 files changed, 80 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/719aac43/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java index 323e59d..a39c8b6 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java @@ -22,6 +22,8 @@ import java.util.List; import java.util.Map.Entry; import java.util.TreeMap; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.impl.Tables; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.impl.KeyExtent; import org.apache.accumulo.server.conf.ServerConfiguration; @@ -143,11 +145,16 @@ public class LargestFirstMemoryManager implements MemoryManager { return mincIdleThresholds.get(tableId); } + boolean tableExists(Instance instance, String tableId) { + return Tables.exists(instance, tableId); + } + @Override public MemoryManagementActions getMemoryManagementActions(List tablets) { if (maxMemory < 0) throw new IllegalStateException("need to initialize " + LargestFirstMemoryManager.class.getName()); + final Instance instance = config.getInstance(); final int maxMinCs = maxConcurrentMincs * numWaitingMultiplier; mincIdleThresholds.clear(); @@ -164,6 +171,12 @@ public class LargestFirstMemoryManager implements MemoryManager { // find the largest and most idle tablets for (TabletState ts : tablets) { + // Make sure that the table still exists + if (!tableExists(instance, ts.getExtent().getTableId().toString())) { + log.info("Ignoring extent for deleted table: " + ts.getExtent()); + continue; + } + final long memTabletSize = ts.getMemTableSize(); final long minorCompactingSize = ts.getMinorCompactingMemTableSize(); final long idleTime = now - Math.max(ts.getLastCommitTime(), ZERO_TIME); http://git-wip-us.apache.org/repos/asf/accumulo/blob/719aac43/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java index 7c15cfc..b08b980 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java @@ -33,6 +33,8 @@ import org.apache.accumulo.server.conf.TableConfiguration; import org.apache.hadoop.io.Text; import org.junit.Test; +import com.google.common.base.Function; + public class LargestFirstMemoryManagerTest { private static final long ZERO = System.currentTimeMillis(); @@ -160,6 +162,52 @@ public class LargestFirstMemoryManagerTest { assertEquals(k("b"), result.tabletsToMinorCompact.get(0)); } + @Test + public void testDeletedTable() throws Exception { + final String deletedTableId = "1"; + Function existenceCheck = new Function() { + public Boolean apply(String tableId) { + return !deletedTableId.equals(tableId); + } + }; + LargestFirstMemoryManagerWithExistenceCheck mgr = new LargestFirstMemoryManagerWithExistenceCheck(existenceCheck); + ServerConfiguration config = new ServerConfiguration() { + ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance()); + + @Override + public AccumuloConfiguration getConfiguration() { + return DefaultConfiguration.getInstance(); + } + + @Override + public TableConfiguration getTableConfiguration(String tableId) { + return delegate.getTableConfiguration(tableId); + } + + @Override + public TableConfiguration getTableConfiguration(KeyExtent extent) { + return delegate.getTableConfiguration(extent); + } + + @Override + public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) { + return delegate.getNamespaceConfiguration(namespaceId); + } + + @Override + public Instance getInstance() { + return delegate.getInstance(); + } + }; + mgr.init(config); + MemoryManagementActions result; + // one tablet is really big and the other is for a nonexistent table + KeyExtent extent = new KeyExtent(new Text("2"), new Text("j"), null); + result = mgr.getMemoryManagementActions(tablets(t(extent, ZERO, ONE_GIG, 0), t(k("j"), ZERO, ONE_GIG, 0))); + assertEquals(1, result.tabletsToMinorCompact.size()); + assertEquals(extent, result.tabletsToMinorCompact.get(0)); + } + private static class LargestFirstMemoryManagerUnderTest extends LargestFirstMemoryManager { public long currentTime = ZERO; @@ -174,6 +222,25 @@ public class LargestFirstMemoryManagerTest { return 15 * 60 * 1000; } + @Override + boolean tableExists(Instance instance, String tableId) { + return true; + } + } + + private static class LargestFirstMemoryManagerWithExistenceCheck extends LargestFirstMemoryManagerUnderTest { + + Function existenceCheck; + + public LargestFirstMemoryManagerWithExistenceCheck(Function existenceCheck) { + super(); + this.existenceCheck = existenceCheck; + } + + @Override + boolean tableExists(Instance instance, String tableId) { + return existenceCheck.apply(tableId); + } } private static KeyExtent k(String endRow) {