accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [14/27] accumulo git commit: ACCUMULO-4312 Fix additional sealing problems in 1.7 branch
Date Fri, 27 May 2016 20:15:08 GMT
ACCUMULO-4312 Fix additional sealing problems in 1.7 branch


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d10840b8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d10840b8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d10840b8

Branch: refs/heads/1.8
Commit: d10840b8e5177f1f026f28705f93943a1921ad82
Parents: 544d661
Author: Christopher Tubbs <ctubbsii@apache.org>
Authored: Fri May 27 15:59:35 2016 -0400
Committer: Christopher Tubbs <ctubbsii@apache.org>
Committed: Fri May 27 15:59:35 2016 -0400

----------------------------------------------------------------------
 .../tabletserver/LargestFirstMemoryManager.java |    2 +-
 .../LargestFirstMemoryManagerTest.java          |  294 ---
 .../tserver/LargestFirstMemoryManagerTest.java  |  298 +++
 .../accumulo/proxy/ProxyDurabilityIT.java       |  139 --
 .../apache/accumulo/proxy/SimpleProxyBase.java  | 2272 -----------------
 .../apache/accumulo/proxy/TBinaryProxyIT.java   |   33 -
 .../apache/accumulo/proxy/TCompactProxyIT.java  |   32 -
 .../accumulo/proxy/TJsonProtocolProxyIT.java    |   33 -
 .../apache/accumulo/proxy/TTupleProxyIT.java    |   33 -
 .../proxy/TestProxyInstanceOperations.java      |   83 -
 .../accumulo/proxy/TestProxyReadWrite.java      |  466 ----
 .../proxy/TestProxySecurityOperations.java      |  146 --
 .../proxy/TestProxyTableOperations.java         |  201 --
 .../accumulo/test/proxy/ProxyDurabilityIT.java  |  141 ++
 .../accumulo/test/proxy/SimpleProxyBase.java    | 2274 ++++++++++++++++++
 .../accumulo/test/proxy/TBinaryProxyIT.java     |   33 +
 .../accumulo/test/proxy/TCompactProxyIT.java    |   32 +
 .../test/proxy/TJsonProtocolProxyIT.java        |   33 +
 .../accumulo/test/proxy/TTupleProxyIT.java      |   33 +
 .../test/proxy/TestProxyInstanceOperations.java |   85 +
 .../accumulo/test/proxy/TestProxyReadWrite.java |  469 ++++
 .../test/proxy/TestProxySecurityOperations.java |  148 ++
 .../test/proxy/TestProxyTableOperations.java    |  203 ++
 23 files changed, 3750 insertions(+), 3733 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
index d73c46e..e49e1af 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
@@ -146,7 +146,7 @@ public class LargestFirstMemoryManager implements MemoryManager {
     return mincIdleThresholds.get(tableId);
   }
 
-  boolean tableExists(Instance instance, String tableId) {
+  protected boolean tableExists(Instance instance, String tableId) {
     return Tables.exists(instance, tableId);
   }
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
deleted file mode 100644
index b08b980..0000000
--- a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.tabletserver;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.mock.MockInstance;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.DefaultConfiguration;
-import org.apache.accumulo.core.data.impl.KeyExtent;
-import org.apache.accumulo.server.conf.NamespaceConfiguration;
-import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
-import org.apache.accumulo.server.conf.TableConfiguration;
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-import com.google.common.base.Function;
-
-public class LargestFirstMemoryManagerTest {
-
-  private static final long ZERO = System.currentTimeMillis();
-  private static final long LATER = ZERO + 20 * 60 * 1000;
-  private static final long ONE_GIG = 1024 * 1024 * 1024;
-  private static final long HALF_GIG = ONE_GIG / 2;
-  private static final long QGIG = ONE_GIG / 4;
-  private static final long ONE_MINUTE = 60 * 1000;
-
-  @Test
-  public void test() throws Exception {
-    LargestFirstMemoryManagerUnderTest mgr = new LargestFirstMemoryManagerUnderTest();
-    ServerConfiguration config = new ServerConfiguration() {
-      ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance());
-
-      @Override
-      public AccumuloConfiguration getConfiguration() {
-        return DefaultConfiguration.getInstance();
-      }
-
-      @Override
-      public TableConfiguration getTableConfiguration(String tableId) {
-        return delegate.getTableConfiguration(tableId);
-      }
-
-      @Override
-      public TableConfiguration getTableConfiguration(KeyExtent extent) {
-        return delegate.getTableConfiguration(extent);
-      }
-
-      @Override
-      public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) {
-        return delegate.getNamespaceConfiguration(namespaceId);
-      }
-
-      @Override
-      public Instance getInstance() {
-        return delegate.getInstance();
-      }
-    };
-    mgr.init(config);
-    MemoryManagementActions result;
-    // nothing to do
-    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1000, 0), t(k("y"), ZERO,
2000, 0)));
-    assertEquals(0, result.tabletsToMinorCompact.size());
-    // one tablet is really big
-    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, ONE_GIG, 0), t(k("y"),
ZERO, 2000, 0)));
-    assertEquals(1, result.tabletsToMinorCompact.size());
-    assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
-    // one tablet is idle
-    mgr.currentTime = LATER;
-    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER,
2000, 0)));
-    assertEquals(1, result.tabletsToMinorCompact.size());
-    assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
-    // one tablet is idle, but one is really big
-    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER,
ONE_GIG, 0)));
-    assertEquals(1, result.tabletsToMinorCompact.size());
-    assertEquals(k("y"), result.tabletsToMinorCompact.get(0));
-    // lots of work to do
-    mgr = new LargestFirstMemoryManagerUnderTest();
-    mgr.init(config);
-    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"),
ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0),
-        t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO,
HALF_GIG + 5, 0), t(k("g"), ZERO, HALF_GIG + 6, 0),
-        t(k("h"), ZERO, HALF_GIG + 7, 0), t(k("i"), ZERO, HALF_GIG + 8, 0)));
-    assertEquals(2, result.tabletsToMinorCompact.size());
-    assertEquals(k("i"), result.tabletsToMinorCompact.get(0));
-    assertEquals(k("h"), result.tabletsToMinorCompact.get(1));
-    // one finished, one in progress, one filled up
-    mgr = new LargestFirstMemoryManagerUnderTest();
-    mgr.init(config);
-    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"),
ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0),
-        t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO,
HALF_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG, 0),
-        t(k("h"), ZERO, 0, HALF_GIG + 7), t(k("i"), ZERO, 0, 0)));
-    assertEquals(1, result.tabletsToMinorCompact.size());
-    assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
-    // memory is very full, lots of candidates
-    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"),
ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0),
-        t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO,
ONE_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG + 6, 0),
-        t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
-    assertEquals(2, result.tabletsToMinorCompact.size());
-    assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
-    assertEquals(k("f"), result.tabletsToMinorCompact.get(1));
-    // only have two compactors, still busy
-    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"),
ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0),
-        t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO,
ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, ONE_GIG + 6),
-        t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
-    assertEquals(0, result.tabletsToMinorCompact.size());
-    // finished one
-    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"),
ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0),
-        t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO,
ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, 0),
-        t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
-    assertEquals(1, result.tabletsToMinorCompact.size());
-    assertEquals(k("e"), result.tabletsToMinorCompact.get(0));
-
-    // many are running: do nothing
-    mgr = new LargestFirstMemoryManagerUnderTest();
-    mgr.init(config);
-    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"),
ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0),
-        t(k("d"), ZERO, 0, HALF_GIG), t(k("e"), ZERO, 0, HALF_GIG), t(k("f"), ZERO, 0, HALF_GIG),
t(k("g"), ZERO, 0, HALF_GIG), t(k("i"), ZERO, 0, HALF_GIG),
-        t(k("j"), ZERO, 0, HALF_GIG), t(k("k"), ZERO, 0, HALF_GIG), t(k("l"), ZERO, 0, HALF_GIG),
t(k("m"), ZERO, 0, HALF_GIG)));
-    assertEquals(0, result.tabletsToMinorCompact.size());
-
-    // observe adjustment:
-    mgr = new LargestFirstMemoryManagerUnderTest();
-    mgr.init(config);
-    // compact the largest
-    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO,
QGIG + 1, 0), t(k("c"), ZERO, QGIG + 2, 0)));
-    assertEquals(1, result.tabletsToMinorCompact.size());
-    assertEquals(k("c"), result.tabletsToMinorCompact.get(0));
-    // show that it is compacting... do nothing
-    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO,
QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2)));
-    assertEquals(0, result.tabletsToMinorCompact.size());
-    // not going to bother compacting any more
-    mgr.currentTime += ONE_MINUTE;
-    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO,
QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2)));
-    assertEquals(0, result.tabletsToMinorCompact.size());
-    // now do nothing
-    mgr.currentTime += ONE_MINUTE;
-    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO,
0, 0), t(k("c"), ZERO, 0, 0)));
-    assertEquals(0, result.tabletsToMinorCompact.size());
-    // on no! more data, this time we compact because we've adjusted
-    mgr.currentTime += ONE_MINUTE;
-    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO,
QGIG + 1, 0), t(k("c"), ZERO, 0, 0)));
-    assertEquals(1, result.tabletsToMinorCompact.size());
-    assertEquals(k("b"), result.tabletsToMinorCompact.get(0));
-  }
-
-  @Test
-  public void testDeletedTable() throws Exception {
-    final String deletedTableId = "1";
-    Function<String,Boolean> existenceCheck = new Function<String,Boolean>()
{
-      public Boolean apply(String tableId) {
-        return !deletedTableId.equals(tableId);
-      }
-    };
-    LargestFirstMemoryManagerWithExistenceCheck mgr = new LargestFirstMemoryManagerWithExistenceCheck(existenceCheck);
-    ServerConfiguration config = new ServerConfiguration() {
-      ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance());
-
-      @Override
-      public AccumuloConfiguration getConfiguration() {
-        return DefaultConfiguration.getInstance();
-      }
-
-      @Override
-      public TableConfiguration getTableConfiguration(String tableId) {
-        return delegate.getTableConfiguration(tableId);
-      }
-
-      @Override
-      public TableConfiguration getTableConfiguration(KeyExtent extent) {
-        return delegate.getTableConfiguration(extent);
-      }
-
-      @Override
-      public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) {
-        return delegate.getNamespaceConfiguration(namespaceId);
-      }
-
-      @Override
-      public Instance getInstance() {
-        return delegate.getInstance();
-      }
-    };
-    mgr.init(config);
-    MemoryManagementActions result;
-    // one tablet is really big and the other is for a nonexistent table
-    KeyExtent extent = new KeyExtent(new Text("2"), new Text("j"), null);
-    result = mgr.getMemoryManagementActions(tablets(t(extent, ZERO, ONE_GIG, 0), t(k("j"),
ZERO, ONE_GIG, 0)));
-    assertEquals(1, result.tabletsToMinorCompact.size());
-    assertEquals(extent, result.tabletsToMinorCompact.get(0));
-  }
-
-  private static class LargestFirstMemoryManagerUnderTest extends LargestFirstMemoryManager
{
-
-    public long currentTime = ZERO;
-
-    @Override
-    protected long currentTimeMillis() {
-      return currentTime;
-    }
-
-    @Override
-    protected long getMinCIdleThreshold(KeyExtent extent) {
-      return 15 * 60 * 1000;
-    }
-
-    @Override
-    boolean tableExists(Instance instance, String tableId) {
-      return true;
-    }
-  }
-
-  private static class LargestFirstMemoryManagerWithExistenceCheck extends LargestFirstMemoryManagerUnderTest
{
-
-    Function<String,Boolean> existenceCheck;
-
-    public LargestFirstMemoryManagerWithExistenceCheck(Function<String,Boolean> existenceCheck)
{
-      super();
-      this.existenceCheck = existenceCheck;
-    }
-
-    @Override
-    boolean tableExists(Instance instance, String tableId) {
-      return existenceCheck.apply(tableId);
-    }
-  }
-
-  private static KeyExtent k(String endRow) {
-    return new KeyExtent(new Text("1"), new Text(endRow), null);
-  }
-
-  private static class TestTabletState implements TabletState {
-
-    private final KeyExtent extent;
-    private final long lastCommit;
-    private final long memSize;
-    private final long compactingSize;
-
-    TestTabletState(KeyExtent extent, long commit, long memsize, long compactingTableSize)
{
-      this.extent = extent;
-      this.lastCommit = commit;
-      this.memSize = memsize;
-      this.compactingSize = compactingTableSize;
-    }
-
-    @Override
-    public KeyExtent getExtent() {
-      return extent;
-    }
-
-    @Override
-    public long getLastCommitTime() {
-      return lastCommit;
-    }
-
-    @Override
-    public long getMemTableSize() {
-      return memSize;
-    }
-
-    @Override
-    public long getMinorCompactingMemTableSize() {
-      return compactingSize;
-    }
-
-  }
-
-  private TabletState t(KeyExtent ke, long lastCommit, long memSize, long compactingSize)
{
-    return new TestTabletState(ke, lastCommit, memSize, compactingSize);
-  }
-
-  private static List<TabletState> tablets(TabletState... states) {
-    return Arrays.asList(states);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java
new file mode 100644
index 0000000..f3bd220
--- /dev/null
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.server.conf.NamespaceConfiguration;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.tabletserver.LargestFirstMemoryManager;
+import org.apache.accumulo.server.tabletserver.MemoryManagementActions;
+import org.apache.accumulo.server.tabletserver.TabletState;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+import com.google.common.base.Function;
+
+public class LargestFirstMemoryManagerTest {
+
+  private static final long ZERO = System.currentTimeMillis();
+  private static final long LATER = ZERO + 20 * 60 * 1000;
+  private static final long ONE_GIG = 1024 * 1024 * 1024;
+  private static final long HALF_GIG = ONE_GIG / 2;
+  private static final long QGIG = ONE_GIG / 4;
+  private static final long ONE_MINUTE = 60 * 1000;
+
+  @Test
+  public void test() throws Exception {
+    LargestFirstMemoryManagerUnderTest mgr = new LargestFirstMemoryManagerUnderTest();
+    ServerConfiguration config = new ServerConfiguration() {
+      ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance());
+
+      @Override
+      public AccumuloConfiguration getConfiguration() {
+        return DefaultConfiguration.getInstance();
+      }
+
+      @Override
+      public TableConfiguration getTableConfiguration(String tableId) {
+        return delegate.getTableConfiguration(tableId);
+      }
+
+      @Override
+      public TableConfiguration getTableConfiguration(KeyExtent extent) {
+        return delegate.getTableConfiguration(extent);
+      }
+
+      @Override
+      public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) {
+        return delegate.getNamespaceConfiguration(namespaceId);
+      }
+
+      @Override
+      public Instance getInstance() {
+        return delegate.getInstance();
+      }
+    };
+    mgr.init(config);
+    MemoryManagementActions result;
+    // nothing to do
+    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1000, 0), t(k("y"), ZERO,
2000, 0)));
+    assertEquals(0, result.tabletsToMinorCompact.size());
+    // one tablet is really big
+    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, ONE_GIG, 0), t(k("y"),
ZERO, 2000, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
+    // one tablet is idle
+    mgr.currentTime = LATER;
+    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER,
2000, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
+    // one tablet is idle, but one is really big
+    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER,
ONE_GIG, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("y"), result.tabletsToMinorCompact.get(0));
+    // lots of work to do
+    mgr = new LargestFirstMemoryManagerUnderTest();
+    mgr.init(config);
+    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"),
ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0),
+        t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO,
HALF_GIG + 5, 0), t(k("g"), ZERO, HALF_GIG + 6, 0),
+        t(k("h"), ZERO, HALF_GIG + 7, 0), t(k("i"), ZERO, HALF_GIG + 8, 0)));
+    assertEquals(2, result.tabletsToMinorCompact.size());
+    assertEquals(k("i"), result.tabletsToMinorCompact.get(0));
+    assertEquals(k("h"), result.tabletsToMinorCompact.get(1));
+    // one finished, one in progress, one filled up
+    mgr = new LargestFirstMemoryManagerUnderTest();
+    mgr.init(config);
+    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"),
ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0),
+        t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO,
HALF_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG, 0),
+        t(k("h"), ZERO, 0, HALF_GIG + 7), t(k("i"), ZERO, 0, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
+    // memory is very full, lots of candidates
+    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"),
ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0),
+        t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO,
ONE_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG + 6, 0),
+        t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
+    assertEquals(2, result.tabletsToMinorCompact.size());
+    assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
+    assertEquals(k("f"), result.tabletsToMinorCompact.get(1));
+    // only have two compactors, still busy
+    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"),
ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0),
+        t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO,
ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, ONE_GIG + 6),
+        t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
+    assertEquals(0, result.tabletsToMinorCompact.size());
+    // finished one
+    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"),
ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0),
+        t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO,
ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, 0),
+        t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("e"), result.tabletsToMinorCompact.get(0));
+
+    // many are running: do nothing
+    mgr = new LargestFirstMemoryManagerUnderTest();
+    mgr.init(config);
+    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"),
ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0),
+        t(k("d"), ZERO, 0, HALF_GIG), t(k("e"), ZERO, 0, HALF_GIG), t(k("f"), ZERO, 0, HALF_GIG),
t(k("g"), ZERO, 0, HALF_GIG), t(k("i"), ZERO, 0, HALF_GIG),
+        t(k("j"), ZERO, 0, HALF_GIG), t(k("k"), ZERO, 0, HALF_GIG), t(k("l"), ZERO, 0, HALF_GIG),
t(k("m"), ZERO, 0, HALF_GIG)));
+    assertEquals(0, result.tabletsToMinorCompact.size());
+
+    // observe adjustment:
+    mgr = new LargestFirstMemoryManagerUnderTest();
+    mgr.init(config);
+    // compact the largest
+    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO,
QGIG + 1, 0), t(k("c"), ZERO, QGIG + 2, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("c"), result.tabletsToMinorCompact.get(0));
+    // show that it is compacting... do nothing
+    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO,
QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2)));
+    assertEquals(0, result.tabletsToMinorCompact.size());
+    // not going to bother compacting any more
+    mgr.currentTime += ONE_MINUTE;
+    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO,
QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2)));
+    assertEquals(0, result.tabletsToMinorCompact.size());
+    // now do nothing
+    mgr.currentTime += ONE_MINUTE;
+    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO,
0, 0), t(k("c"), ZERO, 0, 0)));
+    assertEquals(0, result.tabletsToMinorCompact.size());
+    // on no! more data, this time we compact because we've adjusted
+    mgr.currentTime += ONE_MINUTE;
+    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO,
QGIG + 1, 0), t(k("c"), ZERO, 0, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(k("b"), result.tabletsToMinorCompact.get(0));
+  }
+
+  @Test
+  public void testDeletedTable() throws Exception {
+    final String deletedTableId = "1";
+    Function<String,Boolean> existenceCheck = new Function<String,Boolean>()
{
+      @Override
+      public Boolean apply(String tableId) {
+        return !deletedTableId.equals(tableId);
+      }
+    };
+    LargestFirstMemoryManagerWithExistenceCheck mgr = new LargestFirstMemoryManagerWithExistenceCheck(existenceCheck);
+    ServerConfiguration config = new ServerConfiguration() {
+      ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance());
+
+      @Override
+      public AccumuloConfiguration getConfiguration() {
+        return DefaultConfiguration.getInstance();
+      }
+
+      @Override
+      public TableConfiguration getTableConfiguration(String tableId) {
+        return delegate.getTableConfiguration(tableId);
+      }
+
+      @Override
+      public TableConfiguration getTableConfiguration(KeyExtent extent) {
+        return delegate.getTableConfiguration(extent);
+      }
+
+      @Override
+      public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) {
+        return delegate.getNamespaceConfiguration(namespaceId);
+      }
+
+      @Override
+      public Instance getInstance() {
+        return delegate.getInstance();
+      }
+    };
+    mgr.init(config);
+    MemoryManagementActions result;
+    // one tablet is really big and the other is for a nonexistent table
+    KeyExtent extent = new KeyExtent(new Text("2"), new Text("j"), null);
+    result = mgr.getMemoryManagementActions(tablets(t(extent, ZERO, ONE_GIG, 0), t(k("j"),
ZERO, ONE_GIG, 0)));
+    assertEquals(1, result.tabletsToMinorCompact.size());
+    assertEquals(extent, result.tabletsToMinorCompact.get(0));
+  }
+
+  private static class LargestFirstMemoryManagerUnderTest extends LargestFirstMemoryManager
{
+
+    public long currentTime = ZERO;
+
+    @Override
+    protected long currentTimeMillis() {
+      return currentTime;
+    }
+
+    @Override
+    protected long getMinCIdleThreshold(KeyExtent extent) {
+      return 15 * 60 * 1000;
+    }
+
+    @Override
+    protected boolean tableExists(Instance instance, String tableId) {
+      return true;
+    }
+  }
+
+  private static class LargestFirstMemoryManagerWithExistenceCheck extends LargestFirstMemoryManagerUnderTest
{
+
+    Function<String,Boolean> existenceCheck;
+
+    public LargestFirstMemoryManagerWithExistenceCheck(Function<String,Boolean> existenceCheck)
{
+      super();
+      this.existenceCheck = existenceCheck;
+    }
+
+    @Override
+    protected boolean tableExists(Instance instance, String tableId) {
+      return existenceCheck.apply(tableId);
+    }
+  }
+
+  private static KeyExtent k(String endRow) {
+    return new KeyExtent(new Text("1"), new Text(endRow), null);
+  }
+
+  private static class TestTabletState implements TabletState {
+
+    private final KeyExtent extent;
+    private final long lastCommit;
+    private final long memSize;
+    private final long compactingSize;
+
+    TestTabletState(KeyExtent extent, long commit, long memsize, long compactingTableSize)
{
+      this.extent = extent;
+      this.lastCommit = commit;
+      this.memSize = memsize;
+      this.compactingSize = compactingTableSize;
+    }
+
+    @Override
+    public KeyExtent getExtent() {
+      return extent;
+    }
+
+    @Override
+    public long getLastCommitTime() {
+      return lastCommit;
+    }
+
+    @Override
+    public long getMemTableSize() {
+      return memSize;
+    }
+
+    @Override
+    public long getMinorCompactingMemTableSize() {
+      return compactingSize;
+    }
+
+  }
+
+  private TabletState t(KeyExtent ke, long lastCommit, long memSize, long compactingSize)
{
+    return new TestTabletState(ke, lastCommit, memSize, compactingSize);
+  }
+
+  private static List<TabletState> tablets(TabletState... states) {
+    return Arrays.asList(states);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java b/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
deleted file mode 100644
index b388cbf..0000000
--- a/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.proxy;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.TreeMap;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.minicluster.ServerType;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.minicluster.impl.ProcessReference;
-import org.apache.accumulo.proxy.thrift.AccumuloProxy.Client;
-import org.apache.accumulo.proxy.thrift.Column;
-import org.apache.accumulo.proxy.thrift.ColumnUpdate;
-import org.apache.accumulo.proxy.thrift.Condition;
-import org.apache.accumulo.proxy.thrift.ConditionalStatus;
-import org.apache.accumulo.proxy.thrift.ConditionalUpdates;
-import org.apache.accumulo.proxy.thrift.ConditionalWriterOptions;
-import org.apache.accumulo.proxy.thrift.Durability;
-import org.apache.accumulo.proxy.thrift.TimeType;
-import org.apache.accumulo.proxy.thrift.WriterOptions;
-import org.apache.accumulo.server.util.PortUtils;
-import org.apache.accumulo.test.functional.ConfigurableMacIT;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.thrift.protocol.TJSONProtocol;
-import org.apache.thrift.server.TServer;
-import org.junit.Test;
-
-import com.google.common.collect.Iterators;
-import com.google.common.net.HostAndPort;
-
-public class ProxyDurabilityIT extends ConfigurableMacIT {
-
-  @Override
-  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
-    hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
-    cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "10s");
-    cfg.setNumTservers(1);
-  }
-
-  private static ByteBuffer bytes(String value) {
-    return ByteBuffer.wrap(value.getBytes());
-  }
-
-  @Test
-  public void testDurability() throws Exception {
-    Connector c = getConnector();
-    Properties props = new Properties();
-    // Avoid issues with locally installed client configuration files with custom properties
-    File emptyFile = Files.createTempFile(null, null).toFile();
-    emptyFile.deleteOnExit();
-    props.put("instance", c.getInstance().getInstanceName());
-    props.put("zookeepers", c.getInstance().getZooKeepers());
-    props.put("tokenClass", PasswordToken.class.getName());
-    props.put("clientConfigurationFile", emptyFile.toString());
-
-    TJSONProtocol.Factory protocol = new TJSONProtocol.Factory();
-
-    int proxyPort = PortUtils.getRandomFreePort();
-    final TServer proxyServer = Proxy.createProxyServer(HostAndPort.fromParts("localhost",
proxyPort), protocol, props).server;
-    while (!proxyServer.isServing())
-      UtilWaitThread.sleep(100);
-    Client client = new TestProxyClient("localhost", proxyPort, protocol).proxy();
-    Map<String,String> properties = new TreeMap<String,String>();
-    properties.put("password", ROOT_PASSWORD);
-    ByteBuffer login = client.login("root", properties);
-
-    String tableName = getUniqueNames(1)[0];
-    client.createTable(login, tableName, true, TimeType.MILLIS);
-    assertTrue(c.tableOperations().exists(tableName));
-
-    WriterOptions options = new WriterOptions();
-    options.setDurability(Durability.NONE);
-    String writer = client.createWriter(login, tableName, options);
-    Map<ByteBuffer,List<ColumnUpdate>> cells = new TreeMap<ByteBuffer,List<ColumnUpdate>>();
-    ColumnUpdate column = new ColumnUpdate(bytes("cf"), bytes("cq"));
-    column.setValue("value".getBytes());
-    cells.put(bytes("row"), Collections.singletonList(column));
-    client.update(writer, cells);
-    client.closeWriter(writer);
-    assertEquals(1, count(tableName));
-    restartTServer();
-    assertEquals(0, count(tableName));
-
-    ConditionalWriterOptions cfg = new ConditionalWriterOptions();
-    cfg.setDurability(Durability.SYNC);
-    String cwriter = client.createConditionalWriter(login, tableName, cfg);
-    ConditionalUpdates updates = new ConditionalUpdates();
-    updates.addToConditions(new Condition(new Column(bytes("cf"), bytes("cq"), bytes(""))));
-    updates.addToUpdates(column);
-    Map<ByteBuffer,ConditionalStatus> status = client.updateRowsConditionally(cwriter,
Collections.singletonMap(bytes("row"), updates));
-    assertEquals(ConditionalStatus.ACCEPTED, status.get(bytes("row")));
-    assertEquals(1, count(tableName));
-    restartTServer();
-    assertEquals(1, count(tableName));
-
-    proxyServer.stop();
-  }
-
-  private void restartTServer() throws Exception {
-    for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
-      cluster.killProcess(ServerType.TABLET_SERVER, proc);
-    }
-    cluster.start();
-  }
-
-  private int count(String tableName) throws Exception {
-    return Iterators.size((getConnector().createScanner(tableName, Authorizations.EMPTY)).iterator());
-  }
-
-}


Mime
View raw message