From commits-return-24185-archive-asf-public=cust-asf.ponee.io@accumulo.apache.org Fri Sep 25 19:55:17 2020 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mailroute1-lw-us.apache.org (mailroute1-lw-us.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with ESMTPS id CB0681806D6 for ; Fri, 25 Sep 2020 21:55:17 +0200 (CEST) Received: from mail.apache.org (localhost [127.0.0.1]) by mailroute1-lw-us.apache.org (ASF Mail Server at mailroute1-lw-us.apache.org) with SMTP id 02EFB124323 for ; Fri, 25 Sep 2020 19:55:17 +0000 (UTC) Received: (qmail 18229 invoked by uid 500); 25 Sep 2020 19:55:16 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 18162 invoked by uid 99); 25 Sep 2020 19:55:16 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 25 Sep 2020 19:55:16 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 59D0C82908; Fri, 25 Sep 2020 19:55:16 +0000 (UTC) Date: Fri, 25 Sep 2020 19:55:17 +0000 To: "commits@accumulo.apache.org" Subject: [accumulo] 01/01: Make ITs with compactions run in MINI only. Fixes #1713 MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: mmiller@apache.org In-Reply-To: <160106371611.31365.14513583031235991306@gitbox.apache.org> References: <160106371611.31365.14513583031235991306@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: accumulo X-Git-Refname: refs/heads/compact-its X-Git-Reftype: branch X-Git-Rev: acbc056db5d99d78ccd6a9594c7498cf925b8c50 X-Git-NotificationType: diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated Message-Id: <20200925195516.59D0C82908@gitbox.apache.org> This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch compact-its in repository https://gitbox.apache.org/repos/asf/accumulo.git commit acbc056db5d99d78ccd6a9594c7498cf925b8c50 Author: Mike Miller AuthorDate: Fri Sep 25 15:49:05 2020 -0400 Make ITs with compactions run in MINI only. Fixes #1713 * Prevent several ITs that have compaction behavior from running in standalone mode by overriding the canRunTest method. These tests currently don't work well in standalone mode and will fail. --- .../org/apache/accumulo/test/IteratorEnvIT.java | 8 +++- .../test/RecoveryCompactionsAreFlushesIT.java | 5 +++ .../apache/accumulo/test/TableOperationsIT.java | 13 ++++-- .../accumulo/test/UserCompactionStrategyIT.java | 13 ++++-- .../accumulo/test/functional/CompactionIT.java | 49 +++------------------- .../accumulo/test/functional/MasterMetricsIT.java | 8 +++- .../apache/accumulo/test/functional/SummaryIT.java | 5 +++ 7 files changed, 48 insertions(+), 53 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/IteratorEnvIT.java b/test/src/main/java/org/apache/accumulo/test/IteratorEnvIT.java index b8d5c2e..3dbfe5d 100644 --- a/test/src/main/java/org/apache/accumulo/test/IteratorEnvIT.java +++ b/test/src/main/java/org/apache/accumulo/test/IteratorEnvIT.java @@ -54,6 +54,11 @@ import org.junit.Test; public class IteratorEnvIT extends AccumuloClusterHarness { @Override + public boolean canRunTest(ClusterType type) { + return type == ClusterType.MINI; + } + + @Override public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { cfg.setNumTservers(1); } @@ -168,7 +173,8 @@ public class IteratorEnvIT extends AccumuloClusterHarness { @After public void finish() { - client.close(); + if (client != null) + client.close(); } @Test diff --git a/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java b/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java index 8bd87ff..f89bf7a 100644 --- a/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java +++ b/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java @@ -47,6 +47,11 @@ import com.google.common.collect.Iterators; public class RecoveryCompactionsAreFlushesIT extends AccumuloClusterHarness { @Override + public boolean canRunTest(ClusterType type) { + return type == ClusterType.MINI; + } + + @Override public int defaultTimeoutSeconds() { return 180; } diff --git a/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java b/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java index 89170cb..ff1fdc6 100644 --- a/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java @@ -60,7 +60,6 @@ import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.TablePermission; -import org.apache.accumulo.core.tabletserver.thrift.TabletClientService; import org.apache.accumulo.harness.AccumuloClusterHarness; import org.apache.accumulo.test.functional.BadIterator; import org.apache.accumulo.test.functional.FunctionalTestUtils; @@ -74,10 +73,14 @@ import com.google.common.collect.Sets; public class TableOperationsIT extends AccumuloClusterHarness { - static TabletClientService.Client client; private AccumuloClient accumuloClient; @Override + public boolean canRunTest(ClusterType type) { + return type == ClusterType.MINI; + } + + @Override public int defaultTimeoutSeconds() { return 90; } @@ -89,8 +92,10 @@ public class TableOperationsIT extends AccumuloClusterHarness { @After public void checkForDanglingFateLocks() { - FunctionalTestUtils.assertNoDanglingFateLocks((ClientContext) accumuloClient, getCluster()); - accumuloClient.close(); + if (getClusterType() == ClusterType.MINI) { + FunctionalTestUtils.assertNoDanglingFateLocks((ClientContext) accumuloClient, getCluster()); + accumuloClient.close(); + } } @Test diff --git a/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java b/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java index d7c2930..af6ab21 100644 --- a/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java @@ -66,15 +66,22 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; public class UserCompactionStrategyIT extends AccumuloClusterHarness { @Override + public boolean canRunTest(ClusterType type) { + return type == ClusterType.MINI; + } + + @Override public int defaultTimeoutSeconds() { return 3 * 60; } @After public void checkForDanglingFateLocks() { - try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) { - assertNotNull(c); - FunctionalTestUtils.assertNoDanglingFateLocks((ClientContext) c, getCluster()); + if (getClusterType() == ClusterType.MINI) { + try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) { + assertNotNull(c); + FunctionalTestUtils.assertNoDanglingFateLocks((ClientContext) c, getCluster()); + } } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java index 4d9e4dc..5da58e6 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java @@ -21,7 +21,6 @@ package org.apache.accumulo.test.functional; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -30,14 +29,12 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.client.admin.InstanceOperations; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.harness.AccumuloClusterHarness; -import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; import org.apache.accumulo.test.VerifyIngest; import org.apache.accumulo.test.VerifyIngest.VerifyParams; @@ -46,8 +43,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; import org.apache.hadoop.io.Text; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,6 +54,11 @@ public class CompactionIT extends AccumuloClusterHarness { private static final Logger log = LoggerFactory.getLogger(CompactionIT.class); @Override + public boolean canRunTest(ClusterType type) { + return type == ClusterType.MINI; + } + + @Override public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s"); cfg.setProperty(Property.TSERV_MAJC_THREAD_MAXOPEN, "4"); @@ -73,45 +73,6 @@ public class CompactionIT extends AccumuloClusterHarness { return 4 * 60; } - private String majcThreadMaxOpen, majcDelay, majcMaxConcurrent; - - @Before - public void alterConfig() throws Exception { - if (getClusterType() == ClusterType.STANDALONE) { - try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { - InstanceOperations iops = client.instanceOperations(); - Map config = iops.getSystemConfiguration(); - majcThreadMaxOpen = config.get(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey()); - majcDelay = config.get(Property.TSERV_MAJC_DELAY.getKey()); - majcMaxConcurrent = config.get(Property.TSERV_MAJC_MAXCONCURRENT.getKey()); - - iops.setProperty(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey(), "4"); - iops.setProperty(Property.TSERV_MAJC_DELAY.getKey(), "1"); - iops.setProperty(Property.TSERV_MAJC_MAXCONCURRENT.getKey(), "1"); - - getClusterControl().stopAllServers(ServerType.TABLET_SERVER); - getClusterControl().startAllServers(ServerType.TABLET_SERVER); - } - } - } - - @After - public void resetConfig() throws Exception { - // We set the values.. - if (majcThreadMaxOpen != null) { - try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { - InstanceOperations iops = client.instanceOperations(); - - iops.setProperty(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey(), majcThreadMaxOpen); - iops.setProperty(Property.TSERV_MAJC_DELAY.getKey(), majcDelay); - iops.setProperty(Property.TSERV_MAJC_MAXCONCURRENT.getKey(), majcMaxConcurrent); - - getClusterControl().stopAllServers(ServerType.TABLET_SERVER); - getClusterControl().startAllServers(ServerType.TABLET_SERVER); - } - } - } - @Test public void test() throws Exception { try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) { diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MasterMetricsIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MasterMetricsIT.java index e070dca..e5504a4 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MasterMetricsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MasterMetricsIT.java @@ -52,6 +52,11 @@ import org.slf4j.LoggerFactory; */ public class MasterMetricsIT extends AccumuloClusterHarness { + @Override + public boolean canRunTest(ClusterType type) { + return type == ClusterType.MINI; + } + private static final Logger log = LoggerFactory.getLogger(MasterMetricsIT.class); private AccumuloClient accumuloClient; @@ -87,7 +92,8 @@ public class MasterMetricsIT extends AccumuloClusterHarness { @After public void cleanup() { - metricsTail.close(); + if (metricsTail != null) + metricsTail.close(); } @Override diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SummaryIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SummaryIT.java index ed77986..5ee1863 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/SummaryIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/SummaryIT.java @@ -93,6 +93,11 @@ import com.google.common.collect.Lists; public class SummaryIT extends AccumuloClusterHarness { + @Override + public boolean canRunTest(ClusterType type) { + return type == ClusterType.MINI; + } + private LongSummaryStatistics getTimestampStats(final String table, AccumuloClient c) throws TableNotFoundException { try (Scanner scanner = c.createScanner(table, Authorizations.EMPTY)) {