Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4189819C27 for ; Tue, 12 Apr 2016 13:58:19 +0000 (UTC) Received: (qmail 38571 invoked by uid 500); 12 Apr 2016 13:58:19 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 38507 invoked by uid 500); 12 Apr 2016 13:58:19 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 37659 invoked by uid 99); 12 Apr 2016 13:58:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Apr 2016 13:58:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8C7E4DFBA0; Tue, 12 Apr 2016 13:58:18 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dlmarion@apache.org To: commits@accumulo.apache.org Date: Tue, 12 Apr 2016 13:58:32 -0000 Message-Id: In-Reply-To: <28ce56b801f84481baae935b546b14cd@git.apache.org> References: <28ce56b801f84481baae935b546b14cd@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [15/39] accumulo git commit: ACCUMULO-4175 Move ITs to correct directory ACCUMULO-4175 Move ITs to correct directory Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7cefd872 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7cefd872 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7cefd872 Branch: refs/heads/ACCUMULO-4173 Commit: 7cefd872f9276ebb3a4be4cfcdda8f3112ccd250 Parents: f284027 Author: Christopher Tubbs Authored: Fri Apr 1 14:55:44 2016 -0400 Committer: Christopher Tubbs Committed: Fri Apr 1 14:57:10 2016 -0400 ---------------------------------------------------------------------- .../test/functional/SessionBlockVerifyIT.java | 176 +++++++++++++++++++ .../test/functional/SessionBlockVerifyIT.java | 176 ------------------- 2 files changed, 176 insertions(+), 176 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/7cefd872/test/src/main/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java new file mode 100644 index 0000000..05f304b --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.functional; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.admin.ActiveScan; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Iterables; + +/** + * Verify that we have resolved blocking issue by ensuring that we have not lost scan sessions which we know to currently be running + */ +public class SessionBlockVerifyIT extends ScanSessionTimeOutIT { + private static final Logger log = LoggerFactory.getLogger(SessionBlockVerifyIT.class); + + @Override + public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { + Map siteConfig = cfg.getSiteConfig(); + cfg.setNumTservers(1); + siteConfig.put(Property.TSERV_SESSION_MAXIDLE.getKey(), getMaxIdleTimeString()); + siteConfig.put(Property.TSERV_READ_AHEAD_MAXCONCURRENT.getKey(), "11"); + cfg.setSiteConfig(siteConfig); + } + + @Override + protected int defaultTimeoutSeconds() { + return 60; + } + + @Override + protected String getMaxIdleTimeString() { + return "1s"; + } + + ExecutorService service = Executors.newFixedThreadPool(10); + + @Test + public void run() throws Exception { + Connector c = getConnector(); + String tableName = getUniqueNames(1)[0]; + c.tableOperations().create(tableName); + + BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig()); + + for (int i = 0; i < 1000; i++) { + Mutation m = new Mutation(new Text(String.format("%08d", i))); + for (int j = 0; j < 3; j++) + m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(UTF_8))); + + bw.addMutation(m); + } + + bw.close(); + + Scanner scanner = c.createScanner(tableName, new Authorizations()); + scanner.setReadaheadThreshold(20000); + scanner.setRange(new Range(String.format("%08d", 0), String.format("%08d", 1000))); + + // test by making a slow iterator and then a couple of fast ones. + // when then checking we shouldn't have any running except the slow iterator + IteratorSetting setting = new IteratorSetting(21, SlowIterator.class); + SlowIterator.setSeekSleepTime(setting, Long.MAX_VALUE); + SlowIterator.setSleepTime(setting, Long.MAX_VALUE); + scanner.addScanIterator(setting); + + final Iterator> slow = scanner.iterator(); + + final List> callables = new ArrayList>(); + final CountDownLatch latch = new CountDownLatch(10); + for (int i = 0; i < 10; i++) { + Future callable = service.submit(new Callable() { + public Boolean call() { + latch.countDown(); + while (slow.hasNext()) { + + slow.next(); + } + return slow.hasNext(); + } + }); + callables.add(callable); + } + + latch.await(); + + log.info("Starting SessionBlockVerifyIT"); + + // let's add more for good measure. + for (int i = 0; i < 2; i++) { + Scanner scanner2 = c.createScanner(tableName, new Authorizations()); + + scanner2.setRange(new Range(String.format("%08d", 0), String.format("%08d", 1000))); + + scanner2.setBatchSize(1); + Iterator> iter = scanner2.iterator(); + // call super's verify mechanism + verify(iter, 0, 1000); + + } + + int sessionsFound = 0; + // we have configured 1 tserver, so we can grab the one and only + String tserver = Iterables.getOnlyElement(c.instanceOperations().getTabletServers()); + + final List scans = c.instanceOperations().getActiveScans(tserver); + + for (ActiveScan scan : scans) { + // only here to minimize chance of seeing meta extent scans + + if (tableName.equals(scan.getTable()) && scan.getSsiList().size() > 0) { + assertEquals("Not the expected iterator", 1, scan.getSsiList().size()); + assertTrue("Not the expected iterator", scan.getSsiList().iterator().next().contains("SlowIterator")); + sessionsFound++; + } + + } + + /** + * The message below indicates the problem that we experience within ACCUMULO-3509. The issue manifests as a blockage in the Scanner synchronization that + * prevent us from making the close call against it. Since the close blocks until a read is finished, we ultimately have a block within the sweep of + * SessionManager. As a result never reap subsequent idle sessions AND we will orphan the sessionsToCleanup in the sweep, leading to an inaccurate count + * within sessionsFound. + */ + assertEquals("Must have ten sessions. Failure indicates a synchronization block within the sweep mechanism", 10, sessionsFound); + for (Future callable : callables) { + callable.cancel(true); + } + service.shutdown(); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/7cefd872/test/src/test/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java b/test/src/test/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java deleted file mode 100644 index 05f304b..0000000 --- a/test/src/test/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.functional; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import org.apache.accumulo.core.client.BatchWriter; -import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.client.admin.ActiveScan; -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Iterables; - -/** - * Verify that we have resolved blocking issue by ensuring that we have not lost scan sessions which we know to currently be running - */ -public class SessionBlockVerifyIT extends ScanSessionTimeOutIT { - private static final Logger log = LoggerFactory.getLogger(SessionBlockVerifyIT.class); - - @Override - public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { - Map siteConfig = cfg.getSiteConfig(); - cfg.setNumTservers(1); - siteConfig.put(Property.TSERV_SESSION_MAXIDLE.getKey(), getMaxIdleTimeString()); - siteConfig.put(Property.TSERV_READ_AHEAD_MAXCONCURRENT.getKey(), "11"); - cfg.setSiteConfig(siteConfig); - } - - @Override - protected int defaultTimeoutSeconds() { - return 60; - } - - @Override - protected String getMaxIdleTimeString() { - return "1s"; - } - - ExecutorService service = Executors.newFixedThreadPool(10); - - @Test - public void run() throws Exception { - Connector c = getConnector(); - String tableName = getUniqueNames(1)[0]; - c.tableOperations().create(tableName); - - BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig()); - - for (int i = 0; i < 1000; i++) { - Mutation m = new Mutation(new Text(String.format("%08d", i))); - for (int j = 0; j < 3; j++) - m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(UTF_8))); - - bw.addMutation(m); - } - - bw.close(); - - Scanner scanner = c.createScanner(tableName, new Authorizations()); - scanner.setReadaheadThreshold(20000); - scanner.setRange(new Range(String.format("%08d", 0), String.format("%08d", 1000))); - - // test by making a slow iterator and then a couple of fast ones. - // when then checking we shouldn't have any running except the slow iterator - IteratorSetting setting = new IteratorSetting(21, SlowIterator.class); - SlowIterator.setSeekSleepTime(setting, Long.MAX_VALUE); - SlowIterator.setSleepTime(setting, Long.MAX_VALUE); - scanner.addScanIterator(setting); - - final Iterator> slow = scanner.iterator(); - - final List> callables = new ArrayList>(); - final CountDownLatch latch = new CountDownLatch(10); - for (int i = 0; i < 10; i++) { - Future callable = service.submit(new Callable() { - public Boolean call() { - latch.countDown(); - while (slow.hasNext()) { - - slow.next(); - } - return slow.hasNext(); - } - }); - callables.add(callable); - } - - latch.await(); - - log.info("Starting SessionBlockVerifyIT"); - - // let's add more for good measure. - for (int i = 0; i < 2; i++) { - Scanner scanner2 = c.createScanner(tableName, new Authorizations()); - - scanner2.setRange(new Range(String.format("%08d", 0), String.format("%08d", 1000))); - - scanner2.setBatchSize(1); - Iterator> iter = scanner2.iterator(); - // call super's verify mechanism - verify(iter, 0, 1000); - - } - - int sessionsFound = 0; - // we have configured 1 tserver, so we can grab the one and only - String tserver = Iterables.getOnlyElement(c.instanceOperations().getTabletServers()); - - final List scans = c.instanceOperations().getActiveScans(tserver); - - for (ActiveScan scan : scans) { - // only here to minimize chance of seeing meta extent scans - - if (tableName.equals(scan.getTable()) && scan.getSsiList().size() > 0) { - assertEquals("Not the expected iterator", 1, scan.getSsiList().size()); - assertTrue("Not the expected iterator", scan.getSsiList().iterator().next().contains("SlowIterator")); - sessionsFound++; - } - - } - - /** - * The message below indicates the problem that we experience within ACCUMULO-3509. The issue manifests as a blockage in the Scanner synchronization that - * prevent us from making the close call against it. Since the close blocks until a read is finished, we ultimately have a block within the sweep of - * SessionManager. As a result never reap subsequent idle sessions AND we will orphan the sessionsToCleanup in the sweep, leading to an inaccurate count - * within sessionsFound. - */ - assertEquals("Must have ten sessions. Failure indicates a synchronization block within the sweep mechanism", 10, sessionsFound); - for (Future callable : callables) { - callable.cancel(true); - } - service.shutdown(); - } - -}