From commits-return-22034-archive-asf-public=cust-asf.ponee.io@accumulo.apache.org Sat Aug 11 02:53:52 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 94946180630 for ; Sat, 11 Aug 2018 02:53:50 +0200 (CEST) Received: (qmail 86099 invoked by uid 500); 11 Aug 2018 00:53:49 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 86089 invoked by uid 99); 11 Aug 2018 00:53:49 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 11 Aug 2018 00:53:49 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id D6393853C9; Sat, 11 Aug 2018 00:53:48 +0000 (UTC) Date: Sat, 11 Aug 2018 00:53:48 +0000 To: "commits@accumulo.apache.org" Subject: [accumulo] branch master updated: Fix #591 Cleanup tests that aren't running MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <153394882876.1029.16748055370065653141@gitbox.apache.org> From: ctubbsii@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: accumulo X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: cbc557e0db3107f924676f3a9fc1de5892e34f01 X-Git-Newrev: 6a4d11d06fd724c9ce13f5e25247edd3227842d3 X-Git-Rev: 6a4d11d06fd724c9ce13f5e25247edd3227842d3 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/master by this push: new 6a4d11d Fix #591 Cleanup tests that aren't running 6a4d11d is described below commit 6a4d11d06fd724c9ce13f5e25247edd3227842d3 Author: Christopher Tubbs AuthorDate: Fri Aug 10 20:40:13 2018 -0400 Fix #591 Cleanup tests that aren't running * Remove redundant proxy tests already covered by the Proxy ITs * Move some utility test classes (intended to be run with main() methods) to the util test package and rename them so they don't get confused as Unit tests or ITs * Update an error message in ZooKeeperInstance to be more informative --- .../accumulo/core/client/ZooKeeperInstance.java | 3 +- test/src/main/findbugs/exclude-filter.xml | 2 +- .../accumulo/test/EstimateInMemMapOverhead.java | 50 --- .../accumulo/test/proxy/SimpleProxyBase.java | 8 + .../test/proxy/TestProxyInstanceOperations.java | 86 ---- .../test/proxy/TestProxyNamespaceOperations.java | 175 -------- .../accumulo/test/proxy/TestProxyReadWrite.java | 486 --------------------- .../test/proxy/TestProxySecurityOperations.java | 166 ------- .../test/proxy/TestProxyTableOperations.java | 208 --------- .../test/util/memory/EstimateInMemMapOverhead.java | 50 +++ .../memory/InMemoryMapMemoryUsageCheck.java} | 6 +- .../memory/IntObjectMemoryUsageCheck.java} | 6 +- .../memory/MemoryUsageCheck.java} | 4 +- .../memory/MutationMemoryUsageCheck.java} | 6 +- .../memory/TextMemoryUsageCheck.java} | 6 +- .../metadata/MetadataBatchScan.java} | 8 +- .../nativemap/NativeMapConcurrency.java} | 8 +- .../nativemap/NativeMapPerformance.java} | 6 +- .../nativemap/NativeMapStress.java} | 6 +- .../tablet/WrongTablet.java} | 6 +- 20 files changed, 92 insertions(+), 1204 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java index 557cde8..1402571 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java @@ -94,7 +94,8 @@ public class ZooKeeperInstance implements Instance { this.instanceName = clientConf.get(ClientConfiguration.ClientProperty.INSTANCE_NAME); if ((instanceId == null) == (instanceName == null)) throw new IllegalArgumentException( - "Expected exactly one of instanceName and instanceId to be set"); + "Expected exactly one of instanceName and instanceId to be set; " + + (instanceName == null ? "neither" : "both") + " were set"); this.zooKeepers = clientConf.get(ClientConfiguration.ClientProperty.INSTANCE_ZK_HOST); this.zooKeepersSessionTimeOut = (int) ConfigurationTypeHelper .getTimeInMillis(clientConf.get(ClientConfiguration.ClientProperty.INSTANCE_ZK_TIMEOUT)); diff --git a/test/src/main/findbugs/exclude-filter.xml b/test/src/main/findbugs/exclude-filter.xml index 464df82..346aac5 100644 --- a/test/src/main/findbugs/exclude-filter.xml +++ b/test/src/main/findbugs/exclude-filter.xml @@ -48,7 +48,7 @@ - + diff --git a/test/src/main/java/org/apache/accumulo/test/EstimateInMemMapOverhead.java b/test/src/main/java/org/apache/accumulo/test/EstimateInMemMapOverhead.java deleted file mode 100644 index d4ae4a5..0000000 --- a/test/src/main/java/org/apache/accumulo/test/EstimateInMemMapOverhead.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test; - -public class EstimateInMemMapOverhead { - - private static void runTest(int numEntries, int keyLen, int colFamLen, int colQualLen, - int colVisLen, int dataLen) { - new IntObjectMemoryUsageTest(numEntries).run(); - new InMemoryMapMemoryUsageTest(numEntries, keyLen, colFamLen, colQualLen, colVisLen, dataLen) - .run(); - new TextMemoryUsageTest(numEntries, keyLen, colFamLen, colQualLen, dataLen).run(); - new MutationMemoryUsageTest(numEntries, keyLen, colFamLen, colQualLen, dataLen).run(); - } - - public static void main(String[] args) { - runTest(10000, 10, 4, 4, 4, 20); - runTest(100000, 10, 4, 4, 4, 20); - runTest(500000, 10, 4, 4, 4, 20); - runTest(1000000, 10, 4, 4, 4, 20); - runTest(2000000, 10, 4, 4, 4, 20); - - runTest(10000, 20, 5, 5, 5, 500); - runTest(100000, 20, 5, 5, 5, 500); - runTest(500000, 20, 5, 5, 5, 500); - runTest(1000000, 20, 5, 5, 5, 500); - runTest(2000000, 20, 5, 5, 5, 500); - - runTest(10000, 40, 10, 10, 10, 1000); - runTest(100000, 40, 10, 10, 10, 1000); - runTest(500000, 40, 10, 10, 10, 1000); - runTest(1000000, 40, 10, 10, 10, 1000); - runTest(2000000, 40, 10, 10, 10, 1000); - } - -} diff --git a/test/src/main/java/org/apache/accumulo/test/proxy/SimpleProxyBase.java b/test/src/main/java/org/apache/accumulo/test/proxy/SimpleProxyBase.java index fe35e37..c1d708d 100644 --- a/test/src/main/java/org/apache/accumulo/test/proxy/SimpleProxyBase.java +++ b/test/src/main/java/org/apache/accumulo/test/proxy/SimpleProxyBase.java @@ -1172,6 +1172,14 @@ public abstract class SimpleProxyBase extends SharedMiniClusterBase { public void testSystemProperties() throws Exception { Map cfg = client.getSiteConfiguration(creds); + // set generic property + client.setProperty(creds, "general.custom.test.systemprop", "whistletips"); + assertEquals( + proxyClient.proxy().getSystemConfiguration(creds).get("general.custom.test.systemprop"), + "whistletips"); + client.removeProperty(creds, "general.custom.test.systemprop"); + assertNull(client.getSystemConfiguration(creds).get("general.custom.test.systemprop")); + // set a property in zookeeper client.setProperty(creds, "table.split.threshold", "500M"); diff --git a/test/src/main/java/org/apache/accumulo/test/proxy/TestProxyInstanceOperations.java b/test/src/main/java/org/apache/accumulo/test/proxy/TestProxyInstanceOperations.java deleted file mode 100644 index e4aca9b..0000000 --- a/test/src/main/java/org/apache/accumulo/test/proxy/TestProxyInstanceOperations.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.proxy; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.Properties; - -import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.util.HostAndPort; -import org.apache.accumulo.proxy.Proxy; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TCompactProtocol; -import org.apache.thrift.server.TServer; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TestProxyInstanceOperations { - private static final Logger log = LoggerFactory.getLogger(TestProxyInstanceOperations.class); - - protected static TServer proxy; - protected static TestProxyClient tpc; - protected static ByteBuffer userpass; - protected static final int port = 10197; - - @BeforeClass - public static void setup() throws Exception { - Properties proxyProps = new Properties(); - proxyProps.put("tokenClass", PasswordToken.class.getName()); - - proxy = Proxy.createProxyServer(HostAndPort.fromParts("localhost", port), - new TCompactProtocol.Factory(), proxyProps).server; - log.info("Waiting for proxy to start"); - while (!proxy.isServing()) { - Thread.sleep(500); - } - log.info("Proxy started"); - tpc = new TestProxyClient("localhost", port); - userpass = tpc.proxy.login("root", Collections.singletonMap("password", "")); - } - - @AfterClass - public static void tearDown() throws InterruptedException { - proxy.stop(); - } - - @Test - public void properties() throws TException { - tpc.proxy().setProperty(userpass, "test.systemprop", "whistletips"); - - assertEquals(tpc.proxy().getSystemConfiguration(userpass).get("test.systemprop"), - "whistletips"); - tpc.proxy().removeProperty(userpass, "test.systemprop"); - assertNull(tpc.proxy().getSystemConfiguration(userpass).get("test.systemprop")); - - } - - @Test - public void testClassLoad() throws TException { - assertTrue( - tpc.proxy().testClassLoad(userpass, "org.apache.accumulo.core.iterators.user.RegExFilter", - "org.apache.accumulo.core.iterators.Filter")); - } - -} diff --git a/test/src/main/java/org/apache/accumulo/test/proxy/TestProxyNamespaceOperations.java b/test/src/main/java/org/apache/accumulo/test/proxy/TestProxyNamespaceOperations.java deleted file mode 100644 index 526fe49..0000000 --- a/test/src/main/java/org/apache/accumulo/test/proxy/TestProxyNamespaceOperations.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.proxy; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Properties; -import java.util.Set; - -import org.apache.accumulo.core.client.impl.Namespace; -import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.util.HostAndPort; -import org.apache.accumulo.proxy.Proxy; -import org.apache.accumulo.proxy.thrift.AccumuloException; -import org.apache.accumulo.proxy.thrift.IteratorScope; -import org.apache.accumulo.proxy.thrift.IteratorSetting; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TCompactProtocol; -import org.apache.thrift.server.TServer; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; - -public class TestProxyNamespaceOperations { - - protected static TServer proxy; - protected static TestProxyClient tpc; - protected static ByteBuffer userpass; - protected static final int port = 10198; - protected static final String testnamespace = "testns"; - - @BeforeClass - public static void setup() throws Exception { - Properties prop = new Properties(); - prop.put("tokenClass", PasswordToken.class.getName()); - - proxy = Proxy.createProxyServer(HostAndPort.fromParts("localhost", port), - new TCompactProtocol.Factory(), prop).server; - while (!proxy.isServing()) { - Thread.sleep(500); - } - tpc = new TestProxyClient("localhost", port); - userpass = tpc.proxy().login("root", Collections.singletonMap("password", "")); - } - - @AfterClass - public static void tearDown() throws InterruptedException { - proxy.stop(); - } - - @Before - public void makeTestNamespace() throws Exception { - tpc.proxy().createNamespace(userpass, testnamespace); - } - - @After - public void deleteTestNamespace() throws Exception { - tpc.proxy().deleteNamespace(userpass, testnamespace); - } - - @Test - public void createExistsDelete() throws TException { - tpc.proxy().createNamespace(userpass, "testns2"); - assertTrue(tpc.proxy().namespaceExists(userpass, "testns2")); - tpc.proxy().deleteNamespace(userpass, "testns2"); - assertFalse(tpc.proxy().namespaceExists(userpass, "testns2")); - } - - @Test - public void listRename() throws TException { - assertFalse(tpc.proxy().namespaceExists(userpass, "testns2")); - tpc.proxy().renameNamespace(userpass, testnamespace, "testns2"); - assertTrue(tpc.proxy().namespaceExists(userpass, "testns2")); - tpc.proxy().renameNamespace(userpass, "testns2", testnamespace); - assertTrue(tpc.proxy().listNamespaces(userpass).contains(testnamespace)); - assertFalse(tpc.proxy().listNamespaces(userpass).contains("testns2")); - } - - @Test - public void systemDefault() throws TException { - assertEquals(tpc.proxy().systemNamespace(), Namespace.ACCUMULO); - assertEquals(tpc.proxy().defaultNamespace(), Namespace.DEFAULT); - } - - @Test - public void namespaceProperties() throws TException { - tpc.proxy().setNamespaceProperty(userpass, testnamespace, "test.property1", "wharrrgarbl"); - assertEquals(tpc.proxy().getNamespaceProperties(userpass, testnamespace).get("test.property1"), - "wharrrgarbl"); - tpc.proxy().removeNamespaceProperty(userpass, testnamespace, "test.property1"); - assertNull(tpc.proxy().getNamespaceProperties(userpass, testnamespace).get("test.property1")); - } - - @Ignore("MockInstance doesn't return expected results for this function.") - @Test - public void namespaceIds() throws TException { - assertTrue(tpc.proxy().namespaceIdMap(userpass).containsKey("accumulo")); - assertEquals(tpc.proxy().namespaceIdMap(userpass).get("accumulo"), "+accumulo"); - } - - @Test - public void namespaceIterators() throws TException { - IteratorSetting setting = new IteratorSetting(40, "DebugTheThings", - "org.apache.accumulo.core.iterators.DebugIterator", new HashMap<>()); - Set scopes = new HashSet<>(); - scopes.add(IteratorScope.SCAN); - tpc.proxy().attachNamespaceIterator(userpass, testnamespace, setting, scopes); - assertEquals(setting, tpc.proxy().getNamespaceIteratorSetting(userpass, testnamespace, - "DebugTheThings", IteratorScope.SCAN)); - assertTrue( - tpc.proxy().listNamespaceIterators(userpass, testnamespace).containsKey("DebugTheThings")); - Set scopes2 = new HashSet<>(); - scopes2.add(IteratorScope.MINC); - tpc.proxy().checkNamespaceIteratorConflicts(userpass, testnamespace, setting, scopes2); - tpc.proxy().removeNamespaceIterator(userpass, testnamespace, "DebugTheThings", scopes); - assertFalse( - tpc.proxy().listNamespaceIterators(userpass, testnamespace).containsKey("DebugTheThings")); - } - - @Test(expected = AccumuloException.class) - public void namespaceIteratorConflict() throws TException { - IteratorSetting setting = new IteratorSetting(40, "DebugTheThings", - "org.apache.accumulo.core.iterators.DebugIterator", new HashMap<>()); - Set scopes = new HashSet<>(); - scopes.add(IteratorScope.SCAN); - tpc.proxy().attachNamespaceIterator(userpass, testnamespace, setting, scopes); - tpc.proxy().checkNamespaceIteratorConflicts(userpass, testnamespace, setting, scopes); - } - - @Test - public void namespaceConstraints() throws TException { - int constraintId = tpc.proxy().addNamespaceConstraint(userpass, testnamespace, - "org.apache.accumulo.test.constraints.MaxMutationSize"); - assertTrue(tpc.proxy().listNamespaceConstraints(userpass, testnamespace) - .containsKey("org.apache.accumulo.test.constraints.MaxMutationSize")); - assertEquals(constraintId, (int) tpc.proxy().listNamespaceConstraints(userpass, testnamespace) - .get("org.apache.accumulo.test.constraints.MaxMutationSize")); - tpc.proxy().removeNamespaceConstraint(userpass, testnamespace, constraintId); - assertFalse(tpc.proxy().listNamespaceConstraints(userpass, testnamespace) - .containsKey("org.apache.accumulo.test.constraints.MaxMutationSize")); - } - - @Test - public void classLoad() throws TException { - assertTrue(tpc.proxy().testNamespaceClassLoad(userpass, testnamespace, - "org.apache.accumulo.core.iterators.user.VersioningIterator", - "org.apache.accumulo.core.iterators.SortedKeyValueIterator")); - assertFalse(tpc.proxy().testNamespaceClassLoad(userpass, testnamespace, - "org.apache.accumulo.core.iterators.user.VersioningIterator", "dummy")); - } -} diff --git a/test/src/main/java/org/apache/accumulo/test/proxy/TestProxyReadWrite.java b/test/src/main/java/org/apache/accumulo/test/proxy/TestProxyReadWrite.java deleted file mode 100644 index 685c435..0000000 --- a/test/src/main/java/org/apache/accumulo/test/proxy/TestProxyReadWrite.java +++ /dev/null @@ -1,486 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.proxy; - -import static org.junit.Assert.assertEquals; - -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - -import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.iterators.user.RegExFilter; -import org.apache.accumulo.core.util.HostAndPort; -import org.apache.accumulo.proxy.Proxy; -import org.apache.accumulo.proxy.Util; -import org.apache.accumulo.proxy.thrift.BatchScanOptions; -import org.apache.accumulo.proxy.thrift.ColumnUpdate; -import org.apache.accumulo.proxy.thrift.IteratorSetting; -import org.apache.accumulo.proxy.thrift.Key; -import org.apache.accumulo.proxy.thrift.KeyValue; -import org.apache.accumulo.proxy.thrift.Range; -import org.apache.accumulo.proxy.thrift.ScanColumn; -import org.apache.accumulo.proxy.thrift.ScanOptions; -import org.apache.accumulo.proxy.thrift.ScanResult; -import org.apache.accumulo.proxy.thrift.TimeType; -import org.apache.thrift.protocol.TCompactProtocol; -import org.apache.thrift.server.TServer; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TestProxyReadWrite { - protected static TServer proxy; - protected static TestProxyClient tpc; - protected static ByteBuffer userpass; - protected static final int port = 10194; - protected static final String testtable = "testtable"; - - @BeforeClass - public static void setup() throws Exception { - Properties prop = new Properties(); - prop.put("tokenClass", PasswordToken.class.getName()); - - proxy = Proxy.createProxyServer(HostAndPort.fromParts("localhost", port), - new TCompactProtocol.Factory(), prop).server; - tpc = new TestProxyClient("localhost", port); - userpass = tpc.proxy().login("root", Collections.singletonMap("password", "")); - } - - @AfterClass - public static void tearDown() throws InterruptedException { - proxy.stop(); - } - - @Before - public void makeTestTable() throws Exception { - tpc.proxy().createTable(userpass, testtable, true, TimeType.MILLIS); - } - - @After - public void deleteTestTable() throws Exception { - tpc.proxy().deleteTable(userpass, testtable); - } - - private static void addMutation(Map> mutations, String row, - String cf, String cq, String value) { - ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(cf.getBytes()), - ByteBuffer.wrap(cq.getBytes())); - update.setValue(value.getBytes()); - mutations.put(ByteBuffer.wrap(row.getBytes()), Collections.singletonList(update)); - } - - private static void addMutation(Map> mutations, String row, - String cf, String cq, String vis, String value) { - ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(cf.getBytes()), - ByteBuffer.wrap(cq.getBytes())); - update.setValue(value.getBytes()); - update.setColVisibility(vis.getBytes()); - mutations.put(ByteBuffer.wrap(row.getBytes()), Collections.singletonList(update)); - } - - /** - * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a range so only - * the entries between -Inf...5 come back (there should be 50,000) - */ - @Test - public void readWriteBatchOneShotWithRange() throws Exception { - int maxInserts = 100000; - Map> mutations = new HashMap<>(); - String format = "%1$05d"; - for (int i = 0; i < maxInserts; i++) { - addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10)); - - if (i % 1000 == 0 || i == maxInserts - 1) { - tpc.proxy().updateAndFlush(userpass, testtable, mutations); - mutations.clear(); - } - } - - Key stop = new Key(); - stop.setRow("5".getBytes()); - BatchScanOptions options = new BatchScanOptions(); - options.ranges = Collections.singletonList(new Range(null, false, stop, false)); - String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options); - - int i = 0; - boolean hasNext = true; - - int k = 1000; - while (hasNext) { - ScanResult kvList = tpc.proxy().nextK(cookie, k); - i += kvList.getResultsSize(); - hasNext = kvList.isMore(); - } - assertEquals(i, 50000); - } - - /** - * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a columnFamily so - * only the entries with specified column family come back (there should be 50,000) - */ - @Test - public void readWriteBatchOneShotWithColumnFamilyOnly() throws Exception { - int maxInserts = 100000; - Map> mutations = new HashMap<>(); - String format = "%1$05d"; - for (int i = 0; i < maxInserts; i++) { - - addMutation(mutations, String.format(format, i), "cf" + (i % 2), "cq" + (i % 2), - Util.randString(10)); - - if (i % 1000 == 0 || i == maxInserts - 1) { - tpc.proxy().updateAndFlush(userpass, testtable, mutations); - mutations.clear(); - } - } - - BatchScanOptions options = new BatchScanOptions(); - - ScanColumn sc = new ScanColumn(); - sc.colFamily = ByteBuffer.wrap("cf0".getBytes()); - - options.columns = Collections.singletonList(sc); - String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options); - - int i = 0; - boolean hasNext = true; - - int k = 1000; - while (hasNext) { - ScanResult kvList = tpc.proxy().nextK(cookie, k); - i += kvList.getResultsSize(); - hasNext = kvList.isMore(); - } - assertEquals(i, 50000); - } - - /** - * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a columnFamily + - * columnQualififer so only the entries with specified column come back (there should be 50,000) - */ - @Test - public void readWriteBatchOneShotWithFullColumn() throws Exception { - int maxInserts = 100000; - Map> mutations = new HashMap<>(); - String format = "%1$05d"; - for (int i = 0; i < maxInserts; i++) { - - addMutation(mutations, String.format(format, i), "cf" + (i % 2), "cq" + (i % 2), - Util.randString(10)); - - if (i % 1000 == 0 || i == maxInserts - 1) { - tpc.proxy().updateAndFlush(userpass, testtable, mutations); - mutations.clear(); - } - } - - BatchScanOptions options = new BatchScanOptions(); - - ScanColumn sc = new ScanColumn(); - sc.colFamily = ByteBuffer.wrap("cf0".getBytes()); - sc.colQualifier = ByteBuffer.wrap("cq0".getBytes()); - - options.columns = Collections.singletonList(sc); - String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options); - - int i = 0; - boolean hasNext = true; - - int k = 1000; - while (hasNext) { - ScanResult kvList = tpc.proxy().nextK(cookie, k); - i += kvList.getResultsSize(); - hasNext = kvList.isMore(); - } - assertEquals(i, 50000); - } - - /** - * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Filter the results so - * only the even numbers come back. - */ - @Test - public void readWriteBatchOneShotWithFilterIterator() throws Exception { - int maxInserts = 10000; - Map> mutations = new HashMap<>(); - String format = "%1$05d"; - for (int i = 0; i < maxInserts; i++) { - addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10)); - - if (i % 1000 == 0 || i == maxInserts - 1) { - tpc.proxy().updateAndFlush(userpass, testtable, mutations); - mutations.clear(); - } - - } - - String regex = ".*[02468]"; - - // @formatter:off - org.apache.accumulo.core.client.IteratorSetting is = - new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class); - // @formatter:on - RegExFilter.setRegexs(is, regex, null, null, null, false); - - IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is); - ScanOptions opts = new ScanOptions(); - opts.iterators = Collections.singletonList(pis); - String cookie = tpc.proxy().createScanner(userpass, testtable, opts); - - int i = 0; - boolean hasNext = true; - - int k = 1000; - while (hasNext) { - ScanResult kvList = tpc.proxy().nextK(cookie, k); - for (KeyValue kv : kvList.getResults()) { - assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i); - - i += 2; - } - hasNext = kvList.isMore(); - } - } - - @Test - public void readWriteOneShotWithRange() throws Exception { - int maxInserts = 100000; - Map> mutations = new HashMap<>(); - String format = "%1$05d"; - for (int i = 0; i < maxInserts; i++) { - addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10)); - - if (i % 1000 == 0 || i == maxInserts - 1) { - tpc.proxy().updateAndFlush(userpass, testtable, mutations); - mutations.clear(); - } - } - - Key stop = new Key(); - stop.setRow("5".getBytes()); - ScanOptions opts = new ScanOptions(); - opts.range = new Range(null, false, stop, false); - String cookie = tpc.proxy().createScanner(userpass, testtable, opts); - - int i = 0; - boolean hasNext = true; - - int k = 1000; - while (hasNext) { - ScanResult kvList = tpc.proxy().nextK(cookie, k); - i += kvList.getResultsSize(); - hasNext = kvList.isMore(); - } - assertEquals(i, 50000); - } - - /** - * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Filter the results so - * only the even numbers come back. - */ - @Test - public void readWriteOneShotWithFilterIterator() throws Exception { - int maxInserts = 10000; - Map> mutations = new HashMap<>(); - String format = "%1$05d"; - for (int i = 0; i < maxInserts; i++) { - addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10)); - - if (i % 1000 == 0 || i == maxInserts - 1) { - - tpc.proxy().updateAndFlush(userpass, testtable, mutations); - mutations.clear(); - - } - - } - - String regex = ".*[02468]"; - - // @formatter:off - org.apache.accumulo.core.client.IteratorSetting is = - new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class); - // @formatter:on - RegExFilter.setRegexs(is, regex, null, null, null, false); - - IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is); - ScanOptions opts = new ScanOptions(); - opts.iterators = Collections.singletonList(pis); - String cookie = tpc.proxy().createScanner(userpass, testtable, opts); - - int i = 0; - boolean hasNext = true; - - int k = 1000; - while (hasNext) { - ScanResult kvList = tpc.proxy().nextK(cookie, k); - for (KeyValue kv : kvList.getResults()) { - assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i); - - i += 2; - } - hasNext = kvList.isMore(); - } - } - - // @Test - // This test takes kind of a long time. Enable it if you think you may have memory issues. - public void manyWritesAndReads() throws Exception { - int maxInserts = 1000000; - Map> mutations = new HashMap<>(); - String format = "%1$06d"; - String writer = tpc.proxy().createWriter(userpass, testtable, null); - for (int i = 0; i < maxInserts; i++) { - addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10)); - - if (i % 1000 == 0 || i == maxInserts - 1) { - - tpc.proxy().update(writer, mutations); - mutations.clear(); - - } - - } - - tpc.proxy().flush(writer); - tpc.proxy().closeWriter(writer); - - String cookie = tpc.proxy().createScanner(userpass, testtable, null); - - int i = 0; - boolean hasNext = true; - - int k = 1000; - while (hasNext) { - ScanResult kvList = tpc.proxy().nextK(cookie, k); - for (KeyValue kv : kvList.getResults()) { - assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i); - i++; - } - hasNext = kvList.isMore(); - if (hasNext) - assertEquals(k, kvList.getResults().size()); - } - assertEquals(maxInserts, i); - } - - @Test - public void asynchReadWrite() throws Exception { - int maxInserts = 10000; - Map> mutations = new HashMap<>(); - String format = "%1$05d"; - String writer = tpc.proxy().createWriter(userpass, testtable, null); - for (int i = 0; i < maxInserts; i++) { - addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10)); - - if (i % 1000 == 0 || i == maxInserts - 1) { - tpc.proxy().update(writer, mutations); - mutations.clear(); - } - } - - tpc.proxy().flush(writer); - tpc.proxy().closeWriter(writer); - - String regex = ".*[02468]"; - - // @formatter:off - org.apache.accumulo.core.client.IteratorSetting is = - new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class); - // @formatter:on - RegExFilter.setRegexs(is, regex, null, null, null, false); - - IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is); - ScanOptions opts = new ScanOptions(); - opts.iterators = Collections.singletonList(pis); - String cookie = tpc.proxy().createScanner(userpass, testtable, opts); - - int i = 0; - boolean hasNext = true; - - int k = 1000; - int numRead = 0; - while (hasNext) { - ScanResult kvList = tpc.proxy().nextK(cookie, k); - for (KeyValue kv : kvList.getResults()) { - assertEquals(i, Integer.parseInt(new String(kv.getKey().getRow()))); - numRead++; - i += 2; - } - hasNext = kvList.isMore(); - } - assertEquals(maxInserts / 2, numRead); - } - - @Test - public void testVisibility() throws Exception { - - Set auths = new HashSet<>(); - auths.add(ByteBuffer.wrap("even".getBytes())); - tpc.proxy().changeUserAuthorizations(userpass, "root", auths); - - int maxInserts = 10000; - Map> mutations = new HashMap<>(); - String format = "%1$05d"; - String writer = tpc.proxy().createWriter(userpass, testtable, null); - for (int i = 0; i < maxInserts; i++) { - if (i % 2 == 0) - addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, "even", - Util.randString(10)); - else - addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, "odd", - Util.randString(10)); - - if (i % 1000 == 0 || i == maxInserts - 1) { - tpc.proxy().update(writer, mutations); - mutations.clear(); - } - } - - tpc.proxy().flush(writer); - tpc.proxy().closeWriter(writer); - ScanOptions opts = new ScanOptions(); - opts.authorizations = auths; - String cookie = tpc.proxy().createScanner(userpass, testtable, opts); - - int i = 0; - boolean hasNext = true; - - int k = 1000; - int numRead = 0; - while (hasNext) { - ScanResult kvList = tpc.proxy().nextK(cookie, k); - for (KeyValue kv : kvList.getResults()) { - assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i); - i += 2; - numRead++; - } - hasNext = kvList.isMore(); - - } - assertEquals(maxInserts / 2, numRead); - } - -} diff --git a/test/src/main/java/org/apache/accumulo/test/proxy/TestProxySecurityOperations.java b/test/src/main/java/org/apache/accumulo/test/proxy/TestProxySecurityOperations.java deleted file mode 100644 index d1f905f..0000000 --- a/test/src/main/java/org/apache/accumulo/test/proxy/TestProxySecurityOperations.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.proxy; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.TreeMap; - -import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.util.ByteBufferUtil; -import org.apache.accumulo.core.util.HostAndPort; -import org.apache.accumulo.proxy.Proxy; -import org.apache.accumulo.proxy.thrift.NamespacePermission; -import org.apache.accumulo.proxy.thrift.SystemPermission; -import org.apache.accumulo.proxy.thrift.TablePermission; -import org.apache.accumulo.proxy.thrift.TimeType; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TCompactProtocol; -import org.apache.thrift.server.TServer; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TestProxySecurityOperations { - protected static TServer proxy; - protected static TestProxyClient tpc; - protected static ByteBuffer userpass; - protected static final int port = 10196; - protected static final String testtable = "testtable"; - protected static final String testuser = "VonJines"; - protected static final String testnamespace = "testns"; - protected static final ByteBuffer testpw = ByteBuffer.wrap("fiveones".getBytes()); - - @BeforeClass - public static void setup() throws Exception { - Properties prop = new Properties(); - prop.put("tokenClass", PasswordToken.class.getName()); - - proxy = Proxy.createProxyServer(HostAndPort.fromParts("localhost", port), - new TCompactProtocol.Factory(), prop).server; - while (!proxy.isServing()) { - Thread.sleep(500); - } - tpc = new TestProxyClient("localhost", port); - userpass = tpc.proxy().login("root", Collections.singletonMap("password", "")); - } - - @AfterClass - public static void tearDown() throws InterruptedException { - proxy.stop(); - } - - @Before - public void makeTestTableAndUser() throws Exception { - tpc.proxy().createTable(userpass, testtable, true, TimeType.MILLIS); - tpc.proxy().createLocalUser(userpass, testuser, testpw); - tpc.proxy().createNamespace(userpass, testnamespace); - } - - @After - public void deleteTestTable() throws Exception { - tpc.proxy().deleteTable(userpass, testtable); - tpc.proxy().dropLocalUser(userpass, testuser); - tpc.proxy().deleteNamespace(userpass, testnamespace); - } - - @Test - public void create() throws TException { - tpc.proxy().createLocalUser(userpass, testuser + "2", testpw); - assertTrue(tpc.proxy().listLocalUsers(userpass).contains(testuser + "2")); - tpc.proxy().dropLocalUser(userpass, testuser + "2"); - assertTrue(!tpc.proxy().listLocalUsers(userpass).contains(testuser + "2")); - } - - @Test - public void authenticate() throws TException { - assertTrue(tpc.proxy().authenticateUser(userpass, testuser, bb2pp(testpw))); - assertFalse(tpc.proxy().authenticateUser(userpass, "EvilUser", bb2pp(testpw))); - - tpc.proxy().changeLocalUserPassword(userpass, testuser, ByteBuffer.wrap("newpass".getBytes())); - assertFalse(tpc.proxy().authenticateUser(userpass, testuser, bb2pp(testpw))); - assertTrue(tpc.proxy().authenticateUser(userpass, testuser, - bb2pp(ByteBuffer.wrap("newpass".getBytes())))); - - } - - @Test - public void tablePermissions() throws TException { - tpc.proxy().grantTablePermission(userpass, testuser, testtable, TablePermission.ALTER_TABLE); - assertTrue( - tpc.proxy().hasTablePermission(userpass, testuser, testtable, TablePermission.ALTER_TABLE)); - - tpc.proxy().revokeTablePermission(userpass, testuser, testtable, TablePermission.ALTER_TABLE); - assertFalse( - tpc.proxy().hasTablePermission(userpass, testuser, testtable, TablePermission.ALTER_TABLE)); - - } - - @Test - public void systemPermissions() throws TException { - tpc.proxy().grantSystemPermission(userpass, testuser, SystemPermission.ALTER_USER); - assertTrue(tpc.proxy().hasSystemPermission(userpass, testuser, SystemPermission.ALTER_USER)); - - tpc.proxy().revokeSystemPermission(userpass, testuser, SystemPermission.ALTER_USER); - assertFalse(tpc.proxy().hasSystemPermission(userpass, testuser, SystemPermission.ALTER_USER)); - - } - - @Test - public void auths() throws TException { - HashSet newauths = new HashSet<>(); - newauths.add(ByteBuffer.wrap("BBR".getBytes())); - newauths.add(ByteBuffer.wrap("Barney".getBytes())); - tpc.proxy().changeUserAuthorizations(userpass, testuser, newauths); - List actualauths = tpc.proxy().getUserAuthorizations(userpass, testuser); - assertEquals(actualauths.size(), newauths.size()); - - for (ByteBuffer auth : actualauths) { - assertTrue(newauths.contains(auth)); - } - } - - @Test - public void namespacePermissions() throws TException { - tpc.proxy().grantNamespacePermission(userpass, testuser, testnamespace, - NamespacePermission.ALTER_NAMESPACE); - assertTrue(tpc.proxy().hasNamespacePermission(userpass, testuser, testnamespace, - NamespacePermission.ALTER_NAMESPACE)); - - tpc.proxy().revokeNamespacePermission(userpass, testuser, testnamespace, - NamespacePermission.ALTER_NAMESPACE); - assertFalse(tpc.proxy().hasNamespacePermission(userpass, testuser, testnamespace, - NamespacePermission.ALTER_NAMESPACE)); - } - - private Map bb2pp(ByteBuffer cf) { - Map toRet = new TreeMap<>(); - toRet.put("password", ByteBufferUtil.toString(cf)); - return toRet; - } - -} diff --git a/test/src/main/java/org/apache/accumulo/test/proxy/TestProxyTableOperations.java b/test/src/main/java/org/apache/accumulo/test/proxy/TestProxyTableOperations.java deleted file mode 100644 index b8e14e2..0000000 --- a/test/src/main/java/org/apache/accumulo/test/proxy/TestProxyTableOperations.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.proxy; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - -import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.util.HostAndPort; -import org.apache.accumulo.proxy.Proxy; -import org.apache.accumulo.proxy.thrift.ColumnUpdate; -import org.apache.accumulo.proxy.thrift.TimeType; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TCompactProtocol; -import org.apache.thrift.server.TServer; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TestProxyTableOperations { - - protected static TServer proxy; - protected static TestProxyClient tpc; - protected static ByteBuffer userpass; - protected static final int port = 10195; - protected static final String testtable = "testtable"; - - @BeforeClass - public static void setup() throws Exception { - Properties prop = new Properties(); - prop.put("tokenClass", PasswordToken.class.getName()); - - proxy = Proxy.createProxyServer(HostAndPort.fromParts("localhost", port), - new TCompactProtocol.Factory(), prop).server; - while (!proxy.isServing()) { - Thread.sleep(500); - } - tpc = new TestProxyClient("localhost", port); - userpass = tpc.proxy().login("root", Collections.singletonMap("password", "")); - } - - @AfterClass - public static void tearDown() throws InterruptedException { - proxy.stop(); - } - - @Before - public void makeTestTable() throws Exception { - tpc.proxy().createTable(userpass, testtable, true, TimeType.MILLIS); - } - - @After - public void deleteTestTable() throws Exception { - tpc.proxy().deleteTable(userpass, testtable); - } - - @Test - public void createExistsDelete() throws TException { - assertFalse(tpc.proxy().tableExists(userpass, "testtable2")); - tpc.proxy().createTable(userpass, "testtable2", true, TimeType.MILLIS); - assertTrue(tpc.proxy().tableExists(userpass, "testtable2")); - tpc.proxy().deleteTable(userpass, "testtable2"); - assertFalse(tpc.proxy().tableExists(userpass, "testtable2")); - } - - @Test - public void listRename() throws TException { - assertFalse(tpc.proxy().tableExists(userpass, "testtable2")); - tpc.proxy().renameTable(userpass, testtable, "testtable2"); - assertTrue(tpc.proxy().tableExists(userpass, "testtable2")); - tpc.proxy().renameTable(userpass, "testtable2", testtable); - assertTrue(tpc.proxy().listTables(userpass).contains("testtable")); - - } - - // This test does not yet function because the backing Mock instance does not yet support merging - @Test - public void merge() throws TException { - Set splits = new HashSet<>(); - splits.add(ByteBuffer.wrap("a".getBytes())); - splits.add(ByteBuffer.wrap("c".getBytes())); - splits.add(ByteBuffer.wrap("z".getBytes())); - tpc.proxy().addSplits(userpass, testtable, splits); - - tpc.proxy().mergeTablets(userpass, testtable, ByteBuffer.wrap("b".getBytes()), - ByteBuffer.wrap("d".getBytes())); - - splits.remove(ByteBuffer.wrap("c".getBytes())); - - List tableSplits = tpc.proxy().listSplits(userpass, testtable, 10); - - for (ByteBuffer split : tableSplits) - assertTrue(splits.contains(split)); - assertEquals(tableSplits.size(), splits.size()); - - } - - @Test - public void splits() throws TException { - Set splits = new HashSet<>(); - splits.add(ByteBuffer.wrap("a".getBytes())); - splits.add(ByteBuffer.wrap("b".getBytes())); - splits.add(ByteBuffer.wrap("z".getBytes())); - tpc.proxy().addSplits(userpass, testtable, splits); - - List tableSplits = tpc.proxy().listSplits(userpass, testtable, 10); - - for (ByteBuffer split : tableSplits) - assertTrue(splits.contains(split)); - assertEquals(tableSplits.size(), splits.size()); - } - - @Test - public void constraints() throws TException { - int cid = tpc.proxy().addConstraint(userpass, testtable, "org.apache.accumulo.TestConstraint"); - Map constraints = tpc.proxy().listConstraints(userpass, testtable); - assertEquals((int) constraints.get("org.apache.accumulo.TestConstraint"), cid); - tpc.proxy().removeConstraint(userpass, testtable, cid); - constraints = tpc.proxy().listConstraints(userpass, testtable); - assertNull(constraints.get("org.apache.accumulo.TestConstraint")); - } - - @Test - public void localityGroups() throws TException { - Map> groups = new HashMap<>(); - Set group1 = new HashSet<>(); - group1.add("cf1"); - groups.put("group1", group1); - Set group2 = new HashSet<>(); - group2.add("cf2"); - group2.add("cf3"); - groups.put("group2", group2); - tpc.proxy().setLocalityGroups(userpass, testtable, groups); - - Map> actualGroups = tpc.proxy().getLocalityGroups(userpass, testtable); - - assertEquals(groups.size(), actualGroups.size()); - for (String groupName : groups.keySet()) { - assertTrue(actualGroups.containsKey(groupName)); - assertEquals(groups.get(groupName).size(), actualGroups.get(groupName).size()); - for (String cf : groups.get(groupName)) { - assertTrue(actualGroups.get(groupName).contains(cf)); - } - } - } - - @Test - public void tableProperties() throws TException { - tpc.proxy().setTableProperty(userpass, testtable, "test.property1", "wharrrgarbl"); - assertEquals(tpc.proxy().getTableProperties(userpass, testtable).get("test.property1"), - "wharrrgarbl"); - tpc.proxy().removeTableProperty(userpass, testtable, "test.property1"); - assertNull(tpc.proxy().getTableProperties(userpass, testtable).get("test.property1")); - } - - private static void addMutation(Map> mutations, String row, - String cf, String cq, String value) { - ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(cf.getBytes()), - ByteBuffer.wrap(cq.getBytes())); - update.setValue(value.getBytes()); - mutations.put(ByteBuffer.wrap(row.getBytes()), Collections.singletonList(update)); - } - - @Test - public void tableOperationsRowMethods() throws TException { - Map> mutations = new HashMap<>(); - for (int i = 0; i < 10; i++) { - addMutation(mutations, "" + i, "cf", "cq", ""); - } - tpc.proxy().updateAndFlush(userpass, testtable, mutations); - - assertEquals(tpc.proxy().getMaxRow(userpass, testtable, null, null, true, null, true), - ByteBuffer.wrap("9".getBytes())); - - tpc.proxy().deleteRows(userpass, testtable, ByteBuffer.wrap("51".getBytes()), - ByteBuffer.wrap("99".getBytes())); - assertEquals(tpc.proxy().getMaxRow(userpass, testtable, null, null, true, null, true), - ByteBuffer.wrap("5".getBytes())); - } - -} diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/EstimateInMemMapOverhead.java b/test/src/main/java/org/apache/accumulo/test/util/memory/EstimateInMemMapOverhead.java new file mode 100644 index 0000000..779b170 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/util/memory/EstimateInMemMapOverhead.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.util.memory; + +public class EstimateInMemMapOverhead { + + private static void check(int numEntries, int keyLen, int colFamLen, int colQualLen, + int colVisLen, int dataLen) { + new IntObjectMemoryUsageCheck(numEntries).run(); + new InMemoryMapMemoryUsageCheck(numEntries, keyLen, colFamLen, colQualLen, colVisLen, dataLen) + .run(); + new TextMemoryUsageCheck(numEntries, keyLen, colFamLen, colQualLen, dataLen).run(); + new MutationMemoryUsageCheck(numEntries, keyLen, colFamLen, colQualLen, dataLen).run(); + } + + public static void main(String[] args) { + check(10000, 10, 4, 4, 4, 20); + check(100000, 10, 4, 4, 4, 20); + check(500000, 10, 4, 4, 4, 20); + check(1000000, 10, 4, 4, 4, 20); + check(2000000, 10, 4, 4, 4, 20); + + check(10000, 20, 5, 5, 5, 500); + check(100000, 20, 5, 5, 5, 500); + check(500000, 20, 5, 5, 5, 500); + check(1000000, 20, 5, 5, 5, 500); + check(2000000, 20, 5, 5, 5, 500); + + check(10000, 40, 10, 10, 10, 1000); + check(100000, 40, 10, 10, 10, 1000); + check(500000, 40, 10, 10, 10, 1000); + check(1000000, 40, 10, 10, 10, 1000); + check(2000000, 40, 10, 10, 10, 1000); + } + +} diff --git a/test/src/main/java/org/apache/accumulo/test/InMemoryMapMemoryUsageTest.java b/test/src/main/java/org/apache/accumulo/test/util/memory/InMemoryMapMemoryUsageCheck.java similarity index 93% rename from test/src/main/java/org/apache/accumulo/test/InMemoryMapMemoryUsageTest.java rename to test/src/main/java/org/apache/accumulo/test/util/memory/InMemoryMapMemoryUsageCheck.java index 302a688..83fe8f0 100644 --- a/test/src/main/java/org/apache/accumulo/test/InMemoryMapMemoryUsageTest.java +++ b/test/src/main/java/org/apache/accumulo/test/util/memory/InMemoryMapMemoryUsageCheck.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.accumulo.test; +package org.apache.accumulo.test.util.memory; import java.util.Collections; @@ -26,7 +26,7 @@ import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfiguratio import org.apache.accumulo.tserver.InMemoryMap; import org.apache.hadoop.io.Text; -class InMemoryMapMemoryUsageTest extends MemoryUsageTest { +class InMemoryMapMemoryUsageCheck extends MemoryUsageCheck { private int keyLen; private int colFamLen; @@ -41,7 +41,7 @@ class InMemoryMapMemoryUsageTest extends MemoryUsageTest { private ColumnVisibility colv; private int passes; - InMemoryMapMemoryUsageTest(int passes, int keyLen, int colFamLen, int colQualLen, int colVisLen, + InMemoryMapMemoryUsageCheck(int passes, int keyLen, int colFamLen, int colQualLen, int colVisLen, int dataLen) { this.keyLen = keyLen; this.colFamLen = colFamLen; diff --git a/test/src/main/java/org/apache/accumulo/test/IntObjectMemoryUsageTest.java b/test/src/main/java/org/apache/accumulo/test/util/memory/IntObjectMemoryUsageCheck.java similarity index 89% rename from test/src/main/java/org/apache/accumulo/test/IntObjectMemoryUsageTest.java rename to test/src/main/java/org/apache/accumulo/test/util/memory/IntObjectMemoryUsageCheck.java index 7c135e7..74016a2 100644 --- a/test/src/main/java/org/apache/accumulo/test/IntObjectMemoryUsageTest.java +++ b/test/src/main/java/org/apache/accumulo/test/util/memory/IntObjectMemoryUsageCheck.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.accumulo.test; +package org.apache.accumulo.test.util.memory; -class IntObjectMemoryUsageTest extends MemoryUsageTest { +class IntObjectMemoryUsageCheck extends MemoryUsageCheck { private int passes; private Object data[]; @@ -29,7 +29,7 @@ class IntObjectMemoryUsageTest extends MemoryUsageTest { } } - IntObjectMemoryUsageTest(int numPasses) { + IntObjectMemoryUsageCheck(int numPasses) { this.passes = numPasses; } diff --git a/test/src/main/java/org/apache/accumulo/test/MemoryUsageTest.java b/test/src/main/java/org/apache/accumulo/test/util/memory/MemoryUsageCheck.java similarity index 96% rename from test/src/main/java/org/apache/accumulo/test/MemoryUsageTest.java rename to test/src/main/java/org/apache/accumulo/test/util/memory/MemoryUsageCheck.java index 047822d..7e60124 100644 --- a/test/src/main/java/org/apache/accumulo/test/MemoryUsageTest.java +++ b/test/src/main/java/org/apache/accumulo/test/util/memory/MemoryUsageCheck.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.accumulo.test; +package org.apache.accumulo.test.util.memory; -abstract class MemoryUsageTest { +abstract class MemoryUsageCheck { abstract void addEntry(int i); abstract int getEstimatedBytesPerEntry(); diff --git a/test/src/main/java/org/apache/accumulo/test/MutationMemoryUsageTest.java b/test/src/main/java/org/apache/accumulo/test/util/memory/MutationMemoryUsageCheck.java similarity index 92% rename from test/src/main/java/org/apache/accumulo/test/MutationMemoryUsageTest.java rename to test/src/main/java/org/apache/accumulo/test/util/memory/MutationMemoryUsageCheck.java index 6ded7db..f60d347 100644 --- a/test/src/main/java/org/apache/accumulo/test/MutationMemoryUsageTest.java +++ b/test/src/main/java/org/apache/accumulo/test/util/memory/MutationMemoryUsageCheck.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.accumulo.test; +package org.apache.accumulo.test.util.memory; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.hadoop.io.Text; -class MutationMemoryUsageTest extends MemoryUsageTest { +class MutationMemoryUsageCheck extends MemoryUsageCheck { private int keyLen; private int colFamLen; @@ -33,7 +33,7 @@ class MutationMemoryUsageTest extends MemoryUsageTest { private Text colq; private int passes; - MutationMemoryUsageTest(int passes, int keyLen, int colFamLen, int colQualLen, int dataLen) { + MutationMemoryUsageCheck(int passes, int keyLen, int colFamLen, int colQualLen, int dataLen) { this.keyLen = keyLen; this.colFamLen = colFamLen; this.colQualLen = colQualLen; diff --git a/test/src/main/java/org/apache/accumulo/test/TextMemoryUsageTest.java b/test/src/main/java/org/apache/accumulo/test/util/memory/TextMemoryUsageCheck.java similarity index 91% rename from test/src/main/java/org/apache/accumulo/test/TextMemoryUsageTest.java rename to test/src/main/java/org/apache/accumulo/test/util/memory/TextMemoryUsageCheck.java index d6d8299..72d8a33 100644 --- a/test/src/main/java/org/apache/accumulo/test/TextMemoryUsageTest.java +++ b/test/src/main/java/org/apache/accumulo/test/util/memory/TextMemoryUsageCheck.java @@ -14,14 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.accumulo.test; +package org.apache.accumulo.test.util.memory; import java.util.TreeMap; import org.apache.accumulo.core.data.Value; import org.apache.hadoop.io.Text; -class TextMemoryUsageTest extends MemoryUsageTest { +class TextMemoryUsageCheck extends MemoryUsageCheck { private int keyLen; private int colFamLen; @@ -30,7 +30,7 @@ class TextMemoryUsageTest extends MemoryUsageTest { private TreeMap map; private int passes; - TextMemoryUsageTest(int passes, int keyLen, int colFamLen, int colQualLen, int dataLen) { + TextMemoryUsageCheck(int passes, int keyLen, int colFamLen, int colQualLen, int dataLen) { this.keyLen = keyLen; this.colFamLen = colFamLen; this.colQualLen = colQualLen; diff --git a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java b/test/src/main/java/org/apache/accumulo/test/util/metadata/MetadataBatchScan.java similarity index 97% rename from test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java rename to test/src/main/java/org/apache/accumulo/test/util/metadata/MetadataBatchScan.java index 7bd8e32..9118d3d 100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java +++ b/test/src/main/java/org/apache/accumulo/test/util/metadata/MetadataBatchScan.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.accumulo.test.performance.metadata; +package org.apache.accumulo.test.util.metadata; import static java.nio.charset.StandardCharsets.UTF_8; @@ -54,14 +54,14 @@ import com.google.common.collect.Iterators; * This little program can be used to write a lot of metadata entries and measure the performance of * varying numbers of threads doing metadata lookups using the batch scanner. */ -public class MetadataBatchScanTest { +public class MetadataBatchScan { - private static final Logger log = LoggerFactory.getLogger(MetadataBatchScanTest.class); + private static final Logger log = LoggerFactory.getLogger(MetadataBatchScan.class); public static void main(String[] args) throws Exception { ClientOpts opts = new ClientOpts(); - opts.parseArgs(MetadataBatchScanTest.class.getName(), args); + opts.parseArgs(MetadataBatchScan.class.getName(), args); final Connector connector = opts.getConnector(); TreeSet splits = new TreeSet<>(); diff --git a/test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java b/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapConcurrency.java similarity index 96% rename from test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java rename to test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapConcurrency.java index e0b71e1..5fe214e 100644 --- a/test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java +++ b/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapConcurrency.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.accumulo.test; +package org.apache.accumulo.test.util.nativemap; import static java.nio.charset.StandardCharsets.UTF_8; @@ -33,9 +33,9 @@ import org.slf4j.LoggerFactory; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -public class NativeMapConcurrencyTest { +public class NativeMapConcurrency { - private static final Logger log = LoggerFactory.getLogger(NativeMapConcurrencyTest.class); + private static final Logger log = LoggerFactory.getLogger(NativeMapConcurrency.class); private static final byte ROW_PREFIX[] = {'r'}; private static final byte COL_PREFIX[] = {'c'}; @@ -97,7 +97,7 @@ public class NativeMapConcurrencyTest { public static void main(String[] args) { Opts opts = new Opts(); JCommander jc = new JCommander(opts); - jc.setProgramName(NativeMapConcurrencyTest.class.getName()); + jc.setProgramName(NativeMapConcurrency.class.getName()); jc.parse(args); if (opts.help) { jc.usage(); diff --git a/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java b/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapPerformance.java similarity index 97% rename from test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java rename to test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapPerformance.java index 8d83ebd..2673aee 100644 --- a/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java +++ b/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapPerformance.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.accumulo.test; +package org.apache.accumulo.test.util.nativemap; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly; @@ -35,7 +35,7 @@ import org.apache.accumulo.core.util.FastFormat; import org.apache.accumulo.tserver.NativeMap; import org.apache.hadoop.io.Text; -public class NativeMapPerformanceTest { +public class NativeMapPerformance { private static final byte ROW_PREFIX[] = {'r'}; private static final byte COL_PREFIX[] = {'c'}; @@ -188,7 +188,7 @@ public class NativeMapPerformanceTest { if (args.length != 3) { throw new IllegalArgumentException( - "Usage : " + NativeMapPerformanceTest.class.getName() + " "); + "Usage : " + NativeMapPerformance.class.getName() + " "); } String mapType = args[0]; diff --git a/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java b/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapStress.java similarity index 98% rename from test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java rename to test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapStress.java index 2f84815..3854cd3 100644 --- a/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java +++ b/test/src/main/java/org/apache/accumulo/test/util/nativemap/NativeMapStress.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.accumulo.test; +package org.apache.accumulo.test.util.nativemap; import static java.nio.charset.StandardCharsets.UTF_8; @@ -37,9 +37,9 @@ import org.apache.hadoop.io.Text; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class NativeMapStressTest { +public class NativeMapStress { - private static final Logger log = LoggerFactory.getLogger(NativeMapStressTest.class); + private static final Logger log = LoggerFactory.getLogger(NativeMapStress.class); public static void main(String[] args) { testLotsOfMapDeletes(true); diff --git a/test/src/main/java/org/apache/accumulo/test/WrongTabletTest.java b/test/src/main/java/org/apache/accumulo/test/util/tablet/WrongTablet.java similarity index 94% rename from test/src/main/java/org/apache/accumulo/test/WrongTabletTest.java rename to test/src/main/java/org/apache/accumulo/test/util/tablet/WrongTablet.java index 0525377..47b5a4a 100644 --- a/test/src/main/java/org/apache/accumulo/test/WrongTabletTest.java +++ b/test/src/main/java/org/apache/accumulo/test/util/tablet/WrongTablet.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.accumulo.test; +package org.apache.accumulo.test.util.tablet; import org.apache.accumulo.core.client.impl.ClientContext; import org.apache.accumulo.core.client.impl.Table; @@ -30,7 +30,7 @@ import org.apache.hadoop.io.Text; import com.beust.jcommander.Parameter; -public class WrongTabletTest { +public class WrongTablet { static class Opts extends ClientOpts { @Parameter(names = "--location", required = true) @@ -39,7 +39,7 @@ public class WrongTabletTest { public static void main(String[] args) { final Opts opts = new Opts(); - opts.parseArgs(WrongTabletTest.class.getName(), args); + opts.parseArgs(WrongTablet.class.getName(), args); final HostAndPort location = HostAndPort.fromString(opts.location); final ClientContext context = opts.getClientContext();