Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 502DE1852A for ; Wed, 30 Sep 2015 15:13:00 +0000 (UTC) Received: (qmail 11756 invoked by uid 500); 30 Sep 2015 15:12:50 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 11651 invoked by uid 500); 30 Sep 2015 15:12:50 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 11566 invoked by uid 99); 30 Sep 2015 15:12:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 Sep 2015 15:12:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9401AE08DF; Wed, 30 Sep 2015 15:12:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@accumulo.apache.org Date: Wed, 30 Sep 2015 15:12:52 -0000 Message-Id: In-Reply-To: <30568eb2c80b4045aca2abe48d56cc49@git.apache.org> References: <30568eb2c80b4045aca2abe48d56cc49@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/4] accumulo git commit: Merge branch '1.7' Merge branch '1.7' Conflicts: test/src/main/java/org/apache/accumulo/test/ScanIteratorIT.java test/src/main/java/org/apache/accumulo/test/functional/ScanIteratorIT.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1a2232a1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1a2232a1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1a2232a1 Branch: refs/heads/master Commit: 1a2232a1504e23cca8c74c9523d94d5e3eda8217 Parents: 0cbb1ea f592180 Author: Josh Elser Authored: Wed Sep 30 09:56:40 2015 -0400 Committer: Josh Elser Committed: Wed Sep 30 09:56:40 2015 -0400 ---------------------------------------------------------------------- .../apache/accumulo/test/ScanIteratorIT.java | 170 ------------------- .../test/functional/ScanIteratorIT.java | 121 ++++++++++++- 2 files changed, 119 insertions(+), 172 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/1a2232a1/test/src/main/java/org/apache/accumulo/test/functional/ScanIteratorIT.java ---------------------------------------------------------------------- diff --cc test/src/main/java/org/apache/accumulo/test/functional/ScanIteratorIT.java index 3453303,0000000..eb6b5e3 mode 100644,000000..100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/ScanIteratorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ScanIteratorIT.java @@@ -1,134 -1,0 +1,251 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.functional; + +import static java.nio.charset.StandardCharsets.UTF_8; ++import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; ++import java.util.Map; +import java.util.Map.Entry; + ++import org.apache.accumulo.cluster.ClusterUser; ++import org.apache.accumulo.core.client.AccumuloException; ++import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; ++import org.apache.accumulo.core.client.ClientConfiguration; ++import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.IteratorSetting; ++import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.ScannerBase; ++import org.apache.accumulo.core.client.TableNotFoundException; ++import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.harness.AccumuloClusterHarness; ++import org.apache.accumulo.core.security.TablePermission; ++import org.apache.accumulo.test.functional.AuthsIterator; +import org.apache.hadoop.io.Text; ++import org.apache.hadoop.security.UserGroupInformation; ++import org.junit.After; ++import org.junit.Before; +import org.junit.Test; ++import org.slf4j.Logger; ++import org.slf4j.LoggerFactory; + +public class ScanIteratorIT extends AccumuloClusterHarness { ++ private static final Logger log = LoggerFactory.getLogger(ScanIteratorIT.class); + + @Override + protected int defaultTimeoutSeconds() { - return 30; ++ return 60; ++ } ++ ++ private Connector connector; ++ private String tableName; ++ private String user; ++ private boolean saslEnabled; ++ ++ @Before ++ public void setup() throws Exception { ++ connector = getConnector(); ++ tableName = getUniqueNames(1)[0]; ++ ++ connector.tableOperations().create(tableName); ++ ClientConfiguration clientConfig = cluster.getClientConfig(); ++ ClusterUser clusterUser = getUser(0); ++ user = clusterUser.getPrincipal(); ++ PasswordToken userToken; ++ if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) { ++ userToken = null; ++ saslEnabled = true; ++ } else { ++ userToken = new PasswordToken(clusterUser.getPassword()); ++ saslEnabled = false; ++ } ++ if (connector.securityOperations().listLocalUsers().contains(user)) { ++ log.info("Dropping {}", user); ++ connector.securityOperations().dropLocalUser(user); ++ } ++ connector.securityOperations().createLocalUser(user, userToken); ++ connector.securityOperations().grantTablePermission(user, tableName, TablePermission.READ); ++ connector.securityOperations().grantTablePermission(user, tableName, TablePermission.WRITE); ++ connector.securityOperations().changeUserAuthorizations(user, AuthsIterator.AUTHS); ++ } ++ ++ @After ++ public void tearDown() throws Exception { ++ if (null != user) { ++ if (saslEnabled) { ++ ClusterUser rootUser = getAdminUser(); ++ UserGroupInformation.loginUserFromKeytab(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath()); ++ } ++ connector.securityOperations().dropLocalUser(user); ++ } + } + + @Test + public void run() throws Exception { + String tableName = getUniqueNames(1)[0]; + Connector c = getConnector(); - c.tableOperations().create(tableName); + + BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig()); + + for (int i = 0; i < 1000; i++) { + Mutation m = new Mutation(new Text(String.format("%06d", i))); + m.put(new Text("cf1"), new Text("cq1"), new Value(Integer.toString(1000 - i).getBytes(UTF_8))); + m.put(new Text("cf1"), new Text("cq2"), new Value(Integer.toString(i - 1000).getBytes(UTF_8))); + + bw.addMutation(m); + } + + bw.close(); + + Scanner scanner = c.createScanner(tableName, new Authorizations()); + + setupIter(scanner); + verify(scanner, 1, 999); + + BatchScanner bscanner = c.createBatchScanner(tableName, new Authorizations(), 3); + bscanner.setRanges(Collections.singleton(new Range((Key) null, null))); + + setupIter(bscanner); + verify(bscanner, 1, 999); + + ArrayList ranges = new ArrayList(); + ranges.add(new Range(new Text(String.format("%06d", 1)))); + ranges.add(new Range(new Text(String.format("%06d", 6)), new Text(String.format("%06d", 16)))); + ranges.add(new Range(new Text(String.format("%06d", 20)))); + ranges.add(new Range(new Text(String.format("%06d", 23)))); + ranges.add(new Range(new Text(String.format("%06d", 56)), new Text(String.format("%06d", 61)))); + ranges.add(new Range(new Text(String.format("%06d", 501)), new Text(String.format("%06d", 504)))); + ranges.add(new Range(new Text(String.format("%06d", 998)), new Text(String.format("%06d", 1000)))); + + HashSet got = new HashSet(); + HashSet expected = new HashSet(); + for (int i : new int[] {1, 7, 9, 11, 13, 15, 23, 57, 59, 61, 501, 503, 999}) { + expected.add(i); + } + + bscanner.setRanges(ranges); + + for (Entry entry : bscanner) { + got.add(Integer.parseInt(entry.getKey().getRow().toString())); + } + + System.out.println("got : " + got); + + if (!got.equals(expected)) { + throw new Exception(got + " != " + expected); + } + + bscanner.close(); + + } + + private void verify(Iterable> scanner, int start, int finish) throws Exception { + + int expected = start; + for (Entry entry : scanner) { + if (Integer.parseInt(entry.getKey().getRow().toString()) != expected) { + throw new Exception("Saw unexpexted " + entry.getKey().getRow() + " " + expected); + } + + if (entry.getKey().getColumnQualifier().toString().equals("cq2")) { + expected += 2; + } + } + + if (expected != finish + 2) { + throw new Exception("Ended at " + expected + " not " + (finish + 2)); + } + } + + private void setupIter(ScannerBase scanner) throws Exception { + IteratorSetting dropMod = new IteratorSetting(50, "dropMod", "org.apache.accumulo.test.functional.DropModIter"); + dropMod.addOption("mod", "2"); + dropMod.addOption("drop", "0"); + scanner.addScanIterator(dropMod); + } + ++ @Test ++ public void testAuthsPresentInIteratorEnvironment() throws Exception { ++ runTest(AuthsIterator.AUTHS, false); ++ } ++ ++ @Test ++ public void testAuthsNotPresentInIteratorEnvironment() throws Exception { ++ runTest(new Authorizations("B"), true); ++ } ++ ++ @Test ++ public void testEmptyAuthsInIteratorEnvironment() throws Exception { ++ runTest(Authorizations.EMPTY, true); ++ } ++ ++ private void runTest(ScannerBase scanner, Authorizations auths, boolean shouldFail) throws AccumuloSecurityException, AccumuloException, ++ TableNotFoundException { ++ int count = 0; ++ for (Map.Entry entry : scanner) { ++ assertEquals(shouldFail ? AuthsIterator.FAIL : AuthsIterator.SUCCESS, entry.getKey().getRow().toString()); ++ count++; ++ } ++ ++ assertEquals(1, count); ++ } ++ ++ private void runTest(Authorizations auths, boolean shouldFail) throws Exception { ++ ClusterUser clusterUser = getUser(0); ++ Connector userC = getCluster().getConnector(clusterUser.getPrincipal(), clusterUser.getToken()); ++ writeTestMutation(userC); ++ ++ IteratorSetting setting = new IteratorSetting(10, AuthsIterator.class); ++ ++ Scanner scanner = userC.createScanner(tableName, auths); ++ scanner.addScanIterator(setting); ++ ++ BatchScanner batchScanner = userC.createBatchScanner(tableName, auths, 1); ++ batchScanner.setRanges(Collections.singleton(new Range("1"))); ++ batchScanner.addScanIterator(setting); ++ ++ runTest(scanner, auths, shouldFail); ++ runTest(batchScanner, auths, shouldFail); ++ ++ scanner.close(); ++ batchScanner.close(); ++ } ++ ++ private void writeTestMutation(Connector userC) throws TableNotFoundException, MutationsRejectedException { ++ BatchWriter batchWriter = userC.createBatchWriter(tableName, new BatchWriterConfig()); ++ Mutation m = new Mutation("1"); ++ m.put(new Text("2"), new Text("3"), new Value("".getBytes())); ++ batchWriter.addMutation(m); ++ batchWriter.flush(); ++ batchWriter.close(); ++ ++ } ++ +}