Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D785D200C8B for ; Mon, 22 May 2017 17:16:54 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id D6345160BDA; Mon, 22 May 2017 15:16:54 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id E3899160BD8 for ; Mon, 22 May 2017 17:16:53 +0200 (CEST) Received: (qmail 70281 invoked by uid 500); 22 May 2017 15:16:53 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 70173 invoked by uid 99); 22 May 2017 15:16:53 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 22 May 2017 15:16:53 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DCCA3E9113; Mon, 22 May 2017 15:16:52 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dlmarion@apache.org To: commits@accumulo.apache.org Date: Mon, 22 May 2017 15:16:58 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [07/10] accumulo git commit: Merge branch 'ACCUMULO-4636' into 1.8 archived-at: Mon, 22 May 2017 15:16:55 -0000 Merge branch 'ACCUMULO-4636' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/116da380 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/116da380 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/116da380 Branch: refs/heads/ACCUMULO-4463 Commit: 116da380a36e5088c5bf9be70f9b88850089ac23 Parents: 54c82dc 2ff2678 Author: Mike Miller Authored: Thu May 18 14:29:13 2017 -0400 Committer: Mike Miller Committed: Thu May 18 14:29:13 2017 -0400 ---------------------------------------------------------------------- .../core/client/mock/MockScannerBase.java | 4 +- .../apache/accumulo/core/iterators/Filter.java | 5 +- .../core/iterators/FirstEntryInRowIterator.java | 11 ++-- .../accumulo/core/iterators/IteratorUtil.java | 2 +- .../system/ColumnFamilySkippingIterator.java | 17 ++--- .../iterators/system/ColumnQualifierFilter.java | 68 +++++++++++--------- .../core/iterators/system/DeletingIterator.java | 28 ++++---- .../core/iterators/system/VisibilityFilter.java | 51 ++++++++++----- .../core/iterators/user/VersioningIterator.java | 5 +- .../core/iterators/user/VisibilityFilter.java | 44 +++++++++---- .../core/iterators/system/ColumnFilterTest.java | 58 ++++++++++++----- .../iterators/system/VisibilityFilterTest.java | 20 +++++- .../core/iterators/user/FilterTest.java | 12 ++-- .../user/TransformingIteratorTest.java | 2 +- .../problems/ProblemReportingIterator.java | 4 +- .../apache/accumulo/tserver/InMemoryMap.java | 7 -- .../performance/scan/CollectTabletStats.java | 4 +- 17 files changed, 218 insertions(+), 124 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/core/src/main/java/org/apache/accumulo/core/client/mock/MockScannerBase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java index 981404c,aee9771..a830da3 --- a/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java @@@ -391,12 -372,4 +391,12 @@@ public class IteratorUtil } return toIteratorSettings(ic); } + + public static SortedKeyValueIterator setupSystemScanIterators(SortedKeyValueIterator source, Set cols, Authorizations auths, + byte[] defaultVisibility) throws IOException { + DeletingIterator delIter = new DeletingIterator(source, false); + ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter); + ColumnQualifierFilter colFilter = new ColumnQualifierFilter(cfsi, cols); - return new VisibilityFilter(colFilter, auths, defaultVisibility); ++ return VisibilityFilter.wrap(colFilter, auths, defaultVisibility); + } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java index d02b7f2,ca10c65..7ff08ec --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java @@@ -56,53 -60,41 +56,53 @@@ import org.junit.Assert import org.junit.Before; import org.junit.Test; +import com.google.common.collect.ImmutableMap; + public class TransformingIteratorTest { - private static final String TABLE_NAME = "test_table"; + private static Authorizations authorizations = new Authorizations("vis0", "vis1", "vis2", "vis3", "vis4"); - private Connector connector; - private Scanner scanner; + private static final Map EMPTY_OPTS = ImmutableMap.of(); + private TransformingIterator titer; - @Before - public void setUpMockAccumulo() throws Exception { - MockInstance instance = new MockInstance("test"); - connector = instance.getConnector("user", new PasswordToken("password")); - connector.securityOperations().changeUserAuthorizations("user", authorizations); + private TreeMap data = new TreeMap<>(); - if (connector.tableOperations().exists(TABLE_NAME)) - connector.tableOperations().delete(TABLE_NAME); - connector.tableOperations().create(TABLE_NAME); - BatchWriterConfig bwCfg = new BatchWriterConfig(); - bwCfg.setMaxWriteThreads(1); + @Before + public void createData() throws Exception { + data.clear(); + generateRow(data, "row1"); + generateRow(data, "row2"); + generateRow(data, "row3"); + } - BatchWriter bw = connector.createBatchWriter(TABLE_NAME, bwCfg); - bw.addMutation(createDefaultMutation("row1")); - bw.addMutation(createDefaultMutation("row2")); - bw.addMutation(createDefaultMutation("row3")); + private void setUpTransformIterator(Class clazz) throws IOException { + setUpTransformIterator(clazz, true); + } - bw.flush(); - bw.close(); + private void setUpTransformIterator(Class clazz, boolean setupAuths) throws IOException { + SortedMapIterator source = new SortedMapIterator(data); + ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(source); - VisibilityFilter visFilter = new VisibilityFilter(cfsi, authorizations, new byte[0]); ++ SortedKeyValueIterator visFilter = VisibilityFilter.wrap(cfsi, authorizations, new byte[0]); + ReuseIterator reuserIter = new ReuseIterator(); + reuserIter.init(visFilter, EMPTY_OPTS, null); + try { + titer = clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new RuntimeException(e); + } - scanner = connector.createScanner(TABLE_NAME, authorizations); - scanner.addScanIterator(new IteratorSetting(20, ReuseIterator.class)); - } + IteratorEnvironment iterEnv = EasyMock.createMock(IteratorEnvironment.class); + EasyMock.expect(iterEnv.getIteratorScope()).andReturn(IteratorScope.scan).anyTimes(); + EasyMock.replay(iterEnv); - private void setUpTransformIterator(Class clazz) { - IteratorSetting cfg = new IteratorSetting(21, clazz); - cfg.setName("keyTransformIter"); - TransformingIterator.setAuthorizations(cfg, new Authorizations("vis0", "vis1", "vis2", "vis3")); - scanner.addScanIterator(cfg); + Map opts; + if (setupAuths) { + IteratorSetting cfg = new IteratorSetting(21, clazz); + TransformingIterator.setAuthorizations(cfg, new Authorizations("vis0", "vis1", "vis2", "vis3")); + opts = cfg.getOptions(); + } else { + opts = ImmutableMap.of(); + } + titer.init(reuserIter, opts, iterEnv); } @Test http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReportingIterator.java ---------------------------------------------------------------------- diff --cc server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReportingIterator.java index 83b4615,e419780..7e1676a --- a/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReportingIterator.java +++ b/server/base/src/main/java/org/apache/accumulo/server/problems/ProblemReportingIterator.java @@@ -31,17 -31,17 +31,17 @@@ import org.apache.accumulo.core.iterato import org.apache.accumulo.server.AccumuloServerContext; public class ProblemReportingIterator implements InterruptibleIterator { - private SortedKeyValueIterator source; + private final SortedKeyValueIterator source; private boolean sawError = false; - private boolean continueOnError; + private final boolean continueOnError; private String resource; - private String table; + private String tableId; private final AccumuloServerContext context; - public ProblemReportingIterator(AccumuloServerContext context, String table, String resource, boolean continueOnError, + public ProblemReportingIterator(AccumuloServerContext context, String tableId, String resource, boolean continueOnError, SortedKeyValueIterator source) { this.context = context; - this.table = table; + this.tableId = tableId; this.resource = resource; this.continueOnError = continueOnError; this.source = source; http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/116da380/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java ----------------------------------------------------------------------