Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4824E18DB9 for ; Wed, 3 Feb 2016 03:09:00 +0000 (UTC) Received: (qmail 6067 invoked by uid 500); 3 Feb 2016 03:09:00 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 5904 invoked by uid 500); 3 Feb 2016 03:09:00 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 5530 invoked by uid 99); 3 Feb 2016 03:08:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 03 Feb 2016 03:08:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A6A95DFE60; Wed, 3 Feb 2016 03:08:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ctubbsii@apache.org To: commits@accumulo.apache.org Date: Wed, 03 Feb 2016 03:09:14 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [16/21] accumulo git commit: Merge branch '1.6' into 1.7 Merge branch '1.6' into 1.7 Conflicts: core/src/main/java/org/apache/accumulo/core/Constants.java core/src/test/java/org/apache/accumulo/core/util/ByteBufferUtilTest.java test/src/test/java/org/apache/accumulo/harness/MiniClusterHarness.java test/src/test/java/org/apache/accumulo/test/functional/DeleteIT.java test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4af87bfc Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4af87bfc Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4af87bfc Branch: refs/heads/1.7 Commit: 4af87bfc718f958f19ddc09354f6b55f87809e91 Parents: 668c340 46958bf Author: Christopher Tubbs Authored: Tue Feb 2 21:35:21 2016 -0500 Committer: Christopher Tubbs Committed: Tue Feb 2 21:35:21 2016 -0500 ---------------------------------------------------------------------- .../core/file/rfile/VisMetricsGatherer.java | 5 +++-- .../accumulo/core/util/ByteBufferUtilTest.java | 18 +++++++++--------- .../core/util/UnsynchronizedBufferTest.java | 10 +++++----- .../apache/accumulo/proxy/ProxyServerTest.java | 5 ++--- .../server/util/FileSystemMonitorTest.java | 3 +-- .../vfs/providers/HdfsFileContentInfoFactory.java | 4 +++- .../classloader/vfs/providers/HdfsFileSystem.java | 4 +++- .../vfs/providers/HdfsRandomAccessContent.java | 5 +++-- .../accumulo/harness/MiniClusterHarness.java | 4 ++-- .../accumulo/test/AccumuloOutputFormatIT.java | 2 +- .../apache/accumulo/test/ConditionalWriterIT.java | 6 +++--- .../apache/accumulo/test/TransportCachingIT.java | 2 +- .../test/functional/BulkSplitOptimizationIT.java | 2 +- .../apache/accumulo/test/functional/DeleteIT.java | 5 ++--- .../accumulo/test/functional/ExamplesIT.java | 2 +- .../accumulo/test/functional/ReadWriteIT.java | 6 +++--- .../accumulo/test/functional/RestartIT.java | 11 +++++------ .../accumulo/test/functional/RestartStressIT.java | 7 +++---- .../apache/accumulo/test/functional/ScanIdIT.java | 2 +- .../apache/accumulo/test/functional/SplitIT.java | 7 +++---- 20 files changed, 55 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/core/src/main/java/org/apache/accumulo/core/file/rfile/VisMetricsGatherer.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/VisMetricsGatherer.java index 6050e41,0000000..8c8c949 mode 100644,000000..100644 --- a/core/src/main/java/org/apache/accumulo/core/file/rfile/VisMetricsGatherer.java +++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/VisMetricsGatherer.java @@@ -1,172 -1,0 +1,173 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.file.rfile; + ++import static java.nio.charset.StandardCharsets.UTF_8; ++ +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.accumulo.core.data.ArrayByteSequence; +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.hadoop.io.Text; + - import com.google.common.base.Charsets; +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import com.google.common.util.concurrent.AtomicLongMap; + +/** + * This class provides visibility metrics per locality group. The Map in getMetrics() maps the locality group name to an ArrayList of VisibilityMetric objects. + * These contain the components of a visibility metric; the visibility as a String, the number of times that is seen in a locality group, the percentage of keys + * that contain that visibility in the locality group, the number of blocks in the locality group that contain the visibility, and the percentage of blocks in + * the locality group that contain the visibility. + */ +public class VisMetricsGatherer implements MetricsGatherer>> { + + protected Map> metric; + protected Map> blocks; + protected ArrayList numEntries; + protected ArrayList numBlocks; + private ArrayList inBlock; + protected ArrayList localityGroups; + private int numLG; + private Map> localityGroupCF; + + public VisMetricsGatherer() { + metric = new HashMap<>(); + blocks = new HashMap<>(); + numEntries = new ArrayList<>(); + numBlocks = new ArrayList<>(); + inBlock = new ArrayList<>(); + localityGroups = new ArrayList<>(); + numLG = 0; + } + + @Override + public void init(Map> cf) { + localityGroupCF = cf; + } + + @Override + public void startLocalityGroup(Text oneCF) { + String name = null; + ByteSequence cf = new ArrayByteSequence(oneCF.toString()); + for (Entry> entry : localityGroupCF.entrySet()) { + if (entry.getValue().contains(cf)) { + if (entry.getKey() == null) + name = null; + else + name = entry.getKey().toString(); + break; + } + } + localityGroups.add(name); + metric.put(name, AtomicLongMap.create(new HashMap())); + blocks.put(name, AtomicLongMap.create(new HashMap())); + numLG++; + numEntries.add((long) 0); + numBlocks.add(0); + } + + @Override + public void addMetric(Key key, Value val) { + String myMetric = key.getColumnVisibility().toString(); + String currLG = localityGroups.get(numLG - 1); + if (metric.get(currLG).containsKey(myMetric)) { + metric.get(currLG).getAndIncrement(myMetric); + } else + metric.get(currLG).put(myMetric, 1); + + numEntries.set(numLG - 1, numEntries.get(numLG - 1) + 1); + + if (!inBlock.contains(myMetric) && blocks.get(currLG).containsKey(myMetric)) { + blocks.get(currLG).incrementAndGet(myMetric); + inBlock.add(myMetric); + } else if (!inBlock.contains(myMetric) && !blocks.get(currLG).containsKey(myMetric)) { + blocks.get(currLG).put(myMetric, 1); + inBlock.add(myMetric); + } + + } + + @Override + public void startBlock() { + inBlock.clear(); + numBlocks.set(numLG - 1, numBlocks.get(numLG - 1) + 1); + } + + @Override + public void printMetrics(boolean hash, String metricWord, PrintStream out) { + for (int i = 0; i < numLG; i++) { + String lGName = localityGroups.get(i); + out.print("Locality Group: "); + if (lGName == null) + out.println(""); + else + out.println(localityGroups.get(i)); + out.printf("%-27s", metricWord); + out.println("Number of keys" + "\t " + "Percent of keys" + "\t" + "Number of blocks" + "\t" + "Percent of blocks"); + for (Entry entry : metric.get(lGName).asMap().entrySet()) { + HashFunction hf = Hashing.md5(); - HashCode hc = hf.newHasher().putString(entry.getKey(), Charsets.UTF_8).hash(); ++ HashCode hc = hf.newHasher().putString(entry.getKey(), UTF_8).hash(); + if (hash) + out.printf("%-20s", hc.toString().substring(0, 8)); + else + out.printf("%-20s", entry.getKey()); + out.print("\t\t" + entry.getValue() + "\t\t\t"); + out.printf("%.2f", ((double) entry.getValue() / numEntries.get(i)) * 100); + out.print("%\t\t\t"); + + long blocksIn = blocks.get(lGName).get(entry.getKey()); + + out.print(blocksIn + "\t\t "); + out.printf("%.2f", ((double) blocksIn / numBlocks.get(i)) * 100); + out.print("%"); + + out.println(""); + } + out.println("Number of keys: " + numEntries.get(i)); + out.println(); + } + } + + @Override + public Map> getMetrics() { + Map> getMetrics = new HashMap<>(); + for (int i = 0; i < numLG; i++) { + String lGName = localityGroups.get(i); + ArrayList rows = new ArrayList<>(); + for (Entry entry : metric.get(lGName).asMap().entrySet()) { + long vis = entry.getValue(); + double visPer = ((double) entry.getValue() / numEntries.get(i)) * 100; + + long blocksIn = blocks.get(lGName).get(entry.getKey()); + double blocksPer = ((double) blocksIn / numBlocks.get(i)) * 100; + + rows.add(new VisibilityMetric(entry.getKey(), vis, visPer, blocksIn, blocksPer)); + } + getMetrics.put(lGName, rows); + } + return getMetrics; + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/core/src/test/java/org/apache/accumulo/core/util/ByteBufferUtilTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/util/ByteBufferUtilTest.java index f27a1ba,a21f598..5a8c0dc --- a/core/src/test/java/org/apache/accumulo/core/util/ByteBufferUtilTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/ByteBufferUtilTest.java @@@ -17,9 -17,8 +17,11 @@@ package org.apache.accumulo.core.util; -import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@@ -40,20 -37,9 +40,20 @@@ public class ByteBufferUtilTest List bal = ByteBufferUtil.toBytesList(Collections.singletonList(bb)); Assert.assertEquals(1, bal.size()); - Assert.assertEquals(expected, new String(bal.get(0), Charsets.UTF_8)); + Assert.assertEquals(expected, new String(bal.get(0), UTF_8)); Assert.assertEquals(new ArrayByteSequence(expected), new ArrayByteSequence(bb)); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + try { + ByteBufferUtil.write(dos, bb); + dos.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + - Assert.assertEquals(expected, new String(baos.toByteArray(), Charsets.UTF_8)); ++ Assert.assertEquals(expected, new String(baos.toByteArray(), UTF_8)); } @Test http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/core/src/test/java/org/apache/accumulo/core/util/UnsynchronizedBufferTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/util/UnsynchronizedBufferTest.java index 6416219,64044f9..94feb43 --- a/core/src/test/java/org/apache/accumulo/core/util/UnsynchronizedBufferTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/UnsynchronizedBufferTest.java @@@ -16,6 -16,8 +16,8 @@@ */ package org.apache.accumulo.core.util; -import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; + import java.nio.ByteBuffer; import org.junit.Assert; http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/proxy/src/test/java/org/apache/accumulo/proxy/ProxyServerTest.java ---------------------------------------------------------------------- diff --cc proxy/src/test/java/org/apache/accumulo/proxy/ProxyServerTest.java index 24201e7,956044c..07fdc45 --- a/proxy/src/test/java/org/apache/accumulo/proxy/ProxyServerTest.java +++ b/proxy/src/test/java/org/apache/accumulo/proxy/ProxyServerTest.java @@@ -16,8 -16,9 +16,9 @@@ */ package org.apache.accumulo.proxy; -import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; + import java.nio.ByteBuffer; - import java.nio.charset.Charset; import java.util.HashMap; import java.util.List; import java.util.Map; http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/server/base/src/test/java/org/apache/accumulo/server/util/FileSystemMonitorTest.java ---------------------------------------------------------------------- diff --cc server/base/src/test/java/org/apache/accumulo/server/util/FileSystemMonitorTest.java index ec389fd,5b5b7ac..7387035 --- a/server/base/src/test/java/org/apache/accumulo/server/util/FileSystemMonitorTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/FileSystemMonitorTest.java @@@ -16,6 -16,7 +16,7 @@@ */ package org.apache.accumulo.server.util; -import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileContentInfoFactory.java ---------------------------------------------------------------------- diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileContentInfoFactory.java index 2621255,b1a4abe..ba96b97 --- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileContentInfoFactory.java +++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileContentInfoFactory.java @@@ -16,6 -16,6 +16,8 @@@ */ package org.apache.accumulo.start.classloader.vfs.providers; ++import static java.nio.charset.StandardCharsets.UTF_8; ++ import org.apache.commons.vfs2.FileContent; import org.apache.commons.vfs2.FileContentInfo; import org.apache.commons.vfs2.FileContentInfoFactory; @@@ -28,8 -28,9 +30,8 @@@ import org.apache.commons.vfs2.impl.Def * @since 2.1 */ public class HdfsFileContentInfoFactory implements FileContentInfoFactory { - private static final java.nio.charset.Charset UTF_8 = java.nio.charset.Charset.forName("UTF-8"); private static final String CONTENT = "text/plain"; - private static final String ENCODING = "UTF-8"; + private static final String ENCODING = UTF_8.name(); /** * Creates a FileContentInfo for a the given FileContent. http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java ---------------------------------------------------------------------- diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java index 26aa5c1,8d45555..a544b4a --- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java +++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java @@@ -16,6 -16,6 +16,8 @@@ */ package org.apache.accumulo.start.classloader.vfs.providers; ++import static java.nio.charset.StandardCharsets.UTF_8; ++ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsRandomAccessContent.java ---------------------------------------------------------------------- diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsRandomAccessContent.java index b9b122c,30140ba..b3b58aa --- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsRandomAccessContent.java +++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsRandomAccessContent.java @@@ -16,6 -16,6 +16,8 @@@ */ package org.apache.accumulo.start.classloader.vfs.providers; ++import static java.nio.charset.StandardCharsets.UTF_8; ++ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@@ -113,9 -149,12 +114,9 @@@ public class HdfsRandomAccessContent im return this.fis.readInt(); } - /** - * @see java.io.DataInput#readLine() - */ @Override public String readLine() throws IOException { - BufferedReader d = new BufferedReader(new InputStreamReader(this.fis, Charset.forName("UTF-8"))); + BufferedReader d = new BufferedReader(new InputStreamReader(this.fis, UTF_8)); return d.readLine(); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/test/src/test/java/org/apache/accumulo/harness/MiniClusterHarness.java ---------------------------------------------------------------------- diff --cc test/src/test/java/org/apache/accumulo/harness/MiniClusterHarness.java index 5d32cb1,5fc8b94..8fa4760 --- a/test/src/test/java/org/apache/accumulo/harness/MiniClusterHarness.java +++ b/test/src/test/java/org/apache/accumulo/harness/MiniClusterHarness.java @@@ -16,7 -16,7 +16,8 @@@ */ package org.apache.accumulo.harness; -import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertTrue; import java.io.BufferedOutputStream; import java.io.File; @@@ -39,11 -33,7 +40,10 @@@ import org.apache.accumulo.server.secur import org.apache.accumulo.test.functional.NativeMapIT; import org.apache.accumulo.test.util.CertUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; - import com.google.common.base.Charsets; import com.google.common.base.Preconditions; /** @@@ -97,29 -69,13 +97,29 @@@ public class MiniClusterHarness return create(testClassName, testMethodName, token, MiniClusterConfigurationCallback.NO_CALLBACK); } + public MiniAccumuloClusterImpl create(String testClassName, String testMethodName, AuthenticationToken token, TestingKdc kdc) throws Exception { + return create(testClassName, testMethodName, token, MiniClusterConfigurationCallback.NO_CALLBACK, kdc); + } + public MiniAccumuloClusterImpl create(String testClassName, String testMethodName, AuthenticationToken token, MiniClusterConfigurationCallback configCallback) throws Exception { + return create(testClassName, testMethodName, token, configCallback, null); + } + + public MiniAccumuloClusterImpl create(String testClassName, String testMethodName, AuthenticationToken token, + MiniClusterConfigurationCallback configCallback, TestingKdc kdc) throws Exception { Preconditions.checkNotNull(token); - Preconditions.checkArgument(PasswordToken.class.isAssignableFrom(token.getClass())); + Preconditions.checkArgument(token instanceof PasswordToken || token instanceof KerberosToken, "A PasswordToken or KerberosToken is required"); - String passwd = new String(((PasswordToken) token).getPassword(), UTF_8); - MiniAccumuloConfigImpl cfg = new MiniAccumuloConfigImpl(AccumuloClusterIT.createTestDir(testClassName + "_" + testMethodName), passwd); + String rootPasswd; + if (token instanceof PasswordToken) { - rootPasswd = new String(((PasswordToken) token).getPassword(), Charsets.UTF_8); ++ rootPasswd = new String(((PasswordToken) token).getPassword(), UTF_8); + } else { + rootPasswd = UUID.randomUUID().toString(); + } + + File baseDir = AccumuloClusterIT.createTestDir(testClassName + "_" + testMethodName); + MiniAccumuloConfigImpl cfg = new MiniAccumuloConfigImpl(baseDir, rootPasswd); // Enable native maps by default cfg.setNativeLibPaths(NativeMapIT.nativeMapLocation().getAbsolutePath()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/test/src/test/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java ---------------------------------------------------------------------- diff --cc test/src/test/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java index a2f522e,9d00584..7d67230 --- a/test/src/test/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java +++ b/test/src/test/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java @@@ -16,7 -16,7 +16,7 @@@ */ package org.apache.accumulo.test; --import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; import java.io.File; import java.io.IOException; http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/test/src/test/java/org/apache/accumulo/test/ConditionalWriterIT.java ---------------------------------------------------------------------- diff --cc test/src/test/java/org/apache/accumulo/test/ConditionalWriterIT.java index 15e9d5f,1579b34..94a3d35 --- a/test/src/test/java/org/apache/accumulo/test/ConditionalWriterIT.java +++ b/test/src/test/java/org/apache/accumulo/test/ConditionalWriterIT.java @@@ -17,6 -17,7 +17,7 @@@ package org.apache.accumulo.test; -import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertTrue; import java.io.IOException; @@@ -91,18 -88,15 +92,17 @@@ import org.apache.accumulo.harness.Accu import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl; import org.apache.accumulo.test.functional.BadIterator; import org.apache.accumulo.test.functional.SlowIterator; -import org.apache.accumulo.trace.instrument.Span; -import org.apache.accumulo.trace.instrument.Trace; +import org.apache.accumulo.tracer.TraceDump; +import org.apache.accumulo.tracer.TraceDump.Printer; import org.apache.accumulo.tracer.TraceServer; import org.apache.hadoop.io.Text; -import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Assume; +import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; - import com.google.common.base.Charsets; import com.google.common.collect.Iterables; /** http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/test/src/test/java/org/apache/accumulo/test/TransportCachingIT.java ---------------------------------------------------------------------- diff --cc test/src/test/java/org/apache/accumulo/test/TransportCachingIT.java index 5bfb207,ddbd3e8..8bfd5fc --- a/test/src/test/java/org/apache/accumulo/test/TransportCachingIT.java +++ b/test/src/test/java/org/apache/accumulo/test/TransportCachingIT.java @@@ -16,7 -16,7 +16,7 @@@ */ package org.apache.accumulo.test; --import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/test/src/test/java/org/apache/accumulo/test/functional/BulkSplitOptimizationIT.java ---------------------------------------------------------------------- diff --cc test/src/test/java/org/apache/accumulo/test/functional/BulkSplitOptimizationIT.java index fe26870,442507f..7606083 --- a/test/src/test/java/org/apache/accumulo/test/functional/BulkSplitOptimizationIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkSplitOptimizationIT.java @@@ -16,14 -16,14 +16,14 @@@ */ package org.apache.accumulo.test.functional; --import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; -import java.util.Collections; - -import org.apache.accumulo.cluster.ClusterServerType; import org.apache.accumulo.core.cli.ClientOpts.Password; import org.apache.accumulo.core.cli.ScannerOpts; +import org.apache.accumulo.core.client.ClientConfiguration; import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.KerberosToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.util.UtilWaitThread; http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/test/src/test/java/org/apache/accumulo/test/functional/DeleteIT.java ---------------------------------------------------------------------- diff --cc test/src/test/java/org/apache/accumulo/test/functional/DeleteIT.java index 7a8380b,d3d8b79..3200d96 --- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteIT.java @@@ -16,9 -16,8 +16,10 @@@ */ package org.apache.accumulo.test.functional; -import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import org.apache.accumulo.cluster.AccumuloCluster; import org.apache.accumulo.core.cli.BatchWriterOpts; @@@ -49,12 -42,12 +48,12 @@@ public class DeleteIT extends AccumuloC Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); - PasswordToken token = (PasswordToken) getToken(); - deleteTest(c, getCluster(), new String(token.getPassword(), UTF_8), tableName); - try { - getCluster().getClusterControl().adminStopAll(); - } finally { - getCluster().start(); + AuthenticationToken token = getAdminToken(); + if (token instanceof KerberosToken) { + deleteTest(c, getCluster(), getAdminPrincipal(), null, tableName, getAdminUser().getKeytab().getAbsolutePath()); + } else if (token instanceof PasswordToken) { + PasswordToken passwdToken = (PasswordToken) token; - deleteTest(c, getCluster(), getAdminPrincipal(), new String(passwdToken.getPassword(), Charsets.UTF_8), tableName, null); ++ deleteTest(c, getCluster(), getAdminPrincipal(), new String(passwdToken.getPassword(), UTF_8), tableName, null); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java ---------------------------------------------------------------------- diff --cc test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java index 257c7c1,3204528..edc6aed --- a/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java @@@ -16,7 -16,7 +16,7 @@@ */ package org.apache.accumulo.test.functional; --import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java ---------------------------------------------------------------------- diff --cc test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java index 65ef479,f0a37e4..a49d43c --- a/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java @@@ -16,9 -16,9 +16,10 @@@ */ package org.apache.accumulo.test.functional; -import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; @@@ -95,9 -73,6 +96,8 @@@ import org.junit.Test import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import com.google.common.base.Charsets; +import com.google.common.collect.Iterators; + public class ReadWriteIT extends AccumuloClusterIT { private static final Logger log = LoggerFactory.getLogger(ReadWriteIT.class); @@@ -246,25 -184,10 +246,25 @@@ @Override public Integer call() { try { + ClientConfiguration clientConf = cluster.getClientConfig(); + // Invocation is different for SASL. We're only logged in via this processes memory (not via some credentials cache on disk) + // Need to pass along the keytab because of that. + if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) { + String principal = getAdminPrincipal(); + AuthenticationToken token = getAdminToken(); + assertTrue("Expected KerberosToken, but was " + token.getClass(), token instanceof KerberosToken); + KerberosToken kt = (KerberosToken) token; + assertNotNull("Expected keytab in token", kt.getKeytab()); + return control.exec( + TestMultiTableIngest.class, + args("--count", Integer.toString(ROWS), "-i", instance, "-z", keepers, "--tablePrefix", prefix, "--keytab", kt.getKeytab().getAbsolutePath(), + "-u", principal)); + } + return control.exec( TestMultiTableIngest.class, - args("--count", "" + ROWS, "-u", "root", "-i", instance, "-z", keepers, "-p", new String(((PasswordToken) getToken()).getPassword(), UTF_8), - "--tablePrefix", prefix)); + args("--count", Integer.toString(ROWS), "-u", getAdminPrincipal(), "-i", instance, "-z", keepers, "-p", new String( - ((PasswordToken) getAdminToken()).getPassword(), Charsets.UTF_8), "--tablePrefix", prefix)); ++ ((PasswordToken) getAdminToken()).getPassword(), UTF_8), "--tablePrefix", prefix)); } catch (IOException e) { log.error("Error running MultiTableIngest", e); return -1; @@@ -275,25 -198,10 +275,25 @@@ @Override public Integer call() { try { + ClientConfiguration clientConf = cluster.getClientConfig(); + // Invocation is different for SASL. We're only logged in via this processes memory (not via some credentials cache on disk) + // Need to pass along the keytab because of that. + if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) { + String principal = getAdminPrincipal(); + AuthenticationToken token = getAdminToken(); + assertTrue("Expected KerberosToken, but was " + token.getClass(), token instanceof KerberosToken); + KerberosToken kt = (KerberosToken) token; + assertNotNull("Expected keytab in token", kt.getKeytab()); + return control.exec( + TestMultiTableIngest.class, + args("--count", Integer.toString(ROWS), "--readonly", "-i", instance, "-z", keepers, "--tablePrefix", prefix, "--keytab", kt.getKeytab() + .getAbsolutePath(), "-u", principal)); + } + return control.exec( TestMultiTableIngest.class, - args("--count", "" + ROWS, "--readonly", "-u", "root", "-i", instance, "-z", keepers, "-p", new String( - ((PasswordToken) getToken()).getPassword(), UTF_8), "--tablePrefix", prefix)); + args("--count", Integer.toString(ROWS), "--readonly", "-u", getAdminPrincipal(), "-i", instance, "-z", keepers, "-p", new String( - ((PasswordToken) getAdminToken()).getPassword(), Charsets.UTF_8), "--tablePrefix", prefix)); ++ ((PasswordToken) getAdminToken()).getPassword(), UTF_8), "--tablePrefix", prefix)); } catch (IOException e) { log.error("Error running MultiTableIngest", e); return -1; http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java ---------------------------------------------------------------------- diff --cc test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java index d1fb9f9,a95c8d0..fba8b6d --- a/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java @@@ -16,6 -16,7 +16,7 @@@ */ package org.apache.accumulo.test.functional; -import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import java.io.IOException; @@@ -110,30 -107,12 +109,30 @@@ public class RestartIT extends Accumulo public void restartMaster() throws Exception { Connector c = getConnector(); final String tableName = getUniqueNames(1)[0]; - OPTS.tableName = tableName; - VOPTS.tableName = tableName; + OPTS.setTableName(tableName); + VOPTS.setTableName(tableName); c.tableOperations().create(tableName); - final PasswordToken token = (PasswordToken) getToken(); + final AuthenticationToken token = getAdminToken(); final ClusterControl control = getCluster().getClusterControl(); + final String[] args; + if (token instanceof PasswordToken) { + byte[] password = ((PasswordToken) token).getPassword(); - args = new String[] {"-u", getAdminPrincipal(), "-p", new String(password, Charsets.UTF_8), "-i", cluster.getInstanceName(), "-z", - cluster.getZooKeepers(), "--rows", "" + OPTS.rows, "--table", tableName}; ++ args = new String[] {"-u", getAdminPrincipal(), "-p", new String(password, UTF_8), "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers(), ++ "--rows", "" + OPTS.rows, "--table", tableName}; + OPTS.setPrincipal(getAdminPrincipal()); + VOPTS.setPrincipal(getAdminPrincipal()); + } else if (token instanceof KerberosToken) { + ClusterUser rootUser = getAdminUser(); + args = new String[] {"-u", getAdminPrincipal(), "--keytab", rootUser.getKeytab().getAbsolutePath(), "-i", cluster.getInstanceName(), "-z", + cluster.getZooKeepers(), "--rows", "" + OPTS.rows, "--table", tableName}; + ClientConfiguration clientConfig = cluster.getClientConfig(); + OPTS.updateKerberosCredentials(clientConfig); + VOPTS.updateKerberosCredentials(clientConfig); + } else { + throw new RuntimeException("Unknown token"); + } + Future ret = svc.submit(new Callable() { @Override public Integer call() { @@@ -209,30 -181,11 +208,30 @@@ public void restartMasterSplit() throws Exception { Connector c = getConnector(); final String tableName = getUniqueNames(1)[0]; - final PasswordToken token = (PasswordToken) getToken(); + final AuthenticationToken token = getAdminToken(); final ClusterControl control = getCluster().getClusterControl(); - VOPTS.tableName = tableName; + VOPTS.setTableName(tableName); c.tableOperations().create(tableName); c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "5K"); + + final String[] args; + if (token instanceof PasswordToken) { + byte[] password = ((PasswordToken) token).getPassword(); - args = new String[] {"-u", getAdminPrincipal(), "-p", new String(password, Charsets.UTF_8), "-i", cluster.getInstanceName(), "-z", - cluster.getZooKeepers(), "--rows", Integer.toString(VOPTS.rows), "--table", tableName}; ++ args = new String[] {"-u", getAdminPrincipal(), "-p", new String(password, UTF_8), "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers(), ++ "--rows", Integer.toString(VOPTS.rows), "--table", tableName}; + OPTS.setPrincipal(getAdminPrincipal()); + VOPTS.setPrincipal(getAdminPrincipal()); + } else if (token instanceof KerberosToken) { + ClusterUser rootUser = getAdminUser(); + args = new String[] {"-u", getAdminPrincipal(), "--keytab", rootUser.getKeytab().getAbsolutePath(), "-i", cluster.getInstanceName(), "-z", + cluster.getZooKeepers(), "--rows", Integer.toString(VOPTS.rows), "--table", tableName}; + ClientConfiguration clientConfig = cluster.getClientConfig(); + OPTS.updateKerberosCredentials(clientConfig); + VOPTS.updateKerberosCredentials(clientConfig); + } else { + throw new RuntimeException("Unknown token"); + } + Future ret = svc.submit(new Callable() { @Override public Integer call() { http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java ---------------------------------------------------------------------- diff --cc test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java index 68448eb,57456b6..af2eee1 --- a/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java @@@ -16,8 -16,10 +16,9 @@@ */ package org.apache.accumulo.test.functional; -import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; @@@ -106,19 -104,6 +105,19 @@@ public class RestartStressIT extends Ac c.tableOperations().create(tableName); c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "500K"); final ClusterControl control = getCluster().getClusterControl(); + final String[] args; + if (token instanceof PasswordToken) { + byte[] password = ((PasswordToken) token).getPassword(); - args = new String[] {"-u", getAdminPrincipal(), "-p", new String(password, Charsets.UTF_8), "-i", cluster.getInstanceName(), "-z", - cluster.getZooKeepers(), "--rows", "" + VOPTS.rows, "--table", tableName}; ++ args = new String[] {"-u", getAdminPrincipal(), "-p", new String(password, UTF_8), "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers(), ++ "--rows", "" + VOPTS.rows, "--table", tableName}; + } else if (token instanceof KerberosToken) { + ClusterUser rootUser = getAdminUser(); + args = new String[] {"-u", getAdminPrincipal(), "--keytab", rootUser.getKeytab().getAbsolutePath(), "-i", cluster.getInstanceName(), "-z", + cluster.getZooKeepers(), "--rows", "" + VOPTS.rows, "--table", tableName}; + } else { + throw new RuntimeException("Unrecognized token"); + } + Future retCode = svc.submit(new Callable() { @Override public Integer call() { http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/test/src/test/java/org/apache/accumulo/test/functional/ScanIdIT.java ---------------------------------------------------------------------- diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanIdIT.java index 720f87c,726ab8b..b54d41a --- a/test/src/test/java/org/apache/accumulo/test/functional/ScanIdIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanIdIT.java @@@ -16,7 -16,7 +16,7 @@@ */ package org.apache.accumulo.test.functional; --import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; http://git-wip-us.apache.org/repos/asf/accumulo/blob/4af87bfc/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java ---------------------------------------------------------------------- diff --cc test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java index 8e7e2b5,910fcbc..a6f0d00 --- a/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java @@@ -16,6 -16,7 +16,7 @@@ */ package org.apache.accumulo.test.functional; -import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@@ -161,21 -151,13 +160,21 @@@ public class SplitIT extends AccumuloCl shortened++; count++; } + assertTrue("Shortened should be greater than zero: " + shortened, shortened > 0); assertTrue("Count should be cgreater than 10: " + count, count > 10); - PasswordToken token = (PasswordToken) getToken(); - assertEquals( - 0, - getCluster().getClusterControl().exec(CheckForMetadataProblems.class, - new String[] {"-i", cluster.getInstanceName(), "-u", "root", "-p", new String(token.getPassword(), UTF_8), "-z", cluster.getZooKeepers()})); + + String[] args; + if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) { + ClusterUser rootUser = getAdminUser(); + args = new String[] {"-i", cluster.getInstanceName(), "-u", rootUser.getPrincipal(), "--keytab", rootUser.getKeytab().getAbsolutePath(), "-z", + cluster.getZooKeepers()}; + } else { + PasswordToken token = (PasswordToken) getAdminToken(); - args = new String[] {"-i", cluster.getInstanceName(), "-u", "root", "-p", new String(token.getPassword(), Charsets.UTF_8), "-z", cluster.getZooKeepers()}; ++ args = new String[] {"-i", cluster.getInstanceName(), "-u", "root", "-p", new String(token.getPassword(), UTF_8), "-z", cluster.getZooKeepers()}; + } + + assertEquals(0, getCluster().getClusterControl().exec(CheckForMetadataProblems.class, args)); } @Test @@@ -202,15 -184,9 +201,15 @@@ Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); - PasswordToken token = (PasswordToken) getToken(); c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K"); - DeleteIT.deleteTest(c, getCluster(), new String(token.getPassword(), UTF_8), tableName); + ClientConfiguration clientConfig = getCluster().getClientConfig(); + String password = null, keytab = null; + if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) { + keytab = getAdminUser().getKeytab().getAbsolutePath(); + } else { - password = new String(((PasswordToken) getAdminToken()).getPassword(), Charsets.UTF_8); ++ password = new String(((PasswordToken) getAdminToken()).getPassword(), UTF_8); + } + DeleteIT.deleteTest(c, getCluster(), getAdminPrincipal(), password, tableName, keytab); c.tableOperations().flush(tableName, null, null, true); for (int i = 0; i < 5; i++) { UtilWaitThread.sleep(10 * 1000);