Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id B7E0B200CA4 for ; Tue, 23 May 2017 14:22:51 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id B6687160BC3; Tue, 23 May 2017 12:22:51 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 73173160BDC for ; Tue, 23 May 2017 14:22:49 +0200 (CEST) Received: (qmail 5558 invoked by uid 500); 23 May 2017 12:22:48 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 5160 invoked by uid 99); 23 May 2017 12:22:47 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 23 May 2017 12:22:47 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 2C734DFAEB; Tue, 23 May 2017 12:22:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dlmarion@apache.org To: commits@accumulo.apache.org Date: Tue, 23 May 2017 12:22:50 -0000 Message-Id: <40b1e8163a8a42cd864edd1bad927534@git.apache.org> In-Reply-To: <561248980a3c443fb057fa454baf8b26@git.apache.org> References: <561248980a3c443fb057fa454baf8b26@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [04/13] accumulo git commit: ACCUMULO-4638 Clean up some AccumuloConfiguration archived-at: Tue, 23 May 2017 12:22:51 -0000 ACCUMULO-4638 Clean up some AccumuloConfiguration Lots of small refactorings to clean up AccumuloConfiguration, ServerConfigurationFactory, and related classes. This should clean up some of the internal configuration API, to make it easier to improve implementations later. * Move static type converter methods to helper class * Remove redundant default config accessor * Replace static filter classes with simple lambdas * Remove static helper for client-side get table config (and comment on the now-obvious potential race condition in OfflineIterator for getting this config via the client API) * Improve PropertyTest.testSensitiveKeys() use of DefaultConfiguration * Rename ServerConfigurationFactory.getConfiguration() to ServerConfigurationFactory.getSystemConfiguration() to more accurately reflect what it retrieves * Remove unnecessary KeyExtent version of ServerConfigurationFactory.getTableConfiguration(...) * Use Java 8 streams and lambda syntax to clean up some configuration implementations Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b78edd33 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b78edd33 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b78edd33 Branch: refs/heads/IGNITE Commit: b78edd3329cb7c981329fa2038a9352f79ab466d Parents: 2e171cd Author: Christopher Tubbs Authored: Thu May 11 22:10:44 2017 -0400 Committer: Christopher Tubbs Committed: Thu May 11 22:10:44 2017 -0400 ---------------------------------------------------------------------- .../apache/accumulo/core/cli/ClientOpts.java | 5 +- .../accumulo/core/client/ZooKeeperInstance.java | 4 +- .../core/client/impl/OfflineIterator.java | 5 +- .../lib/impl/FileOutputConfigurator.java | 3 +- .../core/client/mock/MockNamespace.java | 4 +- .../core/client/mock/MockScannerBase.java | 3 +- .../accumulo/core/client/mock/MockTable.java | 6 +- .../core/client/mock/MockTableOperations.java | 4 +- .../core/client/rfile/RFileScanner.java | 6 +- .../client/rfile/RFileSummariesRetriever.java | 3 +- .../core/client/rfile/RFileWriterBuilder.java | 3 +- .../accumulo/core/compaction/SizeType.java | 4 +- .../core/conf/AccumuloConfiguration.java | 208 +------------------ .../accumulo/core/conf/ConfigSanityCheck.java | 2 +- .../core/conf/ConfigurationTypeHelper.java | 139 +++++++++++++ .../core/conf/DefaultConfiguration.java | 24 +-- .../accumulo/core/file/BloomFilterLayer.java | 3 +- .../accumulo/core/file/rfile/CreateEmpty.java | 2 +- .../apache/accumulo/core/file/rfile/RFile.java | 4 +- .../accumulo/core/file/rfile/SplitLarge.java | 2 +- .../iterators/user/RowEncodingIterator.java | 8 +- .../iterators/user/TransformingIterator.java | 6 +- .../accumulo/core/trace/DistributedTrace.java | 3 +- .../client/mock/MockTableOperationsTest.java | 4 +- .../accumulo/core/client/rfile/RFileTest.java | 4 +- .../core/conf/AccumuloConfigurationTest.java | 73 +------ .../core/conf/ConfigurationTypeHelperTest.java | 118 +++++++++++ .../core/conf/DefaultConfigurationTest.java | 6 +- .../apache/accumulo/core/conf/PropertyTest.java | 67 ++---- .../core/file/BloomFilterLayerLookupTest.java | 4 +- .../accumulo/core/file/FileOperationsTest.java | 3 +- .../core/file/rfile/CreateCompatTestFile.java | 4 +- .../core/file/rfile/MultiLevelIndexTest.java | 3 +- .../core/file/rfile/MultiThreadedRFileTest.java | 5 +- .../accumulo/core/file/rfile/RFileTest.java | 13 +- .../iterators/DefaultIteratorEnvironment.java | 3 +- .../iterators/user/IndexedDocIteratorTest.java | 4 +- .../core/security/crypto/CryptoTest.java | 3 +- .../standalone/StandaloneAccumuloCluster.java | 3 +- .../impl/MiniAccumuloClusterImpl.java | 3 +- .../impl/MiniAccumuloConfigImpl.java | 2 +- .../java/org/apache/accumulo/proxy/Proxy.java | 4 +- .../org/apache/accumulo/server/Accumulo.java | 2 +- .../accumulo/server/AccumuloServerContext.java | 2 +- .../server/client/ClientServiceHandler.java | 4 +- .../accumulo/server/conf/ConfigSanityCheck.java | 2 +- .../server/conf/NamespaceConfiguration.java | 24 +-- .../server/conf/ServerConfiguration.java | 5 +- .../server/conf/ServerConfigurationFactory.java | 24 +-- .../accumulo/server/fs/VolumeManagerImpl.java | 2 +- .../apache/accumulo/server/init/Initialize.java | 5 +- .../balancer/HostRegexTableLoadBalancer.java | 12 +- .../master/balancer/RegexGroupBalancer.java | 4 +- .../tabletserver/LargestFirstMemoryManager.java | 4 +- .../org/apache/accumulo/server/util/Admin.java | 2 +- .../accumulo/server/util/LoginProperties.java | 2 +- .../server/AccumuloServerContextTest.java | 2 +- .../conf/ServerConfigurationFactoryTest.java | 2 +- .../BaseHostRegexTableLoadBalancerTest.java | 2 +- .../HostRegexTableLoadBalancerTest.java | 2 +- .../accumulo/server/util/TServerUtilsTest.java | 23 +- .../accumulo/gc/SimpleGarbageCollector.java | 2 +- .../accumulo/gc/SimpleGarbageCollectorTest.java | 2 +- .../java/org/apache/accumulo/master/Master.java | 4 +- .../master/tableOps/WriteExportFiles.java | 3 +- .../org/apache/accumulo/monitor/Monitor.java | 6 +- .../org/apache/accumulo/tracer/TraceServer.java | 4 +- .../apache/accumulo/tserver/FileManager.java | 5 +- .../apache/accumulo/tserver/TabletServer.java | 12 +- .../tserver/TabletServerResourceManager.java | 10 +- .../compaction/SizeLimitCompactionStrategy.java | 4 +- .../compaction/TwoTierCompactionStrategy.java | 4 +- .../replication/AccumuloReplicaSystem.java | 4 +- .../tserver/replication/ReplicationWorker.java | 2 +- .../tserver/tablet/DatafileManager.java | 4 +- .../tserver/LargestFirstMemoryManagerTest.java | 14 +- .../DefaultCompactionStrategyTest.java | 3 +- .../SizeLimitCompactionStrategyTest.java | 7 +- .../TwoTierCompactionStrategyTest.java | 10 +- .../ConfigurableCompactionStrategyTest.java | 8 +- .../tserver/tablet/DatafileManagerTest.java | 4 +- .../accumulo/shell/commands/ConfigCommand.java | 4 +- .../accumulo/shell/commands/DeleteCommand.java | 4 +- .../accumulo/shell/commands/InsertCommand.java | 4 +- .../accumulo/shell/commands/MergeCommand.java | 4 +- .../accumulo/shell/commands/ScanCommand.java | 5 +- .../accumulo/test/BulkImportMonitoringIT.java | 6 +- .../apache/accumulo/test/CreateRandomRFile.java | 4 +- .../org/apache/accumulo/test/ExistingMacIT.java | 4 +- .../apache/accumulo/test/GetFileInfoBulkIT.java | 6 +- .../apache/accumulo/test/LargeSplitRowIT.java | 8 +- .../org/apache/accumulo/test/ShellServerIT.java | 3 +- .../test/TableConfigurationUpdateIT.java | 4 +- .../org/apache/accumulo/test/TestIngest.java | 4 +- .../accumulo/test/TransportCachingIT.java | 4 +- .../accumulo/test/functional/BulkFileIT.java | 2 +- .../test/functional/DeleteEverythingIT.java | 6 +- .../test/functional/DynamicThreadPoolsIT.java | 7 +- .../test/functional/ScanSessionTimeOutIT.java | 7 +- .../accumulo/test/functional/SplitIT.java | 4 +- .../CloseWriteAheadLogReferencesIT.java | 2 +- .../test/mapreduce/AccumuloInputFormatIT.java | 2 +- .../performance/metadata/FastBulkImportIT.java | 4 +- .../performance/scan/CollectTabletStats.java | 6 +- 104 files changed, 537 insertions(+), 581 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java b/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java index 9354f18..2051246 100644 --- a/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java +++ b/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java @@ -40,6 +40,7 @@ import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Prope import org.apache.accumulo.core.client.security.tokens.KerberosToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.security.Authorizations; @@ -65,14 +66,14 @@ public class ClientOpts extends Help { public static class TimeConverter implements IStringConverter { @Override public Long convert(String value) { - return AccumuloConfiguration.getTimeInMillis(value); + return ConfigurationTypeHelper.getTimeInMillis(value); } } public static class MemoryConverter implements IStringConverter { @Override public Long convert(String value) { - return AccumuloConfiguration.getFixedMemoryAsBytes(value); + return ConfigurationTypeHelper.getFixedMemoryAsBytes(value); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java index b983a20..2d65bf5 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java @@ -33,7 +33,7 @@ import org.apache.accumulo.core.client.impl.Credentials; import org.apache.accumulo.core.client.impl.InstanceOperationsImpl; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.util.ByteBufferUtil; import org.apache.accumulo.core.util.OpTimer; @@ -150,7 +150,7 @@ public class ZooKeeperInstance implements Instance { if ((instanceId == null) == (instanceName == null)) throw new IllegalArgumentException("Expected exactly one of instanceName and instanceId to be set"); this.zooKeepers = clientConf.get(ClientProperty.INSTANCE_ZK_HOST); - this.zooKeepersSessionTimeOut = (int) AccumuloConfiguration.getTimeInMillis(clientConf.get(ClientProperty.INSTANCE_ZK_TIMEOUT)); + this.zooKeepersSessionTimeOut = (int) ConfigurationTypeHelper.getTimeInMillis(clientConf.get(ClientProperty.INSTANCE_ZK_TIMEOUT)); zooCache = zcf.getZooCache(zooKeepers, zooKeepersSessionTimeOut); if (null != instanceName) { // Validates that the provided instanceName actually exists http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java index 9f51704..0546ab9 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java @@ -319,7 +319,10 @@ class OfflineIterator implements Iterator> { IOException { // TODO share code w/ tablet - ACCUMULO-1303 - AccumuloConfiguration acuTableConf = AccumuloConfiguration.getTableConfiguration(conn, tableId); + + // possible race condition here, if table is renamed + String tableName = Tables.getTableName(conn.getInstance(), tableId); + AccumuloConfiguration acuTableConf = new ConfigurationCopy(conn.tableOperations().getProperties(tableName)); Configuration conf = CachedConfiguration.getInstance(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java index 5f73e90..1da81fe 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java @@ -25,6 +25,7 @@ import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.client.summary.SummarizerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl; import org.apache.hadoop.conf.Configuration; @@ -100,7 +101,7 @@ public class FileOutputConfigurator extends ConfiguratorBase { */ public static AccumuloConfiguration getAccumuloConfiguration(Class implementingClass, Configuration conf) { String prefix = enumToConfKey(implementingClass, Opts.ACCUMULO_PROPERTIES) + "."; - ConfigurationCopy acuConf = new ConfigurationCopy(AccumuloConfiguration.getDefaultConfiguration()); + ConfigurationCopy acuConf = new ConfigurationCopy(DefaultConfiguration.getInstance()); for (Entry entry : conf) if (entry.getKey().startsWith(prefix)) { String propString = entry.getKey().substring(prefix.length()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/mock/MockNamespace.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockNamespace.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockNamespace.java index 456580b..4f57d50 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockNamespace.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockNamespace.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.security.NamespacePermission; @@ -38,7 +38,7 @@ public class MockNamespace { public MockNamespace() { settings = new HashMap<>(); - for (Entry entry : AccumuloConfiguration.getDefaultConfiguration()) { + for (Entry entry : DefaultConfiguration.getInstance()) { String key = entry.getKey(); if (key.startsWith(Property.TABLE_PREFIX.getKey())) { settings.put(key, entry.getValue()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/mock/MockScannerBase.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockScannerBase.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockScannerBase.java index 6dd5737..294e1d3 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockScannerBase.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockScannerBase.java @@ -27,6 +27,7 @@ import org.apache.accumulo.core.client.ScannerBase; import org.apache.accumulo.core.client.impl.ScannerOptions; import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Column; @@ -81,7 +82,7 @@ public class MockScannerBase extends ScannerOptions implements ScannerBase { @Override public AccumuloConfiguration getConfig() { - return AccumuloConfiguration.getDefaultConfiguration(); + return DefaultConfiguration.getInstance(); } @Override http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java index 6a8bd2f..0124ebd 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java @@ -30,7 +30,7 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ConcurrentSkipListSet; import org.apache.accumulo.core.client.admin.TimeType; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ColumnUpdate; import org.apache.accumulo.core.data.Key; @@ -102,7 +102,7 @@ public class MockTable { this.timeType = timeType; this.tableId = tableId; settings = IteratorUtil.generateInitialTableProperties(limitVersion); - for (Entry entry : AccumuloConfiguration.getDefaultConfiguration()) { + for (Entry entry : DefaultConfiguration.getInstance()) { String key = entry.getKey(); if (key.startsWith(Property.TABLE_PREFIX.getKey())) settings.put(key, entry.getValue()); @@ -129,7 +129,7 @@ public class MockTable { this.timeType = timeType; this.tableId = tableId; settings = properties; - for (Entry entry : AccumuloConfiguration.getDefaultConfiguration()) { + for (Entry entry : DefaultConfiguration.getInstance()) { String key = entry.getKey(); if (key.startsWith(Property.TABLE_PREFIX.getKey())) settings.put(key, entry.getValue()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java index de486d7..72cb939 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java @@ -48,7 +48,7 @@ import org.apache.accumulo.core.client.impl.TableOperationsHelper; import org.apache.accumulo.core.client.impl.Tables; import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.client.summary.SummarizerConfiguration; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; @@ -292,7 +292,7 @@ class MockTableOperations extends TableOperationsHelper { for (FileStatus importStatus : fs.listStatus(importPath)) { try { FileSKVIterator importIterator = FileOperations.getInstance().newReaderBuilder().forFile(importStatus.getPath().toString(), fs, fs.getConf()) - .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).seekToBeginning().build(); + .withTableConfiguration(DefaultConfiguration.getInstance()).seekToBeginning().build(); while (importIterator.hasTop()) { Key key = importIterator.getTopKey(); Value value = importIterator.getTopValue(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java index 1b12fb6..c95531b 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java @@ -34,8 +34,8 @@ import org.apache.accumulo.core.client.impl.BaseIteratorEnvironment; import org.apache.accumulo.core.client.impl.ScannerOptions; import org.apache.accumulo.core.client.rfile.RFileScannerBuilder.InputArgs; import org.apache.accumulo.core.client.sample.SamplerConfiguration; -import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Column; @@ -73,7 +73,7 @@ class RFileScanner extends ScannerOptions implements Scanner { private int batchSize = 1000; private long readaheadThreshold = 3; - private static final long CACHE_BLOCK_SIZE = AccumuloConfiguration.getDefaultConfiguration().getAsBytes(Property.TSERV_DEFAULT_BLOCKSIZE); + private static final long CACHE_BLOCK_SIZE = DefaultConfiguration.getInstance().getAsBytes(Property.TSERV_DEFAULT_BLOCKSIZE); static class Opts { InputArgs in; @@ -281,7 +281,7 @@ class RFileScanner extends ScannerOptions implements Scanner { for (int i = 0; i < sources.length; i++) { FSDataInputStream inputStream = (FSDataInputStream) sources[i].getInputStream(); readers.add(new RFile.Reader(new CachableBlockFile.Reader(inputStream, sources[i].getLength(), opts.in.getConf(), dataCache, indexCache, - AccumuloConfiguration.getDefaultConfiguration()))); + DefaultConfiguration.getInstance()))); } if (getSamplerConfiguration() != null) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java index 367172a..b1f8954 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java @@ -30,6 +30,7 @@ import org.apache.accumulo.core.client.rfile.RFileScannerBuilder.InputArgs; import org.apache.accumulo.core.client.summary.SummarizerConfiguration; import org.apache.accumulo.core.client.summary.Summary; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.summary.SummaryReader; import org.apache.accumulo.core.summary.Gatherer; import org.apache.accumulo.core.summary.SummarizerFactory; @@ -79,7 +80,7 @@ class RFileSummariesRetriever implements SummaryInputArguments, SummaryFSOptions @Override public Collection read() throws IOException { SummarizerFactory factory = new SummarizerFactory(); - AccumuloConfiguration acuconf = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration acuconf = DefaultConfiguration.getInstance(); Configuration conf = in.getFileSystem().getConf(); RFileSource[] sources = in.getSources(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java index a7decb1..9097a8b 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java @@ -33,6 +33,7 @@ import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.client.summary.SummarizerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.file.FileOperations; import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl; import org.apache.hadoop.fs.FSDataOutputStream; @@ -83,7 +84,7 @@ class RFileWriterBuilder implements RFile.OutputArguments, RFile.WriterFSOptions @Override public RFileWriter build() throws IOException { FileOperations fileops = FileOperations.getInstance(); - AccumuloConfiguration acuconf = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration acuconf = DefaultConfiguration.getInstance(); HashMap userProps = new HashMap<>(); userProps.putAll(tableConfig); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/compaction/SizeType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/compaction/SizeType.java b/core/src/main/java/org/apache/accumulo/core/compaction/SizeType.java index ec53f87..cc44d51 100644 --- a/core/src/main/java/org/apache/accumulo/core/compaction/SizeType.java +++ b/core/src/main/java/org/apache/accumulo/core/compaction/SizeType.java @@ -18,12 +18,12 @@ package org.apache.accumulo.core.compaction; import static com.google.common.base.Preconditions.checkArgument; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; class SizeType implements Type { @Override public String convert(String str) { - long size = AccumuloConfiguration.getFixedMemoryAsBytes(str); + long size = ConfigurationTypeHelper.getFixedMemoryAsBytes(str); checkArgument(size > 0); return Long.toString(size); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java index a9ce195..3beeb66 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java @@ -22,14 +22,9 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.TreeMap; -import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.client.impl.Tables; import org.apache.accumulo.core.conf.PropertyType.PortRange; import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; @@ -41,52 +36,6 @@ import org.slf4j.LoggerFactory; */ public abstract class AccumuloConfiguration implements Iterable> { - /** - * A filter that accepts properties whose keys are an exact match. - */ - public static class MatchFilter implements Predicate { - - private String match; - - /** - * Creates a new filter. - * - * @param match - * prefix of property keys to accept - */ - public MatchFilter(String match) { - this.match = match; - } - - @Override - public boolean test(String key) { - return Objects.equals(match, key); - } - } - - /** - * A filter that accepts properties whose keys begin with a prefix. - */ - public static class PrefixFilter implements Predicate { - - private String prefix; - - /** - * Creates a new filter. - * - * @param prefix - * prefix of property keys to accept - */ - public PrefixFilter(String prefix) { - this.prefix = prefix; - } - - @Override - public boolean test(String key) { - return key.startsWith(prefix); - } - } - private static final Logger log = LoggerFactory.getLogger(AccumuloConfiguration.class); /** @@ -101,7 +50,7 @@ public abstract class AccumuloConfiguration implements Iterable propMap = new HashMap<>(1); - getProperties(propMap, new MatchFilter(property)); + getProperties(propMap, key -> Objects.equals(property, key)); return propMap.get(property); } @@ -138,7 +87,7 @@ public abstract class AccumuloConfiguration implements Iterable propMap = new HashMap<>(); - getProperties(propMap, new PrefixFilter(property.getKey())); + getProperties(propMap, key -> key.startsWith(property.getKey())); return propMap; } @@ -176,77 +125,15 @@ public abstract class AccumuloConfiguration implements Iterable= 100) { - throw new IllegalArgumentException("The value '" + str + "' is not a valid memory setting."); - } - return Runtime.getRuntime().maxMemory() * percent / 100; - } catch (Exception ex) { - throw new IllegalArgumentException("The value '" + str + "' is not a valid memory setting."); - } - } - return getFixedMemoryAsBytes(str); - } - - /** * Gets a property of type {@link PropertyType#TIMEDURATION}, interpreting the value properly. * * @param property @@ -254,48 +141,11 @@ public abstract class AccumuloConfiguration implements Iterable 0 && str.charAt(str.length() - 1) == '%') - return Double.parseDouble(str.substring(0, str.length() - 1)) / 100.0; - return Double.parseDouble(str); + return ConfigurationTypeHelper.getFraction(get(property)); } /** @@ -349,7 +185,6 @@ public abstract class AccumuloConfiguration implements Iterable= 100) { + throw new IllegalArgumentException("The value '" + str + "' is not a valid memory setting."); + } + return Runtime.getRuntime().maxMemory() * percent / 100; + } catch (Exception ex) { + throw new IllegalArgumentException("The value '" + str + "' is not a valid memory setting."); + } + } + return getFixedMemoryAsBytes(str); + } + + /** + * Interprets a string specifying a time duration. A time duration is specified as a long integer followed by an optional d (days), h (hours), m (minutes), s + * (seconds), or ms (milliseconds). A value without a unit is interpreted as seconds. + * + * @param str + * string value + * @return interpreted time duration in milliseconds + */ + public static long getTimeInMillis(String str) { + TimeUnit timeUnit; + int unitsLen = 1; + switch (str.charAt(str.length() - 1)) { + case 'd': + timeUnit = TimeUnit.DAYS; + break; + case 'h': + timeUnit = TimeUnit.HOURS; + break; + case 'm': + timeUnit = TimeUnit.MINUTES; + break; + case 's': + timeUnit = TimeUnit.SECONDS; + if (str.endsWith("ms")) { + timeUnit = TimeUnit.MILLISECONDS; + unitsLen = 2; + } + break; + default: + timeUnit = TimeUnit.SECONDS; + unitsLen = 0; + break; + } + return timeUnit.toMillis(Long.parseLong(str.substring(0, str.length() - unitsLen))); + } + + /** + * Interprets a string specifying a fraction. A fraction is specified as a double. An optional % at the end signifies a percentage. + * + * @param str + * string value + * @return interpreted fraction as a decimal value + */ + public static double getFraction(String str) { + if (str.length() > 0 && str.charAt(str.length() - 1) == '%') + return Double.parseDouble(str.substring(0, str.length() - 1)) / 100.0; + return Double.parseDouble(str); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java index 9386f99..756e241 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java @@ -16,26 +16,20 @@ */ package org.apache.accumulo.core.conf; -import java.util.Collections; -import java.util.HashMap; +import java.util.Arrays; import java.util.Map; -import java.util.Map.Entry; import java.util.function.Predicate; +import java.util.stream.Collectors; /** * An {@link AccumuloConfiguration} that contains only default values for properties. This class is a singleton. */ public class DefaultConfiguration extends AccumuloConfiguration { - private final static Map resolvedProps; - static { - Map m = new HashMap<>(); - for (Property prop : Property.values()) { - if (!prop.getType().equals(PropertyType.PREFIX)) { - m.put(prop.getKey(), prop.getDefaultValue()); - } - } - resolvedProps = Collections.unmodifiableMap(m); - } + + private static final Map resolvedProps = Arrays.stream(Property.values()).filter(p -> p.getType() != PropertyType.PREFIX) + .collect(Collectors.toMap(p -> p.getKey(), p -> p.getDefaultValue())); + + private DefaultConfiguration() {} /** * Gets a default configuration. @@ -53,8 +47,6 @@ public class DefaultConfiguration extends AccumuloConfiguration { @Override public void getProperties(Map props, Predicate filter) { - for (Entry entry : resolvedProps.entrySet()) - if (filter.test(entry.getKey())) - props.put(entry.getKey(), entry.getValue()); + resolvedProps.entrySet().stream().filter(p -> filter.test(p.getKey())).forEach(e -> props.put(e.getKey(), e.getValue())); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java b/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java index 4808da3..01fb699 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java +++ b/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java @@ -41,6 +41,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.accumulo.core.bloomfilter.DynamicBloomFilter; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.PartialKey; @@ -448,7 +449,7 @@ public class BloomFilterLayer { ArrayList vals = new ArrayList<>(valsSet); Collections.sort(vals); - ConfigurationCopy acuconf = new ConfigurationCopy(AccumuloConfiguration.getDefaultConfiguration()); + ConfigurationCopy acuconf = new ConfigurationCopy(DefaultConfiguration.getInstance()); acuconf.set(Property.TABLE_BLOOM_ENABLED, "true"); acuconf.set(Property.TABLE_BLOOM_KEY_FUNCTOR, "accumulo.core.file.keyfunctor.ColumnFamilyFunctor"); acuconf.set(Property.TABLE_FILE_TYPE, RFile.EXTENSION); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java index 3a41e95..d48a0eb 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java +++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java @@ -78,7 +78,7 @@ public class CreateEmpty { Path path = new Path(arg); log.info("Writing to file '" + path + "'"); FileSKVWriter writer = (new RFileOperations()).newWriterBuilder().forFile(arg, path.getFileSystem(conf), conf) - .withTableConfiguration(DefaultConfiguration.getDefaultConfiguration()).withCompression(opts.codec).build(); + .withTableConfiguration(DefaultConfiguration.getInstance()).withCompression(opts.codec).build(); writer.close(); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java index 16db01d..e7d5cd4 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java +++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java @@ -41,7 +41,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.sample.Sampler; import org.apache.accumulo.core.client.sample.SamplerConfiguration; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; @@ -518,7 +518,7 @@ public class RFile { private Sampler sampler; public Writer(BlockFileWriter bfw, int blockSize) throws IOException { - this(bfw, blockSize, (int) AccumuloConfiguration.getDefaultConfiguration().getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX), null, null); + this(bfw, blockSize, (int) DefaultConfiguration.getInstance().getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX), null, null); } public Writer(BlockFileWriter bfw, int blockSize, int indexBlockSize, SamplerConfigurationImpl samplerConfig, Sampler sampler) throws IOException { http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java index 1f414d5..4816990 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java +++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java @@ -57,7 +57,7 @@ public class SplitLarge { opts.parseArgs(SplitLarge.class.getName(), args); for (String file : opts.files) { - AccumuloConfiguration aconf = DefaultConfiguration.getDefaultConfiguration(); + AccumuloConfiguration aconf = DefaultConfiguration.getInstance(); Path path = new Path(file); CachableBlockFile.Reader rdr = new CachableBlockFile.Reader(fs, path, conf, null, null, aconf); try (Reader iter = new RFile.Reader(rdr)) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java b/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java index 150d1fa..ebfbea9 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java @@ -19,12 +19,12 @@ package org.apache.accumulo.core.iterators.user; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.SortedMap; -import java.util.HashMap; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.PartialKey; @@ -155,7 +155,7 @@ public abstract class RowEncodingIterator implements SortedKeyValueIterator source, Map options, IteratorEnvironment env) throws IOException { sourceIter = source; if (options.containsKey(MAX_BUFFER_SIZE_OPT)) { - maxBufferSize = AccumuloConfiguration.getFixedMemoryAsBytes(options.get(MAX_BUFFER_SIZE_OPT)); + maxBufferSize = ConfigurationTypeHelper.getFixedMemoryAsBytes(options.get(MAX_BUFFER_SIZE_OPT)); } } @@ -172,7 +172,7 @@ public abstract class RowEncodingIterator implements SortedKeyValueIterator options) { String maxBufferSizeStr = options.get(MAX_BUFFER_SIZE_OPT); try { - AccumuloConfiguration.getFixedMemoryAsBytes(maxBufferSizeStr); + ConfigurationTypeHelper.getFixedMemoryAsBytes(maxBufferSizeStr); } catch (Exception e) { throw new IllegalArgumentException("Failed to parse opt " + MAX_BUFFER_SIZE_OPT + " " + maxBufferSizeStr, e); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java index 2dfd726..bd8ae15 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java @@ -30,7 +30,7 @@ import java.util.Map.Entry; import java.util.NoSuchElementException; import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.PartialKey; @@ -120,7 +120,7 @@ abstract public class TransformingIterator extends WrappingIterator implements O } if (options.containsKey(MAX_BUFFER_SIZE_OPT)) { - maxBufferSize = AccumuloConfiguration.getFixedMemoryAsBytes(options.get(MAX_BUFFER_SIZE_OPT)); + maxBufferSize = ConfigurationTypeHelper.getFixedMemoryAsBytes(options.get(MAX_BUFFER_SIZE_OPT)); } else { maxBufferSize = DEFAULT_MAX_BUFFER_SIZE; } @@ -150,7 +150,7 @@ abstract public class TransformingIterator extends WrappingIterator implements O if (option.getKey().equals(AUTH_OPT)) { new Authorizations(option.getValue().getBytes(UTF_8)); } else if (option.getKey().equals(MAX_BUFFER_SIZE_OPT)) { - AccumuloConfiguration.getFixedMemoryAsBytes(option.getValue()); + ConfigurationTypeHelper.getFixedMemoryAsBytes(option.getValue()); } } catch (Exception e) { throw new IllegalArgumentException("Failed to parse opt " + option.getKey() + " " + option.getValue(), e); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java b/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java index f833b11..649cb3b 100644 --- a/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java +++ b/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java @@ -25,6 +25,7 @@ import org.apache.accumulo.core.client.ClientConfiguration; import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.fate.zookeeper.ZooReader; import org.apache.hadoop.conf.Configuration; @@ -92,7 +93,7 @@ public class DistributedTrace { public static void enable(String hostname, String service, ClientConfiguration conf) { String spanReceivers = conf.get(ClientProperty.TRACE_SPAN_RECEIVERS); String zookeepers = conf.get(ClientProperty.INSTANCE_ZK_HOST); - long timeout = AccumuloConfiguration.getTimeInMillis(conf.get(ClientProperty.INSTANCE_ZK_TIMEOUT)); + long timeout = ConfigurationTypeHelper.getTimeInMillis(conf.get(ClientProperty.INSTANCE_ZK_TIMEOUT)); String zkPath = conf.get(ClientProperty.TRACE_ZK_PATH); Map properties = conf.getAllPropertiesWithPrefix(ClientProperty.TRACE_SPAN_RECEIVER_PREFIX); enableTracing(hostname, service, spanReceivers, zookeepers, timeout, zkPath, properties); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java b/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java index 58f3777..e19204a 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java @@ -42,7 +42,7 @@ import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.client.admin.TimeType; import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; @@ -226,7 +226,7 @@ public class MockTableOperationsTest { fs.mkdirs(failures); fs.mkdirs(tempFile.getParent()); FileSKVWriter writer = FileOperations.getInstance().newWriterBuilder().forFile(tempFile.toString(), fs, defaultConf) - .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build(); + .withTableConfiguration(DefaultConfiguration.getInstance()).build(); writer.startDefaultLocalityGroup(); List> keyVals = new ArrayList<>(); for (int i = 0; i < 5; i++) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java index 3352b0f..98f9983 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java @@ -42,7 +42,7 @@ import org.apache.accumulo.core.client.summary.SummarizerConfiguration; import org.apache.accumulo.core.client.summary.Summary; import org.apache.accumulo.core.client.summary.summarizers.FamilySummarizer; import org.apache.accumulo.core.client.summary.summarizers.VisibilitySummarizer; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; @@ -774,7 +774,7 @@ public class RFileTest { private Reader getReader(LocalFileSystem localFs, String testFile) throws IOException { Reader reader = (Reader) FileOperations.getInstance().newReaderBuilder().forFile(testFile).inFileSystem(localFs, localFs.getConf()) - .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build(); + .withTableConfiguration(DefaultConfiguration.getInstance()).build(); return reader; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java b/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java index f7a37bf..81b5791 100644 --- a/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java +++ b/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java @@ -19,70 +19,13 @@ package org.apache.accumulo.core.conf; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import java.util.Arrays; -import java.util.List; -import java.util.function.Function; - import org.junit.Test; public class AccumuloConfigurationTest { @Test - public void testGetMemoryInBytes() throws Exception { - List> funcs = Arrays.asList(AccumuloConfiguration::getFixedMemoryAsBytes, AccumuloConfiguration::getMemoryAsBytes); - for (Function memFunc : funcs) { - assertEquals(42l, memFunc.apply("42").longValue()); - assertEquals(42l, memFunc.apply("42b").longValue()); - assertEquals(42l, memFunc.apply("42B").longValue()); - assertEquals(42l * 1024l, memFunc.apply("42K").longValue()); - assertEquals(42l * 1024l, memFunc.apply("42k").longValue()); - assertEquals(42l * 1024l * 1024l, memFunc.apply("42M").longValue()); - assertEquals(42l * 1024l * 1024l, memFunc.apply("42m").longValue()); - assertEquals(42l * 1024l * 1024l * 1024l, memFunc.apply("42G").longValue()); - assertEquals(42l * 1024l * 1024l * 1024l, memFunc.apply("42g").longValue()); - } - assertEquals(Runtime.getRuntime().maxMemory() / 10, AccumuloConfiguration.getMemoryAsBytes("10%")); - assertEquals(Runtime.getRuntime().maxMemory() / 5, AccumuloConfiguration.getMemoryAsBytes("20%")); - } - - @Test - public void testGetTimeInMillis() { - assertEquals(42L * 24 * 60 * 60 * 1000, AccumuloConfiguration.getTimeInMillis("42d")); - assertEquals(42L * 60 * 60 * 1000, AccumuloConfiguration.getTimeInMillis("42h")); - assertEquals(42L * 60 * 1000, AccumuloConfiguration.getTimeInMillis("42m")); - assertEquals(42L * 1000, AccumuloConfiguration.getTimeInMillis("42s")); - assertEquals(42L * 1000, AccumuloConfiguration.getTimeInMillis("42")); - assertEquals(42L, AccumuloConfiguration.getTimeInMillis("42ms")); - } - - @Test(expected = IllegalArgumentException.class) - public void testGetFixedMemoryAsBytesFailureCases1() throws Exception { - AccumuloConfiguration.getFixedMemoryAsBytes("42x"); - } - - @Test(expected = IllegalArgumentException.class) - public void testGetFixedMemoryAsBytesFailureCases2() throws Exception { - AccumuloConfiguration.getFixedMemoryAsBytes("FooBar"); - } - - @Test(expected = IllegalArgumentException.class) - public void testGetFixedMemoryAsBytesFailureCases3() throws Exception { - AccumuloConfiguration.getFixedMemoryAsBytes("40%"); - } - - @Test(expected = IllegalArgumentException.class) - public void testGetMemoryAsBytesFailureCases1() throws Exception { - AccumuloConfiguration.getMemoryAsBytes("42x"); - } - - @Test(expected = IllegalArgumentException.class) - public void testGetMemoryAsBytesFailureCases2() throws Exception { - AccumuloConfiguration.getMemoryAsBytes("FooBar"); - } - - @Test public void testGetPropertyByString() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); boolean found = false; for (Property p : Property.values()) { if (p.getType() != PropertyType.PREFIX) { @@ -98,7 +41,7 @@ public class AccumuloConfigurationTest { @Test public void testGetSinglePort() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "9997"); int[] ports = cc.getPort(Property.TSERV_CLIENTPORT); @@ -108,7 +51,7 @@ public class AccumuloConfigurationTest { @Test public void testGetAnyPort() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "0"); int[] ports = cc.getPort(Property.TSERV_CLIENTPORT); @@ -118,7 +61,7 @@ public class AccumuloConfigurationTest { @Test public void testGetInvalidPort() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "1020"); int[] ports = cc.getPort(Property.TSERV_CLIENTPORT); @@ -128,7 +71,7 @@ public class AccumuloConfigurationTest { @Test public void testGetPortRange() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "9997-9999"); int[] ports = cc.getPort(Property.TSERV_CLIENTPORT); @@ -140,7 +83,7 @@ public class AccumuloConfigurationTest { @Test(expected = IllegalArgumentException.class) public void testGetPortRangeInvalidLow() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "1020-1026"); int[] ports = cc.getPort(Property.TSERV_CLIENTPORT); @@ -152,7 +95,7 @@ public class AccumuloConfigurationTest { @Test(expected = IllegalArgumentException.class) public void testGetPortRangeInvalidHigh() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "65533-65538"); int[] ports = cc.getPort(Property.TSERV_CLIENTPORT); @@ -164,7 +107,7 @@ public class AccumuloConfigurationTest { @Test(expected = IllegalArgumentException.class) public void testGetPortInvalidSyntax() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "[65533,65538]"); cc.getPort(Property.TSERV_CLIENTPORT); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/conf/ConfigurationTypeHelperTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/conf/ConfigurationTypeHelperTest.java b/core/src/test/java/org/apache/accumulo/core/conf/ConfigurationTypeHelperTest.java new file mode 100644 index 0000000..48d1f32 --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/conf/ConfigurationTypeHelperTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.conf; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.function.Function; + +import org.junit.Test; + +public class ConfigurationTypeHelperTest { + + @Test + public void testGetMemoryInBytes() throws Exception { + Arrays.> asList(ConfigurationTypeHelper::getFixedMemoryAsBytes, ConfigurationTypeHelper::getMemoryAsBytes).stream() + .forEach(memFunc -> { + assertEquals(42l, memFunc.apply("42").longValue()); + assertEquals(42l, memFunc.apply("42b").longValue()); + assertEquals(42l, memFunc.apply("42B").longValue()); + assertEquals(42l * 1024l, memFunc.apply("42K").longValue()); + assertEquals(42l * 1024l, memFunc.apply("42k").longValue()); + assertEquals(42l * 1024l * 1024l, memFunc.apply("42M").longValue()); + assertEquals(42l * 1024l * 1024l, memFunc.apply("42m").longValue()); + assertEquals(42l * 1024l * 1024l * 1024l, memFunc.apply("42G").longValue()); + assertEquals(42l * 1024l * 1024l * 1024l, memFunc.apply("42g").longValue()); + }); + assertEquals(Runtime.getRuntime().maxMemory() / 10, ConfigurationTypeHelper.getMemoryAsBytes("10%")); + assertEquals(Runtime.getRuntime().maxMemory() / 5, ConfigurationTypeHelper.getMemoryAsBytes("20%")); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFixedMemoryAsBytesFailureCases1() throws Exception { + ConfigurationTypeHelper.getFixedMemoryAsBytes("42x"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFixedMemoryAsBytesFailureCases2() throws Exception { + ConfigurationTypeHelper.getFixedMemoryAsBytes("FooBar"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFixedMemoryAsBytesFailureCases3() throws Exception { + ConfigurationTypeHelper.getFixedMemoryAsBytes("40%"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetMemoryAsBytesFailureCases1() throws Exception { + ConfigurationTypeHelper.getMemoryAsBytes("42x"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetMemoryAsBytesFailureCases2() throws Exception { + ConfigurationTypeHelper.getMemoryAsBytes("FooBar"); + } + + @Test + public void testGetTimeInMillis() { + assertEquals(42L * 24 * 60 * 60 * 1000, ConfigurationTypeHelper.getTimeInMillis("42d")); + assertEquals(42L * 60 * 60 * 1000, ConfigurationTypeHelper.getTimeInMillis("42h")); + assertEquals(42L * 60 * 1000, ConfigurationTypeHelper.getTimeInMillis("42m")); + assertEquals(42L * 1000, ConfigurationTypeHelper.getTimeInMillis("42s")); + assertEquals(42L * 1000, ConfigurationTypeHelper.getTimeInMillis("42")); + assertEquals(42L, ConfigurationTypeHelper.getTimeInMillis("42ms")); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetTimeInMillisFailureCase1() { + ConfigurationTypeHelper.getTimeInMillis("abc"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetTimeInMillisFailureCase2() { + ConfigurationTypeHelper.getTimeInMillis("ms"); + } + + @Test + public void testGetFraction() { + double delta = 0.0000000000001; + assertEquals(0.5d, ConfigurationTypeHelper.getFraction("0.5"), delta); + assertEquals(3.0d, ConfigurationTypeHelper.getFraction("3"), delta); + assertEquals(-0.25d, ConfigurationTypeHelper.getFraction("-25%"), delta); + assertEquals(0.99546d, ConfigurationTypeHelper.getFraction("99.546%"), delta); + assertEquals(0.0d, ConfigurationTypeHelper.getFraction("0%"), delta); + assertEquals(0.0d, ConfigurationTypeHelper.getFraction("-0.000"), delta); + assertEquals(0.001d, ConfigurationTypeHelper.getFraction(".1%"), delta); + assertEquals(1d, ConfigurationTypeHelper.getFraction("1."), delta); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFractionFailureCase1() { + ConfigurationTypeHelper.getFraction("%"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFractionFailureCase2() { + ConfigurationTypeHelper.getFraction("abc0%"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFractionFailureCase3() { + ConfigurationTypeHelper.getFraction(".%"); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/conf/DefaultConfigurationTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/conf/DefaultConfigurationTest.java b/core/src/test/java/org/apache/accumulo/core/conf/DefaultConfigurationTest.java index 9e4c5ec..b7e968a 100644 --- a/core/src/test/java/org/apache/accumulo/core/conf/DefaultConfigurationTest.java +++ b/core/src/test/java/org/apache/accumulo/core/conf/DefaultConfigurationTest.java @@ -17,6 +17,8 @@ package org.apache.accumulo.core.conf; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.util.Map; @@ -28,7 +30,7 @@ public class DefaultConfigurationTest { @Before public void setUp() { - c = new DefaultConfiguration(); + c = DefaultConfiguration.getInstance(); } @Test @@ -41,6 +43,8 @@ public class DefaultConfigurationTest { Map p = new java.util.HashMap<>(); c.getProperties(p, x -> true); assertEquals(Property.MASTER_CLIENTPORT.getDefaultValue(), p.get(Property.MASTER_CLIENTPORT.getKey())); + assertFalse(p.containsKey(Property.MASTER_PREFIX.getKey())); + assertTrue(p.containsKey(Property.TSERV_DEFAULT_BLOCKSIZE.getKey())); } @Test http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java index 79f2f21..a80183c 100644 --- a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java +++ b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java @@ -23,10 +23,12 @@ import static org.junit.Assert.assertTrue; import java.io.File; import java.util.HashSet; -import java.util.Iterator; import java.util.Map.Entry; import java.util.TreeMap; -import java.util.TreeSet; +import java.util.function.Predicate; +import java.util.stream.Collector; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.junit.Test; @@ -84,7 +86,7 @@ public class PropertyTest { @Test public void testRawDefaultValues() { - AccumuloConfiguration conf = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration conf = DefaultConfiguration.getInstance(); assertEquals("${java.io.tmpdir}" + File.separator + "accumulo-vfs-cache-${user.name}", Property.VFS_CLASSLOADER_CACHE_DIR.getRawDefaultValue()); assertEquals(new File(System.getProperty("java.io.tmpdir"), "accumulo-vfs-cache-" + System.getProperty("user.name")).getAbsolutePath(), conf.get(Property.VFS_CLASSLOADER_CACHE_DIR)); @@ -96,49 +98,26 @@ public class PropertyTest { assertEquals("", Property.GENERAL_MAVEN_PROJECT_BASEDIR.getDefaultValue()); } + // This test verifies all "sensitive" properties are properly marked as sensitive @Test public void testSensitiveKeys() { - final TreeMap extras = new TreeMap<>(); - extras.put("trace.token.property.blah", "something"); - - AccumuloConfiguration conf = new DefaultConfiguration() { - @Override - public Iterator> iterator() { - final Iterator> parent = super.iterator(); - final Iterator> mine = extras.entrySet().iterator(); - - return new Iterator>() { - - @Override - public boolean hasNext() { - return parent.hasNext() || mine.hasNext(); - } - - @Override - public Entry next() { - return parent.hasNext() ? parent.next() : mine.next(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - }; - TreeSet expected = new TreeSet<>(); - for (Entry entry : conf) { - String key = entry.getKey(); - if (key.equals(Property.INSTANCE_SECRET.getKey()) || key.toLowerCase().contains("password") || key.toLowerCase().endsWith("secret") - || key.startsWith(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey())) - expected.add(key); - } - TreeSet actual = new TreeSet<>(); - for (Entry entry : conf) { - String key = entry.getKey(); - if (Property.isSensitive(key)) - actual.add(key); - } + // add trace token, because it's a sensitive property not in the default configuration + ConfigurationCopy conf = new ConfigurationCopy(DefaultConfiguration.getInstance()); + conf.set("trace.token.property.blah", "something"); + + // ignores duplicates because ConfigurationCopy already de-duplicates + Collector,?,TreeMap> treeMapCollector = Collectors.toMap(e -> e.getKey(), e -> e.getValue(), (a, b) -> a, TreeMap::new); + + Predicate> sensitiveNames = e -> e.getKey().equals(Property.INSTANCE_SECRET.getKey()) || e.getKey().toLowerCase().contains("password") + || e.getKey().toLowerCase().endsWith("secret") || e.getKey().startsWith(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey()); + + Predicate> isMarkedSensitive = e -> Property.isSensitive(e.getKey()); + + TreeMap expected = StreamSupport.stream(conf.spliterator(), false).filter(sensitiveNames).collect(treeMapCollector); + TreeMap actual = StreamSupport.stream(conf.spliterator(), false).filter(isMarkedSensitive).collect(treeMapCollector); + + // make sure trace token property wasn't excluded from both + assertEquals("something", expected.get("trace.token.property.blah")); assertEquals(expected, actual); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java b/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java index 065438c..4547cd4 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java @@ -25,8 +25,8 @@ import java.util.Collections; import java.util.HashSet; import java.util.Random; -import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; @@ -67,7 +67,7 @@ public class BloomFilterLayerLookupTest { ArrayList vals = new ArrayList<>(valsSet); Collections.sort(vals); - ConfigurationCopy acuconf = new ConfigurationCopy(AccumuloConfiguration.getDefaultConfiguration()); + ConfigurationCopy acuconf = new ConfigurationCopy(DefaultConfiguration.getInstance()); acuconf.set(Property.TABLE_BLOOM_ENABLED, "true"); acuconf.set(Property.TABLE_BLOOM_KEY_FUNCTOR, ColumnFamilyFunctor.class.getName()); acuconf.set(Property.TABLE_FILE_TYPE, RFile.EXTENSION); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java b/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java index a8e4b7f..fb05e67 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.IOException; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.file.rfile.RFile; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; @@ -50,7 +51,7 @@ public class FileOperationsTest { FileOperations fileOperations = FileOperations.getInstance(); Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf); - AccumuloConfiguration acuconf = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration acuconf = DefaultConfiguration.getInstance(); writer = fileOperations.newWriterBuilder().forFile(filename, fs, conf).withTableConfiguration(acuconf).build(); writer.close(); } catch (Exception ex) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java index 34d01ad..cf10172 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java @@ -19,7 +19,7 @@ package org.apache.accumulo.core.file.rfile; import java.util.HashSet; import java.util.Set; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; @@ -56,7 +56,7 @@ public class CreateCompatTestFile { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); - CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(fs, new Path(args[0]), "gz", null, conf, AccumuloConfiguration.getDefaultConfiguration()); + CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(fs, new Path(args[0]), "gz", null, conf, DefaultConfiguration.getInstance()); RFile.Writer writer = new RFile.Writer(_cbw, 1000); writer.startNewLocalityGroup("lg1", newColFamSequence(formatStr("cf_", 1), formatStr("cf_", 2))); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java index 391bea1..1585baf 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java @@ -22,6 +22,7 @@ import java.util.Random; import junit.framework.TestCase; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.file.blockfile.ABlockWriter; import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile; @@ -52,7 +53,7 @@ public class MultiLevelIndexTest extends TestCase { } private void runTest(int maxBlockSize, int num) throws IOException { - AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration aconf = DefaultConfiguration.getInstance(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); FSDataOutputStream dos = new FSDataOutputStream(baos, new FileSystem.Statistics("a")); CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(PositionedOutputs.wrap(dos), "gz", CachedConfiguration.getInstance(), aconf); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java index ba255bf..bf747f6 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java @@ -33,6 +33,7 @@ import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.sample.Sampler; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; @@ -110,7 +111,7 @@ public class MultiThreadedRFileTest { public TestRFile(AccumuloConfiguration accumuloConfiguration) { this.accumuloConfiguration = accumuloConfiguration; if (this.accumuloConfiguration == null) - this.accumuloConfiguration = AccumuloConfiguration.getDefaultConfiguration(); + this.accumuloConfiguration = DefaultConfiguration.getInstance(); } public void close() throws IOException { @@ -174,7 +175,7 @@ public class MultiThreadedRFileTest { Path path = new Path("file://" + rfile.toString()); // the caches used to obfuscate the multithreaded issues - CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, AccumuloConfiguration.getDefaultConfiguration()); + CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, DefaultConfiguration.getInstance()); reader = new RFile.Reader(_cbr); iter = new ColumnFamilySkippingIterator(reader);