Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E7A5911D4B for ; Fri, 19 Sep 2014 18:59:59 +0000 (UTC) Received: (qmail 85136 invoked by uid 500); 19 Sep 2014 18:59:59 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 85106 invoked by uid 500); 19 Sep 2014 18:59:59 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 85095 invoked by uid 99); 19 Sep 2014 18:59:59 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 19 Sep 2014 18:59:59 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 7B5F0A1EF8F; Fri, 19 Sep 2014 18:59:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@accumulo.apache.org Date: Fri, 19 Sep 2014 19:00:05 -0000 Message-Id: <949787abc9814303ba1c7d1684aaa989@git.apache.org> In-Reply-To: <0678d329b07c4c06b46b1bbef9511d67@git.apache.org> References: <0678d329b07c4c06b46b1bbef9511d67@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [08/15] git commit: Merge branch '1.5.3-SNAPSHOT' into 1.6.1-SNAPSHOT Merge branch '1.5.3-SNAPSHOT' into 1.6.1-SNAPSHOT Conflicts: assemble/pom.xml core/pom.xml examples/pom.xml examples/simple/pom.xml fate/pom.xml minicluster/pom.xml minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java pom.xml proxy/pom.xml server/tserver/pom.xml start/pom.xml test/pom.xml trace/pom.xml Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/51ec5a16 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/51ec5a16 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/51ec5a16 Branch: refs/heads/1.6.1-SNAPSHOT Commit: 51ec5a16d4deaa9be426497ebe4b8c673df454ae Parents: 5878a77 72fd6be Author: Josh Elser Authored: Fri Sep 19 14:07:21 2014 -0400 Committer: Josh Elser Committed: Fri Sep 19 14:07:21 2014 -0400 ---------------------------------------------------------------------- .../impl/MiniAccumuloConfigImpl.java | 64 ++++++++++---------- .../minicluster/MiniAccumuloClusterTest.java | 15 +++++ pom.xml | 2 +- 3 files changed, 49 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/51ec5a16/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloConfigImpl.java ---------------------------------------------------------------------- diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloConfigImpl.java index 4878967,0000000..e9ad045 mode 100644,000000..100644 --- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloConfigImpl.java +++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloConfigImpl.java @@@ -1,568 -1,0 +1,570 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.minicluster.impl; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.accumulo.cluster.AccumuloConfig; +import org.apache.accumulo.core.conf.CredentialProviderFactoryShim; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.minicluster.MemoryUnit; +import org.apache.accumulo.minicluster.ServerType; +import org.apache.accumulo.server.util.PortUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.Logger; + +/** + * Holds configuration for {@link MiniAccumuloClusterImpl}. Required configurations must be passed to constructor(s) and all other configurations are optional. - * ++ * + * @since 1.6.0 + */ +public class MiniAccumuloConfigImpl implements AccumuloConfig { + private static final Logger log = Logger.getLogger(MiniAccumuloConfigImpl.class); + private static final String DEFAULT_INSTANCE_SECRET = "DONTTELL"; + + private File dir = null; + private String rootPassword = null; + private Map siteConfig = new HashMap(); + private Map configuredSiteConig = new HashMap(); + private int numTservers = 2; + private Map memoryConfig = new HashMap(); + private boolean jdwpEnabled = false; + private Map systemProperties = new HashMap(); + + private String instanceName = "miniInstance"; + + private File libDir; + private File libExtDir; + private File confDir; + private File zooKeeperDir; + private File accumuloDir; + private File logDir; + private File walogDir; + + private int zooKeeperPort = 0; + private int configuredZooKeeperPort = 0; + private long zooKeeperStartupTime = 20*1000; + + private long defaultMemorySize = 128 * 1024 * 1024; + + private boolean initialized = false; + + private boolean useMiniDFS = false; + + private boolean useCredentialProvider = false; + + private String[] classpathItems = null; + + private String[] nativePathItems = null; + + /** + * @param dir + * An empty or nonexistant directory that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava, and + * Junit provide methods for creating temporary directories. + * @param rootPassword + * The initial password for the Accumulo root user + */ + public MiniAccumuloConfigImpl(File dir, String rootPassword) { + this.dir = dir; + this.rootPassword = rootPassword; + } + + /** + * Set directories and fully populate site config + */ + MiniAccumuloConfigImpl initialize() { + + // Sanity checks + if (this.getDir().exists() && !this.getDir().isDirectory()) + throw new IllegalArgumentException("Must pass in directory, " + this.getDir() + " is a file"); + + if (this.getDir().exists() && this.getDir().list().length != 0) + throw new IllegalArgumentException("Directory " + this.getDir() + " is not empty"); + + if (!initialized) { + libDir = new File(dir, "lib"); + libExtDir = new File(libDir, "ext"); + confDir = new File(dir, "conf"); + accumuloDir = new File(dir, "accumulo"); + zooKeeperDir = new File(dir, "zookeeper"); + logDir = new File(dir, "logs"); + walogDir = new File(dir, "walogs"); + + // TODO ACCUMULO-XXXX replace usage of instance.dfs.{dir,uri} with instance.volumes + setInstanceLocation(); + + mergeProp(Property.INSTANCE_SECRET.getKey(), DEFAULT_INSTANCE_SECRET); + mergeProp(Property.TSERV_PORTSEARCH.getKey(), "true"); + mergeProp(Property.LOGGER_DIR.getKey(), walogDir.getAbsolutePath()); + mergeProp(Property.TSERV_DATACACHE_SIZE.getKey(), "10M"); + mergeProp(Property.TSERV_INDEXCACHE_SIZE.getKey(), "10M"); + mergeProp(Property.TSERV_MAXMEM.getKey(), "50M"); + mergeProp(Property.TSERV_WALOG_MAX_SIZE.getKey(), "100M"); + mergeProp(Property.TSERV_NATIVEMAP_ENABLED.getKey(), "false"); + mergeProp(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + "password", getRootPassword()); + // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5 + mergeProp(Property.TSERV_MAJC_DELAY.getKey(), "3"); + mergeProp(Property.GENERAL_CLASSPATHS.getKey(), libDir.getAbsolutePath() + "/[^.].*[.]jar"); + mergeProp(Property.GENERAL_DYNAMIC_CLASSPATHS.getKey(), libExtDir.getAbsolutePath() + "/[^.].*[.]jar"); + mergeProp(Property.GC_CYCLE_DELAY.getKey(), "4s"); + mergeProp(Property.GC_CYCLE_START.getKey(), "0s"); + mergePropWithRandomPort(Property.MASTER_CLIENTPORT.getKey()); + mergePropWithRandomPort(Property.TRACE_PORT.getKey()); + mergePropWithRandomPort(Property.TSERV_CLIENTPORT.getKey()); + mergePropWithRandomPort(Property.MONITOR_PORT.getKey()); + mergePropWithRandomPort(Property.GC_PORT.getKey()); ++ mergePropWithRandomPort(Property.MONITOR_LOG4J_PORT.getKey()); + + if (isUseCredentialProvider()) { + updateConfigForCredentialProvider(); + } + + // zookeeper port should be set explicitly in this class, not just on the site config + if (zooKeeperPort == 0) + zooKeeperPort = PortUtils.getRandomFreePort(); + siteConfig.put(Property.INSTANCE_ZK_HOST.getKey(), "localhost:" + zooKeeperPort); + initialized = true; + } + return this; + } + + private void updateConfigForCredentialProvider() { + String cpPaths = siteConfig.get(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey()); + if (null != cpPaths && !Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getDefaultValue().equals(cpPaths)) { + // Already configured + return; + } + + if (!CredentialProviderFactoryShim.isHadoopCredentialProviderAvailable()) { + throw new RuntimeException("Cannot use CredentialProvider when implementation is not available. Be sure to use >=Hadoop-2.6.0"); + } + + File keystoreFile = new File(getConfDir(), "credential-provider.jks"); + String keystoreUri = "jceks://file" + keystoreFile.getAbsolutePath(); + Configuration conf = CredentialProviderFactoryShim.getConfiguration(keystoreUri); + + // Set the URI on the siteCfg + siteConfig.put(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey(), keystoreUri); + + Iterator> entries = siteConfig.entrySet().iterator(); + while (entries.hasNext()) { + Entry entry = entries.next(); + + // Not a @Sensitive Property, ignore it + if (!Property.isSensitive(entry.getKey())) { + continue; + } + + // Add the @Sensitive Property to the CredentialProvider + try { + CredentialProviderFactoryShim.createEntry(conf, entry.getKey(), entry.getValue().toCharArray()); + } catch (IOException e) { + log.warn("Attempted to add " + entry.getKey() + " to CredentialProvider but failed", e); + continue; + } + + // Only remove it from the siteCfg if we succeeded in adding it to the CredentialProvider + entries.remove(); + } + } + + @SuppressWarnings("deprecation") + private void setInstanceLocation() { + mergeProp(Property.INSTANCE_DFS_URI.getKey(), "file:///"); + mergeProp(Property.INSTANCE_DFS_DIR.getKey(), accumuloDir.getAbsolutePath()); + } + + /** + * Set a given key/value on the site config if it doesn't already exist + */ + private void mergeProp(String key, String value) { + if (!siteConfig.containsKey(key)) { + siteConfig.put(key, value); + } + } + + /** + * Sets a given key with a random port for the value on the site config if it doesn't already exist. + */ + private void mergePropWithRandomPort(String key) { + if (!siteConfig.containsKey(key)) { + siteConfig.put(key, "0"); + } + } + + /** + * Calling this method is optional. If not set, it defaults to two. - * ++ * + * @param numTservers + * the number of tablet servers that mini accumulo cluster should start + */ + @Override + public MiniAccumuloConfigImpl setNumTservers(int numTservers) { + if (numTservers < 1) + throw new IllegalArgumentException("Must have at least one tablet server"); + this.numTservers = numTservers; + return this; + } + + /** + * Calling this method is optional. If not set, defaults to 'miniInstance' - * ++ * + * @since 1.6.0 + */ + @Override + public MiniAccumuloConfigImpl setInstanceName(String instanceName) { + this.instanceName = instanceName; + return this; + } + + /** + * Calling this method is optional. If not set, it defaults to an empty map. - * ++ * + * @param siteConfig + * key/values that you normally put in accumulo-site.xml can be put here. + */ + @Override + public MiniAccumuloConfigImpl setSiteConfig(Map siteConfig) { + this.siteConfig = new HashMap(siteConfig); + this.configuredSiteConig = new HashMap(siteConfig); + return this; + } + + /** + * Calling this method is optional. A random port is generated by default - * ++ * + * @param zooKeeperPort + * A valid (and unused) port to use for the zookeeper - * ++ * + * @since 1.6.0 + */ + @Override + public MiniAccumuloConfigImpl setZooKeeperPort(int zooKeeperPort) { + this.configuredZooKeeperPort = zooKeeperPort; + this.zooKeeperPort = zooKeeperPort; + return this; + } + + /** + * Configure the time to wait for ZooKeeper to startup. + * Calling this method is optional. The default is 20000 milliseconds - * ++ * + * @param zooKeeperStartupTime + * Time to wait for ZooKeeper to startup, in milliseconds - * ++ * + * @since 1.6.1 + */ + @Override + public MiniAccumuloConfigImpl setZooKeeperStartupTime(long zooKeeperStartupTime) { + this.zooKeeperStartupTime = zooKeeperStartupTime; + return this; + } + + /** + * Sets the amount of memory to use in the master process. Calling this method is optional. Default memory is 128M - * ++ * + * @param serverType + * the type of server to apply the memory settings + * @param memory + * amount of memory to set - * ++ * + * @param memoryUnit + * the units for which to apply with the memory size - * ++ * + * @since 1.6.0 + */ + @Override + public MiniAccumuloConfigImpl setMemory(ServerType serverType, long memory, MemoryUnit memoryUnit) { + this.memoryConfig.put(serverType, memoryUnit.toBytes(memory)); + return this; + } + + /** + * Sets the default memory size to use. This value is also used when a ServerType has not been configured explicitly. Calling this method is optional. Default + * memory is 128M - * ++ * + * @param memory + * amount of memory to set - * ++ * + * @param memoryUnit + * the units for which to apply with the memory size - * ++ * + * @since 1.6.0 + */ + @Override + public MiniAccumuloConfigImpl setDefaultMemory(long memory, MemoryUnit memoryUnit) { + this.defaultMemorySize = memoryUnit.toBytes(memory); + return this; + } + + /** + * @return a copy of the site config + */ + @Override + public Map getSiteConfig() { + return new HashMap(siteConfig); + } + + public Map getConfiguredSiteConfig() { + return new HashMap(configuredSiteConig); + } + + /** + * @return name of configured instance - * ++ * + * @since 1.6.0 + */ + @Override + public String getInstanceName() { + return instanceName; + } + + /** + * @return The configured zookeeper port - * ++ * + * @since 1.6.0 + */ ++ @Override + public int getZooKeeperPort() { + return zooKeeperPort; + } + + public int getConfiguredZooKeeperPort() { + return configuredZooKeeperPort; + } + + public long getZooKeeperStartupTime() { + return zooKeeperStartupTime; + } + + File getLibDir() { + return libDir; + } + + File getLibExtDir() { + return libExtDir; + } + + public File getConfDir() { + return confDir; + } + + File getZooKeeperDir() { + return zooKeeperDir; + } + + public File getAccumuloDir() { + return accumuloDir; + } + + public File getLogDir() { + return logDir; + } + + File getWalogDir() { + return walogDir; + } + + /** + * @param serverType + * get configuration for this server type - * ++ * + * @return memory configured in bytes, returns default if this server type is not configured - * ++ * + * @since 1.6.0 + */ + @Override + public long getMemory(ServerType serverType) { + return memoryConfig.containsKey(serverType) ? memoryConfig.get(serverType) : defaultMemorySize; + } + + /** + * @return memory configured in bytes - * ++ * + * @since 1.6.0 + */ + @Override + public long getDefaultMemory() { + return defaultMemorySize; + } + + /** + * @return zookeeper connection string - * ++ * + * @since 1.6.0 + */ + public String getZooKeepers() { + return siteConfig.get(Property.INSTANCE_ZK_HOST.getKey()); + } + + /** + * @return the base directory of the cluster configuration + */ + public File getDir() { + return dir; + } + + /** + * @return the root password of this cluster configuration + */ + @Override + public String getRootPassword() { + return rootPassword; + } + + /** + * @return the number of tservers configured for this cluster + */ + @Override + public int getNumTservers() { + return numTservers; + } + + /** + * @return is the current configuration in jdwpEnabled mode? - * ++ * + * @since 1.6.0 + */ + public boolean isJDWPEnabled() { + return jdwpEnabled; + } + + /** + * @param jdwpEnabled + * should the processes run remote jdwpEnabled servers? + * @return the current instance - * ++ * + * @since 1.6.0 + */ + public MiniAccumuloConfigImpl setJDWPEnabled(boolean jdwpEnabled) { + this.jdwpEnabled = jdwpEnabled; + return this; + } + + public boolean useMiniDFS() { + return useMiniDFS; + } + + public void useMiniDFS(boolean useMiniDFS) { + this.useMiniDFS = useMiniDFS; + } + + /** + * @return location of client conf file containing connection parameters for connecting to this minicluster - * ++ * + * @since 1.6.0 + */ + public File getClientConfFile() { + return new File(getConfDir(), "client.conf"); + } + + /** + * sets system properties set for service processes - * ++ * + * @since 1.6.0 + */ + public void setSystemProperties(Map systemProperties) { + this.systemProperties = new HashMap(systemProperties); + } + + /** + * @return a copy of the system properties for service processes - * ++ * + * @since 1.6.0 + */ + public Map getSystemProperties() { + return new HashMap(systemProperties); + } + + /** + * Gets the classpath elements to use when spawning processes. - * ++ * + * @return the classpathItems, if set - * ++ * + * @since 1.6.0 + */ + public String[] getClasspathItems() { + return classpathItems; + } + + /** + * Sets the classpath elements to use when spawning processes. - * ++ * + * @param classpathItems + * the classpathItems to set + * @since 1.6.0 + */ + public void setClasspathItems(String... classpathItems) { + this.classpathItems = classpathItems; + } + + /** + * @return the paths to use for loading native libraries - * ++ * + * @since 1.6.0 + */ + @Override + public String[] getNativeLibPaths() { + return this.nativePathItems == null ? new String[0] : this.nativePathItems; + } + + /** + * Sets the path for processes to use for loading native libraries - * ++ * + * @param nativePathItems + * the nativePathItems to set + * @since 1.6.0 + */ + @Override + public MiniAccumuloConfigImpl setNativeLibPaths(String... nativePathItems) { + this.nativePathItems = nativePathItems; + return this; + } + + /** + * Sets arbitrary configuration properties. - * ++ * + * @since 1.6.0 + */ + public void setProperty(Property p, String value) { + this.siteConfig.put(p.getKey(), value); + } + + /** + * @return the useCredentialProvider + */ + public boolean isUseCredentialProvider() { + return useCredentialProvider; + } + + /** + * @param useCredentialProvider the useCredentialProvider to set + */ + public void setUseCredentialProvider(boolean useCredentialProvider) { + this.useCredentialProvider = useCredentialProvider; + } + + @Override + public MiniAccumuloClusterImpl build() throws IOException { + return new MiniAccumuloClusterImpl(this); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/51ec5a16/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java ---------------------------------------------------------------------- diff --cc minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java index ec228af,d1d64aa..341ec8b --- a/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java +++ b/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java @@@ -40,9 -37,8 +40,10 @@@ import org.apache.accumulo.core.iterato import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.security.TablePermission; +import org.apache.accumulo.core.util.Pair; import org.apache.commons.io.FileUtils; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@@ -197,27 -175,23 +198,41 @@@ public class MiniAccumuloClusterTest conn.instanceOperations().removeProperty(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1"); conn.tableOperations().delete("table2"); + } + + @Test(timeout = 10000) + public void testDebugPorts() { + + Set> debugPorts = accumulo.getDebugPorts(); + Assert.assertEquals(5, debugPorts.size()); + for (Pair debugPort : debugPorts) { + Assert.assertTrue(debugPort.getSecond() > 0); + } + } + @Test + public void testConfig() { + // ensure what user passed in is what comes back + Assert.assertEquals(0, accumulo.getConfig().getZooKeeperPort()); + HashMap site = new HashMap(); + site.put(Property.TSERV_WORKQ_THREADS.getKey(), "2"); + Assert.assertEquals(site, accumulo.getConfig().getSiteConfig()); } + @Test + public void testRandomPorts() throws Exception { - File confDir = new File(folder.getRoot(), "conf"); ++ File confDir = new File(testDir, "conf"); + File accumuloSite = new File(confDir, "accumulo-site.xml"); + Configuration conf = new Configuration(false); + conf.addResource(accumuloSite.toURI().toURL()); + for (Property randomPortProp : new Property[] {Property.TSERV_CLIENTPORT, Property.MONITOR_PORT, Property.MONITOR_LOG4J_PORT, Property.MASTER_CLIENTPORT, + Property.TRACE_PORT, Property.GC_PORT}) { + String value = conf.get(randomPortProp.getKey()); + Assert.assertNotNull("Found no value for " + randomPortProp, value); + Assert.assertEquals("0", value); + } + } + @AfterClass public static void tearDownMiniCluster() throws Exception { accumulo.stop(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/51ec5a16/pom.xml ----------------------------------------------------------------------