accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mwa...@apache.org
Subject [accumulo] branch master updated: ACCUMULO-4706 Updates to support Accumulo docker image (#316)
Date Wed, 01 Nov 2017 20:27:20 GMT
This is an automated email from the ASF dual-hosted git repository.

mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/master by this push:
     new cd5b73b  ACCUMULO-4706 Updates to support Accumulo docker image (#316)
cd5b73b is described below

commit cd5b73b6def3be480934e6fc33c1895eed4367b6
Author: Mike Walch <mwalch@apache.org>
AuthorDate: Wed Nov 1 16:27:18 2017 -0400

    ACCUMULO-4706 Updates to support Accumulo docker image (#316)
    
    * Added -o option to Accumulo services to override configuration
    * Added --upload-accumulo-site option to 'accumulo init' command
    * Tserver replication service now supports port search
---
 .../accumulo/core/conf/CliConfiguration.java       | 63 ++++++++++++++++++++++
 .../accumulo/core/conf/SiteConfiguration.java      | 19 +++++--
 .../accumulo/core/conf/CliConfigurationTest.java   | 50 +++++++++++++++++
 .../accumulo/core/conf/SiteConfigurationTest.java  | 11 ++++
 .../org/apache/accumulo/server/ServerOpts.java     | 53 +++++++++++++++++-
 .../apache/accumulo/server/init/Initialize.java    | 26 +++++++++
 .../org/apache/accumulo/server/ServerOptsTest.java | 11 ++++
 .../apache/accumulo/gc/SimpleGarbageCollector.java |  6 +--
 .../java/org/apache/accumulo/master/Master.java    |  3 +-
 .../java/org/apache/accumulo/monitor/Monitor.java  |  3 +-
 .../org/apache/accumulo/tracer/TraceServer.java    |  2 +-
 .../org/apache/accumulo/tserver/TabletServer.java  |  5 +-
 12 files changed, 237 insertions(+), 15 deletions(-)

diff --git a/core/src/main/java/org/apache/accumulo/core/conf/CliConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/CliConfiguration.java
new file mode 100644
index 0000000..7be1053
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/conf/CliConfiguration.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.conf;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Predicate;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CliConfiguration {
+
+  private static final Logger log = LoggerFactory.getLogger(CliConfiguration.class);
+  private static volatile Map<String,String> config = new HashMap<>();
+
+  /**
+   * Sets CliConfiguration with map of configuration. Additional calls will overwrite existing
properties and values.
+   *
+   * @param conf
+   *          Map of configuration
+   */
+  public static void set(Map<String,String> conf) {
+    Objects.requireNonNull(conf);
+    config = conf;
+  }
+
+  public static void print() {
+    log.info("The following configuration was set on the command line:");
+    for (Map.Entry<String,String> entry : config.entrySet()) {
+      String key = entry.getKey();
+      log.info(key + " = " + (Property.isSensitive(key) ? "<hidden>" : entry.getValue()));
+    }
+  }
+
+  public static String get(Property property) {
+    return config.get(property.getKey());
+  }
+
+  public static void getProperties(Map<String,String> props, Predicate<String>
filter) {
+    for (Map.Entry<String,String> entry : config.entrySet()) {
+      if (filter.test(entry.getKey())) {
+        props.put(entry.getKey(), entry.getValue());
+      }
+    }
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
index cd35ad8..4135de0 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
@@ -34,8 +34,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * An {@link AccumuloConfiguration} which loads properties from an XML file, usually accumulo-site.xml.
This implementation supports defaulting undefined
- * property values to a parent configuration's definitions.
+ * An {@link AccumuloConfiguration} which first loads any properties set on the command-line
(using the -o option) and then from an XML file, usually
+ * accumulo-site.xml. This implementation supports defaulting undefined property values to
a parent configuration's definitions.
  * <p>
  * The system property "accumulo.configuration" can be used to specify the location of the
XML configuration file on the classpath or filesystem if the path is
  * prefixed with 'file://'. If the system property is not defined, it defaults to "accumulo-site.xml"
and will look on classpath for file.
@@ -119,8 +119,11 @@ public class SiteConfiguration extends AccumuloConfiguration {
 
   @Override
   public String get(Property property) {
-    String key = property.getKey();
+    if (CliConfiguration.get(property) != null) {
+      return CliConfiguration.get(property);
+    }
 
+    String key = property.getKey();
     // If the property is sensitive, see if CredentialProvider was configured.
     if (property.isSensitive()) {
       Configuration hadoopConf = getHadoopConfiguration();
@@ -145,12 +148,19 @@ public class SiteConfiguration extends AccumuloConfiguration {
         log.error("Using default value for {} due to improperly formatted {}: {}", key, property.getType(),
value);
       value = parent.get(property);
     }
+
     return value;
   }
 
   @Override
   public void getProperties(Map<String,String> props, Predicate<String> filter)
{
-    parent.getProperties(props, filter);
+    getProperties(props, filter, true);
+  }
+
+  public void getProperties(Map<String,String> props, Predicate<String> filter,
boolean useDefaults) {
+    if (useDefaults) {
+      parent.getProperties(props, filter);
+    }
 
     for (Entry<String,String> entry : getXmlConfig())
       if (filter.test(entry.getKey()))
@@ -176,6 +186,7 @@ public class SiteConfiguration extends AccumuloConfiguration {
         log.warn("Failed to extract sensitive properties from Hadoop CredentialProvider,
falling back to accumulo-site.xml", e);
       }
     }
+    CliConfiguration.getProperties(props, filter);
   }
 
   protected Configuration getHadoopConfiguration() {
diff --git a/core/src/test/java/org/apache/accumulo/core/conf/CliConfigurationTest.java b/core/src/test/java/org/apache/accumulo/core/conf/CliConfigurationTest.java
new file mode 100644
index 0000000..e7f76dc
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/conf/CliConfigurationTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.conf;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableMap;
+
+public class CliConfigurationTest {
+
+  @Test
+  public void testBasic() {
+    try {
+      CliConfiguration.set(null);
+      Assert.fail();
+    } catch (NullPointerException e) {
+      // expected
+    }
+    CliConfiguration.set(new HashMap<>());
+
+    Map<String,String> expected = new HashMap<>();
+    expected.put(Property.TRACE_USER.getKey(), "test");
+    expected.put(Property.TSERV_CLIENTPORT.getKey(), "123");
+    CliConfiguration.set(expected);
+
+    Assert.assertEquals("test", CliConfiguration.get(Property.TRACE_USER));
+
+    Map<String,String> results = new HashMap<>();
+    CliConfiguration.getProperties(results, p -> p.startsWith("trace"));
+    Assert.assertEquals(ImmutableMap.of(Property.TRACE_USER.getKey(), "test"), results);
+  }
+}
diff --git a/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java b/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
index 92a0da4..ee51d28 100644
--- a/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
@@ -28,6 +28,8 @@ import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import com.google.common.collect.ImmutableMap;
+
 public class SiteConfigurationTest {
   private static boolean isCredentialProviderAvailable;
 
@@ -71,4 +73,13 @@ public class SiteConfigurationTest {
     Assert.assertEquals(null, props.get("ignored.property"));
     Assert.assertEquals(Property.GENERAL_RPC_TIMEOUT.getDefaultValue(), props.get(Property.GENERAL_RPC_TIMEOUT.getKey()));
   }
+
+  @Test
+  public void testCliConfig() {
+    SiteConfiguration conf = SiteConfiguration.getInstance();
+    Assert.assertEquals("localhost:2181", conf.get(Property.INSTANCE_ZK_HOST));
+
+    CliConfiguration.set(ImmutableMap.of(Property.INSTANCE_ZK_HOST.getKey(), "myhost:2181"));
+    Assert.assertEquals("myhost:2181", conf.get(Property.INSTANCE_ZK_HOST));
+  }
 }
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerOpts.java b/server/base/src/main/java/org/apache/accumulo/server/ServerOpts.java
index bbe0dd2..b173ae3 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerOpts.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerOpts.java
@@ -16,17 +16,68 @@
  */
 package org.apache.accumulo.server;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.conf.CliConfiguration;
 
 import com.beust.jcommander.Parameter;
+import com.beust.jcommander.converters.IParameterSplitter;
 
 public class ServerOpts extends Help {
   @Parameter(names = {"-a", "--address"}, description = "address to bind to")
-  String address = null;
+  private String address = null;
+
+  public static class NullSplitter implements IParameterSplitter {
+    @Override
+    public List<String> split(String value) {
+      return Collections.singletonList(value);
+    }
+  }
+
+  @Parameter(names = "-o", splitter = NullSplitter.class,
+      description = "Overrides configuration set in accumulo-site.xml (but NOT system-wide
config set in Zookeeper). Expected format: -o <key>=<value>")
+  private List<String> properties = new ArrayList<>();
 
   public String getAddress() {
     if (address != null)
       return address;
     return "0.0.0.0";
   }
+
+  public List<String> getProperties() {
+    return properties;
+  }
+
+  public Map<String,String> getConfig() {
+    Map<String,String> config = new HashMap<>();
+    for (String prop : getProperties()) {
+      String[] propArgs = prop.split("=", 2);
+      if (propArgs.length == 2) {
+        String key = propArgs[0].trim();
+        String value = propArgs[1].trim();
+        if (key.isEmpty() || value.isEmpty()) {
+          throw new IllegalArgumentException("Invalid command line -o option: " + prop);
+        } else {
+          config.put(key, value);
+        }
+      } else {
+        throw new IllegalArgumentException("Invalid command line -o option: " + prop);
+      }
+    }
+    return config;
+  }
+
+  @Override
+  public void parseArgs(String programName, String[] args, Object... others) {
+    super.parseArgs(programName, args, others);
+    CliConfiguration.set(getConfig());
+    if (getConfig().size() > 0) {
+      CliConfiguration.print();
+    }
+  }
 }
diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index be3869b..8198418 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -29,6 +29,7 @@ import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeMap;
@@ -100,6 +101,7 @@ import org.apache.accumulo.server.security.SecurityUtil;
 import org.apache.accumulo.server.tables.TableManager;
 import org.apache.accumulo.server.tablets.TabletTime;
 import org.apache.accumulo.server.util.ReplicationTableUtil;
+import org.apache.accumulo.server.util.SystemPropUtil;
 import org.apache.accumulo.server.util.TablePropUtil;
 import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 import org.apache.accumulo.start.spi.KeywordExecutable;
@@ -393,6 +395,28 @@ public class Initialize implements KeywordExecutable {
       log.error("FATAL: Failed to initialize security", e);
       return false;
     }
+
+    if (opts.uploadAccumuloSite) {
+      try {
+        log.info("Uploading properties in accumulo-site.xml to Zookeeper. Properties that
cannot be set in Zookeeper will be skipped:");
+        Map<String,String> entries = new TreeMap<>();
+        SiteConfiguration.getInstance().getProperties(entries, x -> true, false);
+        for (Map.Entry<String,String> entry : entries.entrySet()) {
+          String key = entry.getKey();
+          String value = entry.getValue();
+          if (Property.isValidZooPropertyKey(key)) {
+            SystemPropUtil.setSystemProperty(key, value);
+            log.info("Uploaded - {} = {}", key, Property.isSensitive(key) ? "<hidden>"
: value);
+          } else {
+            log.info("Skipped - {} = {}", key, Property.isSensitive(key) ? "<hidden>"
: value);
+          }
+        }
+      } catch (Exception e) {
+        log.error("FATAL: Failed to upload accumulo-site.xml to Zookeeper", e);
+        return false;
+      }
+    }
+
     return true;
   }
 
@@ -760,6 +784,8 @@ public class Initialize implements KeywordExecutable {
     boolean forceResetSecurity = false;
     @Parameter(names = "--clear-instance-name", description = "delete any existing instance
name without prompting")
     boolean clearInstanceName = false;
+    @Parameter(names = "--upload-accumulo-site", description = "Uploads properties in accumulo-site.xml
to Zookeeper")
+    boolean uploadAccumuloSite = false;
     @Parameter(names = "--instance-name", description = "the instance name, if not provided,
will prompt")
     String cliInstanceName;
     @Parameter(names = "--password", description = "set the password on the command line")
diff --git a/server/base/src/test/java/org/apache/accumulo/server/ServerOptsTest.java b/server/base/src/test/java/org/apache/accumulo/server/ServerOptsTest.java
index 2c6889b..fbd8e14 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/ServerOptsTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/ServerOptsTest.java
@@ -18,6 +18,9 @@ package org.apache.accumulo.server;
 
 import static org.junit.Assert.assertEquals;
 
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -40,4 +43,12 @@ public class ServerOptsTest {
     opts.parseArgs(ServerOptsTest.class.getName(), new String[] {});
     assertEquals("0.0.0.0", opts.getAddress());
   }
+
+  @Test
+  public void testOverrideConfig() {
+    SiteConfiguration siteConf = SiteConfiguration.getInstance();
+    Assert.assertEquals("localhost:2181", siteConf.get(Property.INSTANCE_ZK_HOST));
+    opts.parseArgs(ServerOptsTest.class.getName(), new String[] {"-o", "instance.zookeeper.host=test:123"});
+    Assert.assertEquals("test:123", siteConf.get(Property.INSTANCE_ZK_HOST));
+  }
 }
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index ba23505..74ef6c9 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -144,8 +144,10 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements
Ifa
 
   private GCStatus status = new GCStatus(new GcCycleStats(), new GcCycleStats(), new GcCycleStats(),
new GcCycleStats());
 
-  public static void main(String[] args) throws UnknownHostException, IOException {
+  public static void main(String[] args) throws IOException {
     final String app = "gc";
+    Opts opts = new Opts();
+    opts.parseArgs(app, args);
     SecurityUtil.serverLogin(SiteConfiguration.getInstance());
     Instance instance = HdfsZooInstance.getInstance();
     ServerConfigurationFactory conf = new ServerConfigurationFactory(instance);
@@ -153,8 +155,6 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements
Ifa
     log.info("Instance " + instance.getInstanceID());
     final VolumeManager fs = VolumeManagerImpl.get();
     Accumulo.init(fs, instance, conf, app);
-    Opts opts = new Opts();
-    opts.parseArgs(app, args);
     SimpleGarbageCollector gc = new SimpleGarbageCollector(opts, instance, fs, conf);
 
     DistributedTrace.enable(opts.getAddress(), app, conf.getSystemConfiguration());
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index 8660751..1e72e4b 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -1438,10 +1438,9 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List
   public static void main(String[] args) throws Exception {
     try {
       final String app = "master";
-      SecurityUtil.serverLogin(SiteConfiguration.getInstance());
-
       ServerOpts opts = new ServerOpts();
       opts.parseArgs(app, args);
+      SecurityUtil.serverLogin(SiteConfiguration.getInstance());
       String hostname = opts.getAddress();
       Instance instance = HdfsZooInstance.getInstance();
       ServerConfigurationFactory conf = new ServerConfigurationFactory(instance);
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index b1d6db4..68317b3 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -418,11 +418,10 @@ public class Monitor implements HighlyAvailableService {
 
   public static void main(String[] args) throws Exception {
     final String app = "monitor";
-    SecurityUtil.serverLogin(SiteConfiguration.getInstance());
-
     ServerOpts opts = new ServerOpts();
     opts.parseArgs(app, args);
     String hostname = opts.getAddress();
+    SecurityUtil.serverLogin(SiteConfiguration.getInstance());
 
     VolumeManager fs = VolumeManagerImpl.get();
     instance = HdfsZooInstance.getInstance();
diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index c3e6d5f..16d72db 100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@ -379,9 +379,9 @@ public class TraceServer implements Watcher {
 
   public static void main(String[] args) throws Exception {
     final String app = "tracer";
-    loginTracer(SiteConfiguration.getInstance());
     ServerOpts opts = new ServerOpts();
     opts.parseArgs(app, args);
+    loginTracer(SiteConfiguration.getInstance());
     Instance instance = HdfsZooInstance.getInstance();
     ServerConfigurationFactory conf = new ServerConfigurationFactory(instance);
     VolumeManager fs = VolumeManagerImpl.get();
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 5e24e59..6bdcfd0 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -2441,7 +2441,8 @@ public class TabletServer extends AccumuloServerContext implements Runnable
{
     AccumuloConfiguration conf = getServerConfigurationFactory().getSystemConfiguration();
     Property maxMessageSizeProperty = (conf.get(Property.TSERV_MAX_MESSAGE_SIZE) != null
? Property.TSERV_MAX_MESSAGE_SIZE : Property.GENERAL_MAX_MESSAGE_SIZE);
     ServerAddress sp = TServerUtils.startServer(this, clientAddress.getHost(), Property.REPLICATION_RECEIPT_SERVICE_PORT,
processor,
-        "ReplicationServicerHandler", "Replication Servicer", null, Property.REPLICATION_MIN_THREADS,
Property.REPLICATION_THREADCHECK, maxMessageSizeProperty);
+        "ReplicationServicerHandler", "Replication Servicer", Property.TSERV_PORTSEARCH,
Property.REPLICATION_MIN_THREADS, Property.REPLICATION_THREADCHECK,
+        maxMessageSizeProperty);
     this.replServer = sp.server;
     log.info("Started replication service on {}", sp.address);
 
@@ -3061,9 +3062,9 @@ public class TabletServer extends AccumuloServerContext implements Runnable
{
   public static void main(String[] args) throws IOException {
     try {
       final String app = "tserver";
-      SecurityUtil.serverLogin(SiteConfiguration.getInstance());
       ServerOpts opts = new ServerOpts();
       opts.parseArgs(app, args);
+      SecurityUtil.serverLogin(SiteConfiguration.getInstance());
       String hostname = opts.getAddress();
       Instance instance = HdfsZooInstance.getInstance();
       ServerConfigurationFactory conf = new ServerConfigurationFactory(instance);

-- 
To stop receiving notification emails like this one, please contact
['"commits@accumulo.apache.org" <commits@accumulo.apache.org>'].

Mime
View raw message