flume-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bes...@apache.org
Subject flume git commit: FLUME-2993. Add support for environment variables in configuration files
Date Fri, 06 Jan 2017 18:45:39 GMT
Repository: flume
Updated Branches:
  refs/heads/trunk 2ff2dbbd1 -> 5e52ac4ad


FLUME-2993. Add support for environment variables in configuration files

Flume does not currently support environment variable interpolation in the properties file
configuration.

Enabling it helps with:
* removing security credentials from config files
* copy-pastes in configuration files when defining multiple agents

This closes #97

Reviewers: Lior Zeno, Jeff Holoman, Shang Wu

(Bessenyei Balázs Donát via Bessenyei Balázs Donát)


Project: http://git-wip-us.apache.org/repos/asf/flume/repo
Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/5e52ac4a
Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/5e52ac4a
Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/5e52ac4a

Branch: refs/heads/trunk
Commit: 5e52ac4ad18a7776f4c74499c1314fbfd3513160
Parents: 2ff2dbb
Author: Bessenyei Balázs Donát <bessbd@apache.org>
Authored: Fri Jan 6 18:32:50 2017 +0000
Committer: Bessenyei Balázs Donát <bessbd@apache.org>
Committed: Fri Jan 6 18:37:14 2017 +0000

----------------------------------------------------------------------
 flume-ng-doc/sphinx/FlumeUserGuide.rst          | 19 ++++++
 flume-ng-node/pom.xml                           |  7 ++
 .../flume/node/EnvVarResolverProperties.java    | 60 +++++++++++++++++
 .../PropertiesFileConfigurationProvider.java    | 13 +++-
 .../node/TestEnvVarResolverProperties.java      | 69 ++++++++++++++++++++
 .../flume-conf-with-envvars.properties          | 35 ++++++++++
 6 files changed, 202 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-doc/sphinx/FlumeUserGuide.rst
----------------------------------------------------------------------
diff --git a/flume-ng-doc/sphinx/FlumeUserGuide.rst b/flume-ng-doc/sphinx/FlumeUserGuide.rst
index 3c316c6..6bc9dba 100644
--- a/flume-ng-doc/sphinx/FlumeUserGuide.rst
+++ b/flume-ng-doc/sphinx/FlumeUserGuide.rst
@@ -234,6 +234,25 @@ The original Flume terminal will output the event in a log message.
 
 Congratulations - you've successfully configured and deployed a Flume agent! Subsequent sections
cover agent configuration in much more detail.
 
+Using environment variables in configuration files
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Flume has the ability to substitute environment variables in the configuration. For example::
+
+  a1.sources = r1
+  a1.sources.r1.type = netcat
+  a1.sources.r1.bind = 0.0.0.0
+  a1.sources.r1.port = ${NC_PORT}
+  a1.sources.r1.channels = c1
+
+NB: it currently works for values only, not for keys. (Ie. only on the "right side" of the
`=` mark of the config lines.)
+
+This can be enabled via Java system properties on agent invocation by setting `propertiesImplementation
= org.apache.flume.node.EnvVarResolverProperties`.
+
+For example::
+  $ NC_PORT=44444 bin/flume-ng agent --conf conf --conf-file example.conf --name a1 -Dflume.root.logger=INFO,console
-DpropertiesImplementation=org.apache.flume.node.EnvVarResolverProperties
+
+Note the above is just an example, environment variables can be configured in other ways,
including being set in `conf/flume-env.sh`.
+
 Logging raw data
 ~~~~~~~~~~~~~~~~
 

http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/pom.xml
----------------------------------------------------------------------
diff --git a/flume-ng-node/pom.xml b/flume-ng-node/pom.xml
index 6d1067d..f4fb958 100644
--- a/flume-ng-node/pom.xml
+++ b/flume-ng-node/pom.xml
@@ -145,6 +145,13 @@
     </dependency>
 
     <dependency>
+      <groupId>com.github.stefanbirkner</groupId>
+      <artifactId>system-rules</artifactId>
+      <version>1.16.0</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
       <groupId>org.codehaus.jackson</groupId>
       <artifactId>jackson-core-asl</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/src/main/java/org/apache/flume/node/EnvVarResolverProperties.java
----------------------------------------------------------------------
diff --git a/flume-ng-node/src/main/java/org/apache/flume/node/EnvVarResolverProperties.java
b/flume-ng-node/src/main/java/org/apache/flume/node/EnvVarResolverProperties.java
new file mode 100644
index 0000000..e0b0d22
--- /dev/null
+++ b/flume-ng-node/src/main/java/org/apache/flume/node/EnvVarResolverProperties.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flume.node;
+
+import com.google.common.base.Preconditions;
+
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+
+/**
+ * A class that extends the Java built-in Properties overriding
+ * {@link java.util.Properties#getProperty(String)} to allow ${ENV_VAR_NAME}-style environment
+ * variable inclusions
+ */
+public class EnvVarResolverProperties extends Properties {
+  /**
+   * @param input The input string with ${ENV_VAR_NAME}-style environment variable names
+   * @return The output string with ${ENV_VAR_NAME} replaced with their environment variable
values
+   */
+  protected static String resolveEnvVars(String input) {
+    Preconditions.checkNotNull(input);
+    // match ${ENV_VAR_NAME}
+    Pattern p = Pattern.compile("\\$\\{(\\w+)\\}");
+    Matcher m = p.matcher(input);
+    StringBuffer sb = new StringBuffer();
+    while (m.find()) {
+      String envVarName = m.group(1);
+      String envVarValue = System.getenv(envVarName);
+      m.appendReplacement(sb, null == envVarValue ? "" : envVarValue);
+    }
+    m.appendTail(sb);
+    return sb.toString();
+  }
+
+  /**
+   * @param key the property key
+   * @return the value of the property key with ${ENV_VAR_NAME}-style environment variables
replaced
+   */
+  @Override
+  public String getProperty(String key) {
+    return resolveEnvVars(super.getProperty(key));
+  }
+}

http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java
b/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java
index b428c9e..75f45db 100644
--- a/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java
+++ b/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java
@@ -171,6 +171,7 @@ public class PropertiesFileConfigurationProvider extends
 
   private static final Logger LOGGER = LoggerFactory
       .getLogger(PropertiesFileConfigurationProvider.class);
+  private static final String DEFAULT_PROPERTIES_IMPLEMENTATION = "java.util.Properties";
 
   private final File file;
 
@@ -184,12 +185,22 @@ public class PropertiesFileConfigurationProvider extends
     BufferedReader reader = null;
     try {
       reader = new BufferedReader(new FileReader(file));
-      Properties properties = new Properties();
+      String resolverClassName = System.getProperty("propertiesImplementation",
+          DEFAULT_PROPERTIES_IMPLEMENTATION);
+      Class<? extends Properties> propsclass = Class.forName(resolverClassName)
+          .asSubclass(Properties.class);
+      Properties properties = propsclass.newInstance();
       properties.load(reader);
       return new FlumeConfiguration(toMap(properties));
     } catch (IOException ex) {
       LOGGER.error("Unable to load file:" + file
           + " (I/O failure) - Exception follows.", ex);
+    } catch (ClassNotFoundException e) {
+      LOGGER.error("Configuration resolver class not found", e);
+    } catch (InstantiationException e) {
+      LOGGER.error("Instantiation exception", e);
+    } catch (IllegalAccessException e) {
+      LOGGER.error("Illegal access exception", e);
     } finally {
       if (reader != null) {
         try {

http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/src/test/java/org/apache/flume/node/TestEnvVarResolverProperties.java
----------------------------------------------------------------------
diff --git a/flume-ng-node/src/test/java/org/apache/flume/node/TestEnvVarResolverProperties.java
b/flume-ng-node/src/test/java/org/apache/flume/node/TestEnvVarResolverProperties.java
new file mode 100644
index 0000000..27aeef1
--- /dev/null
+++ b/flume-ng-node/src/test/java/org/apache/flume/node/TestEnvVarResolverProperties.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flume.node;
+
+import junit.framework.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.contrib.java.lang.system.EnvironmentVariables;
+
+import java.io.File;
+
+public class TestEnvVarResolverProperties {
+  private static final File TESTFILE = new File(
+      TestEnvVarResolverProperties.class.getClassLoader()
+          .getResource("flume-conf-with-envvars.properties").getFile());
+
+  @Rule
+  public final EnvironmentVariables environmentVariables = new EnvironmentVariables();
+  private PropertiesFileConfigurationProvider provider;
+
+  @Before
+  public void setUp() throws Exception {
+    provider = new PropertiesFileConfigurationProvider("a1", TESTFILE);
+  }
+
+  @Test
+  public void resolveEnvVar() throws Exception {
+    environmentVariables.set("VARNAME", "varvalue");
+    String resolved = EnvVarResolverProperties.resolveEnvVars("padding ${VARNAME} padding");
+    Assert.assertEquals("padding varvalue padding", resolved);
+  }
+
+  @Test
+  public void resolveEnvVars() throws Exception {
+    environmentVariables.set("VARNAME1", "varvalue1");
+    environmentVariables.set("VARNAME2", "varvalue2");
+    String resolved = EnvVarResolverProperties
+        .resolveEnvVars("padding ${VARNAME1} ${VARNAME2} padding");
+    Assert.assertEquals("padding varvalue1 varvalue2 padding", resolved);
+  }
+
+  @Test
+  public void getProperty() throws Exception {
+    String NC_PORT = "6667";
+    environmentVariables.set("NC_PORT", NC_PORT);
+    System.setProperty("propertiesImplementation",
+        "org.apache.flume.node.EnvVarResolverProperties");
+
+    Assert.assertEquals(NC_PORT, provider.getFlumeConfiguration()
+        .getConfigurationFor("a1")
+        .getSourceContext().get("r1").getParameters().get("port"));
+  }
+}

http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/src/test/resources/flume-conf-with-envvars.properties
----------------------------------------------------------------------
diff --git a/flume-ng-node/src/test/resources/flume-conf-with-envvars.properties b/flume-ng-node/src/test/resources/flume-conf-with-envvars.properties
new file mode 100644
index 0000000..536eb02
--- /dev/null
+++ b/flume-ng-node/src/test/resources/flume-conf-with-envvars.properties
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+a1.sources = r1
+a1.sources.r1.type = netcat
+a1.sources.r1.bind = 0.0.0.0
+a1.sources.r1.port = ${NC_PORT}
+a1.sources.r1.channels = c1
+
+a1.channels = c1
+a1.channels.c1.type = memory
+a1.channels.c1.capacity = 10000
+a1.channels.c1.transactionCapacity = 10000
+a1.channels.c1.byteCapacityBufferPercentage = 20
+a1.channels.c1.byteCapacity = 800000
+
+a1.channels = c1
+a1.sinks = k1
+a1.sinks.k1.type = logger
+a1.sinks.k1.channel = c1


Mime
View raw message