openwhisk-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rab...@apache.org
Subject [incubator-openwhisk] branch master updated: Configure components via pureconfig. (#2576)
Date Mon, 27 Nov 2017 17:12:38 GMT
This is an automated email from the ASF dual-hosted git repository.

rabbah pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git


The following commit(s) were added to refs/heads/master by this push:
     new aa697f6  Configure components via pureconfig. (#2576)
aa697f6 is described below

commit aa697f620ae5f07828da677d06fa0f62b4f60adc
Author: Markus Thömmes <markusthoemmes@me.com>
AuthorDate: Mon Nov 27 18:12:35 2017 +0100

    Configure components via pureconfig. (#2576)
    
    This adds the ability to configure the main components
    (controller/invoker) via pureconfig case-classes vs. using
    WhiskConfig.
    
    Benefits are:
    
    - Automatic type derivation
    - Static loading of config vs. passing one instance around
    - Proper scoping of configuration
    - It's the standard in Scala (based on TypesafeConfig)
    
    Implementation:
    
    TypesafeConfig reads JSON-like configuration files, which we can nest
    as we please. Those namespaces can be read by each component
    independently.  While TypesafeConfig supports expansion of environment
    variables, one has to do so explicitly. Which System properties, one
    can override specific values directly without the added overhead. To
    achieve the best of both worlds (for containers we only have
    environment variables available) there is a small script which
    translates environment variables to system properties in Java.
---
 ansible/roles/controller/tasks/deploy.yml          | 24 +++++++-----
 ansible/roles/invoker/tasks/deploy.yml             |  2 +-
 common/scala/Dockerfile                            |  3 ++
 common/scala/build.gradle                          |  2 +
 common/scala/src/main/resources/application.conf   |  5 ---
 common/scala/transformEnvironment.sh               | 45 ++++++++++++++++++++++
 core/controller/Dockerfile                         |  6 ++-
 core/controller/init.sh                            |  6 +++
 .../controller/src/main/resources/application.conf | 18 ++++-----
 .../core/loadBalancer/LoadBalancerService.scala    | 16 ++++----
 core/invoker/Dockerfile                            |  4 ++
 core/invoker/init.sh                               |  6 +++
 docs/dev/configuration.md                          | 17 ++++++++
 settings.gradle                                    |  2 +-
 14 files changed, 121 insertions(+), 35 deletions(-)

diff --git a/ansible/roles/controller/tasks/deploy.yml b/ansible/roles/controller/tasks/deploy.yml
index 2e5d276..3978316 100644
--- a/ansible/roles/controller/tasks/deploy.yml
+++ b/ansible/roles/controller/tasks/deploy.yml
@@ -75,27 +75,33 @@
       "LIMITS_ACTIONS_INVOKES_CONCURRENTINSYSTEM": "{{ limits.concurrentInvocationsSystem
}}"
       "LIMITS_TRIGGERS_FIRES_PERMINUTE": "{{ limits.firesPerMinute }}"
       "LIMITS_ACTIONS_SEQUENCE_MAXLENGTH": "{{ limits.sequenceMaxLength }}"
-      "CONTROLLER_BLACKBOXFRACTION": "{{ controller.blackboxFraction }}"
-      "LOADBALANCER_INVOKERBUSYTHRESHOLD": "{{ invoker.busyThreshold }}"
 
       "RUNTIMES_MANIFEST": "{{ runtimesManifest | to_json }}"
       "CONTROLLER_LOCALBOOKKEEPING": "{{ controller.localBookkeeping }}"
-      "AKKA_CLUSTER_PORT": "{{ controller.akka.cluster.basePort + groups['controllers'].index(inventory_hostname)
}}"
-      "AKKA_CLUSTER_HOST": "{{ controller.akka.cluster.host[groups['controllers'].index(inventory_hostname)]
}}"
       "AKKA_CLUSTER_SEED_NODES": "{{seed_nodes_list | join(' ') }}"
-      "AKKA_CLUSTER_BIND_PORT": "{{ controller.akka.cluster.bindPort }}"
-      "AKKA_ACTOR_PROVIDER": "{{ controller.akka.provider }}"
       "METRICS_KAMON": "{{ metrics.kamon.enabled }}"
       "METRICS_LOG": "{{ metrics.log.enabled }}"
-      "METRICS_KAMON_HOST": "{{ metrics.kamon.host }}"
-      "METRICS_KAMON_PORT": "{{ metrics.kamon.port }}"
       "CONTROLLER_HA": "{{ controller.ha }}"
+
+      "METRICS_KAMON": "{{ metrics.kamon.enabled }}"
+      "METRICS_LOG": "{{ metrics.log.enabled }}"
+
+      "CONFIG_whisk_loadbalancer_invokerBusyThreshold": "{{ invoker.busyThreshold }}"
+      "CONFIG_whisk_loadbalancer_blackboxFraction": "{{ controller.blackboxFraction }}"
+
+      "CONFIG_akka_actor_provider": "{{ controller.akka.provider }}"
+      "CONFIG_akka_remote_netty_tcp_hostname": "{{ controller.akka.cluster.host[groups['controllers'].index(inventory_hostname)]
}}"
+      "CONFIG_akka_remote_netty_tcp_port": "{{ controller.akka.cluster.basePort + groups['controllers'].index(inventory_hostname)
}}"
+      "CONFIG_akka_remote_netty_tcp_bindPort": "{{ controller.akka.cluster.bindPort }}"
+
+      "CONFIG_kamon_statsd_hostname": "{{ metrics.kamon.host }}"
+      "CONFIG_kamon_statsd_port": "{{ metrics.kamon.port }}"
     volumes:
       - "{{ whisk_logs_dir }}/controller{{ groups['controllers'].index(inventory_hostname)
}}:/logs"
     ports:
       - "{{ controller.basePort + groups['controllers'].index(inventory_hostname) }}:8080"
       - "{{ controller.akka.cluster.basePort + groups['controllers'].index(inventory_hostname)
}}:{{ controller.akka.cluster.bindPort }}"
-    command: /bin/sh -c "export CONTAINER_IP=$(hostname -I); controller/bin/controller {{
groups['controllers'].index(inventory_hostname) }} >> /logs/controller{{ groups['controllers'].index(inventory_hostname)
}}_logs.log 2>&1"
+    command: /bin/sh -c "exec /init.sh {{ groups['controllers'].index(inventory_hostname)
}} >> /logs/controller{{ groups['controllers'].index(inventory_hostname) }}_logs.log
2>&1"
 
 - name: wait until the Controller in this host is up and running
   uri:
diff --git a/ansible/roles/invoker/tasks/deploy.yml b/ansible/roles/invoker/tasks/deploy.yml
index 560d64a..94c06a3 100644
--- a/ansible/roles/invoker/tasks/deploy.yml
+++ b/ansible/roles/invoker/tasks/deploy.yml
@@ -161,7 +161,7 @@
         -v {{ docker_sock | default('/var/run/docker.sock') }}:/var/run/docker.sock
         -p {{ invoker.port + groups['invokers'].index(inventory_hostname) }}:8080
         {{ docker_registry }}{{ docker.image.prefix }}/invoker:{{ docker.image.tag }}
-        /bin/sh -c "exec /invoker/bin/invoker {{ groups['invokers'].index(inventory_hostname)
}} >> /logs/invoker{{ groups['invokers'].index(inventory_hostname) }}_logs.log 2>&1"
+        /bin/sh -c "exec /init.sh {{ groups['invokers'].index(inventory_hostname) }} >>
/logs/invoker{{ groups['invokers'].index(inventory_hostname) }}_logs.log 2>&1"
 
 # todo: re-enable docker_container module once https://github.com/ansible/ansible-modules-core/issues/5054
is resolved
 
diff --git a/common/scala/Dockerfile b/common/scala/Dockerfile
index 750b1d5..15e5bdf 100644
--- a/common/scala/Dockerfile
+++ b/common/scala/Dockerfile
@@ -29,3 +29,6 @@ RUN update-alternatives --install "/usr/bin/java" "java" "${JRE_HOME}/bin/java"
   update-alternatives --set java "${JRE_HOME}/bin/java" && \
   update-alternatives --set javac "${JAVA_HOME}/bin/javac" && \
   mkdir /logs
+
+COPY transformEnvironment.sh /
+RUN chmod +x transformEnvironment.sh
\ No newline at end of file
diff --git a/common/scala/build.gradle b/common/scala/build.gradle
index 8777f47..31781f4 100644
--- a/common/scala/build.gradle
+++ b/common/scala/build.gradle
@@ -11,11 +11,13 @@ repositories {
 dependencies {
     compile "org.scala-lang:scala-library:${gradle.scala.version}"
 
+    compile 'com.github.pureconfig:pureconfig_2.11:0.7.2'
     compile 'io.spray:spray-json_2.11:1.3.3'
 
     compile 'com.typesafe.akka:akka-actor_2.11:2.5.6'
     compile 'com.typesafe.akka:akka-stream_2.11:2.5.6'
     compile 'com.typesafe.akka:akka-slf4j_2.11:2.5.6'
+
     compile 'com.typesafe.akka:akka-http-core_2.11:10.0.10'
     compile 'com.typesafe.akka:akka-http-spray-json_2.11:10.0.10'
 
diff --git a/common/scala/src/main/resources/application.conf b/common/scala/src/main/resources/application.conf
index 1968e90..41b41ce 100644
--- a/common/scala/src/main/resources/application.conf
+++ b/common/scala/src/main/resources/application.conf
@@ -21,11 +21,6 @@ kamon {
     }
 
     statsd {
-        # Hostname and port in which your StatsD is running. Remember that StatsD packets
are sent using UDP and
-        # setting unreachable hosts and/or not open ports wont be warned by the Kamon, your
data wont go anywhere.
-        hostname = ${?METRICS_KAMON_HOST}
-        port = ${?METRICS_KAMON_PORT}
-
         # Interval between metrics data flushes to StatsD. It's value must be equal or greater
than the
         # kamon.metrics.tick-interval setting.
         flush-interval = 1 second
diff --git a/common/scala/transformEnvironment.sh b/common/scala/transformEnvironment.sh
new file mode 100755
index 0000000..05039ff
--- /dev/null
+++ b/common/scala/transformEnvironment.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+#
+# Transforms environment variables starting with `prefix` to kebab-cased JVM system properties
+# 
+# "_"           becomes "."
+# "camelCased"  becomes "camel-cased"
+# "PascalCased" stays   "PascalCased" -> classnames stay untouched
+# 
+# Examples:
+# CONFIG_whisk_loadbalancer_invokerBusyThreshold -> -Dwhisk.loadbalancer.invoker-busy-threshold
+# CONFIG_akka_remote_netty_tcp_bindPort          -> -Dakka.remote.netty.tcp.bind-port
+# CONFIG_whisk_spi_LogStoreProvider              -> -Dwhisk.spi.LogStoreProvider
+#
+
+prefix="CONFIG_"
+configVariables=$(compgen -v | grep $prefix)
+
+props=()
+
+for var in $configVariables
+do
+    value=$(printenv "$var")
+    if [ ! -z "$value" ]
+    then
+        sansConfig=${var#$prefix} # remove the CONFIG_ prefix
+        parts=${sansConfig//_/ } # "split" the name by replacing '_' with ' '
+
+        transformedParts=()
+        for part in $parts
+        do
+            if [[ $part =~ ^[A-Z] ]] # if the current part starts with an uppercase letter
(is PascalCased)
+            then
+                transformedParts+=($part) # leave it alone
+            else
+                transformedParts+=($(echo "$part" | sed -r 's/([a-z0-9])([A-Z])/\1-\L\2/g'))
# rewrite camelCased to kebab-cased
+            fi
+        done
+
+        key=$(IFS=.; echo "${transformedParts[*]}") # reassemble the parts delimited by a
'.'
+        props+=("-D$key='$value'") # assemble a JVM system property
+    fi
+done
+
+echo "${props[@]}"
\ No newline at end of file
diff --git a/core/controller/Dockerfile b/core/controller/Dockerfile
index ecfc0ec..51f673d 100644
--- a/core/controller/Dockerfile
+++ b/core/controller/Dockerfile
@@ -11,7 +11,11 @@ RUN wget --no-verbose https://github.com/swagger-api/swagger-ui/archive/v2.1.4.t
 
 #
 # Copy app jars
-COPY build/distributions/controller.tar ./
+COPY build/distributions/controller.tar /
 RUN tar xf controller.tar
 
+COPY init.sh /
+RUN chmod +x init.sh
+
 EXPOSE 8080
+CMD ["init.sh", "0"]
\ No newline at end of file
diff --git a/core/controller/init.sh b/core/controller/init.sh
new file mode 100644
index 0000000..232c405
--- /dev/null
+++ b/core/controller/init.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export CONTROLLER_OPTS
+CONTROLLER_OPTS="$CONTROLLER_OPTS -Dakka.remote.netty.tcp.bind-hostname=$(hostname -I) $(./transformEnvironment.sh)"
+
+exec controller/bin/controller "$@"
\ No newline at end of file
diff --git a/core/controller/src/main/resources/application.conf b/core/controller/src/main/resources/application.conf
index 6e13354..10615d3 100644
--- a/core/controller/src/main/resources/application.conf
+++ b/core/controller/src/main/resources/application.conf
@@ -2,6 +2,13 @@
 include "logging"
 include "akka-http-version"
 
+whisk {
+  loadbalancer {
+    invoker-busy-threshold: 4
+    blackbox-fraction: 10%
+  }
+}
+
 # http://doc.akka.io/docs/akka-http/current/scala/http/configuration.html
 # descriptions inlined below for convenience
 akka.http {
@@ -59,26 +66,15 @@ akka.http {
 # Check out all akka-remote-2.5.4 options here:
 # http://doc.akka.io/docs/akka/2.5.4/scala/general/configuration.html#config-akka-remote
 akka {
-  actor {
-    provider = ${?AKKA_ACTOR_PROVIDER}
-  }
   remote {
     log-remote-lifecycle-events = DEBUG
     log-received-messages = on
     log-sent-messages = on
-
-    netty.tcp {
-      hostname = ${?AKKA_CLUSTER_HOST}
-      port = ${?AKKA_CLUSTER_PORT}
-      bind-port = ${?AKKA_CLUSTER_BIND_PORT}
-      bind-hostname = ${?CONTAINER_IP}
-    }
   }
   cluster {
     # Disable legacy metrics in akka-cluster.
     metrics.enabled=off
 
-    auto-down-unreachable-after = ${?AUTO_DOWN_UNREACHABLE_AFTER}
     #distributed-data.notify-subscribers-interval = 0.01
   }
 }
\ No newline at end of file
diff --git a/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala
b/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala
index a8be73b..e38f69c 100644
--- a/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala
+++ b/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala
@@ -55,6 +55,10 @@ import whisk.core.entity.size._
 import whisk.core.entity.types.EntityStore
 import whisk.spi.SpiLoader
 
+import pureconfig._
+
+case class LoadbalancerConfig(blackboxFraction: Double, invokerBusyThreshold: Int)
+
 trait LoadBalancer {
 
   val activeAckTimeoutGrace = 1.minute
@@ -87,11 +91,13 @@ class LoadBalancerService(config: WhiskConfig, instance: InstanceId, entityStore
   logging: Logging)
     extends LoadBalancer {
 
+  private val lbConfig = loadConfigOrThrow[LoadbalancerConfig]("whisk.loadbalancer")
+
   /** The execution context for futures */
   implicit val executionContext: ExecutionContext = actorSystem.dispatcher
 
   /** How many invokers are dedicated to blackbox images.  We range bound to something sensical
regardless of configuration. */
-  private val blackboxFraction: Double = Math.max(0.0, Math.min(1.0, config.controllerBlackboxFraction))
+  private val blackboxFraction: Double = Math.max(0.0, Math.min(1.0, lbConfig.blackboxFraction))
   logging.info(this, s"blackboxFraction = $blackboxFraction")(TransactionId.loadbalancer)
 
   /** Feature switch for shared load balancer data **/
@@ -324,7 +330,7 @@ class LoadBalancerService(config: WhiskConfig, instance: InstanceId, entityStore
           case (instance, state) => (instance, state, currentActivations.getOrElse(instance.toString,
0))
         }
 
-        LoadBalancerService.schedule(invokersWithUsage, config.loadbalancerInvokerBusyThreshold,
hash) match {
+        LoadBalancerService.schedule(invokersWithUsage, lbConfig.invokerBusyThreshold, hash)
match {
           case Some(invoker) => Future.successful(invoker)
           case None =>
             logging.error(this, s"all invokers down")(TransactionId.invokerHealth)
@@ -347,11 +353,7 @@ object LoadBalancerService {
         kafkaTopicsCompletedRetentionBytes -> 1024.MB.toBytes.toString,
         kafkaTopicsCompletedRetentionMS -> 1.hour.toMillis.toString,
         kafkaTopicsCompletedSegmentBytes -> 512.MB.toBytes.toString) ++
-      Map(
-        loadbalancerInvokerBusyThreshold -> null,
-        controllerBlackboxFraction -> null,
-        controllerLocalBookkeeping -> null,
-        controllerSeedNodes -> null)
+      Map(controllerLocalBookkeeping -> null, controllerSeedNodes -> null)
 
   /** Memoizes the result of `f` for later use. */
   def memoize[I, O](f: I => O): I => O = new scala.collection.mutable.HashMap[I, O]()
{
diff --git a/core/invoker/Dockerfile b/core/invoker/Dockerfile
index 3a7a53f..2a8b6ee 100644
--- a/core/invoker/Dockerfile
+++ b/core/invoker/Dockerfile
@@ -15,4 +15,8 @@ COPY build/distributions/invoker.tar ./
 RUN tar xf invoker.tar && \
 rm -f invoker.tar
 
+COPY init.sh /
+RUN chmod +x init.sh
+
 EXPOSE 8080
+CMD ["init.sh", "0"]
\ No newline at end of file
diff --git a/core/invoker/init.sh b/core/invoker/init.sh
new file mode 100644
index 0000000..beb5e71
--- /dev/null
+++ b/core/invoker/init.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export INVOKER_OPTS
+INVOKER_OPTS="$INVOKER_OPTS $(./transformEnvironment.sh)"
+
+exec invoker/bin/invoker "$@"
\ No newline at end of file
diff --git a/docs/dev/configuration.md b/docs/dev/configuration.md
new file mode 100644
index 0000000..cedc31c
--- /dev/null
+++ b/docs/dev/configuration.md
@@ -0,0 +1,17 @@
+# Scala component configuration
+
+Our Scala components are in the process of switching from our own `WhiskConfig` to the TypesafeConfig
based `pureconfig`. That will give us a more sane way of parsing environment values, overriding
them and defining sensible defaults for them.
+
+TypesafeConfig is able to override its values via JVM system properties. Unfortunately, docker
only supports passing environment variables in a comfortable way.
+
+To solve that mismatch, we use a script to transform environment variables into JVM system
properties to spare the developer of manually overriding all the values via explicit environment
-> TypesafeConfig value mapping. That script applies the following transformations to all
environment variables starting with a defined prefix ("CONFIG_" in our case):
+
+- `_` becomes `.` (TypesafeConfig hierarchies are built using `.`)
+- `camelCased` becomes `camel-cased` (Kebabcase is usually used for TypesafeConfig keys)
+- `PascalCased` stays `PascalCased` (Defining classnames for overriding SPIs is crucial)
+
+### Examples:
+
+- `CONFIG_whisk_loadbalancer_invokerBusyThreshold` becomes `-Dwhisk.loadbalancer.invoker-busy-threshold`
+- `CONFIG_akka_remote_netty_tcp_bindPort` becomes `-Dakka.remote.netty.tcp.bind-port`
+- `CONFIG_whisk_spi_LogStoreProvider` becomes `-Dwhisk.spi.LogStoreProvider`
diff --git a/settings.gradle b/settings.gradle
index 19d3506..441c6b6 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -21,7 +21,7 @@ include 'tests:dat:blackbox:badproxy'
 rootProject.name = 'openwhisk'
 
 gradle.ext.scala = [
-    version: '2.11.8',
+    version: '2.11.11',
     compileFlags: ['-feature', '-unchecked', '-deprecation', '-Xfatal-warnings', '-Ywarn-unused-import']
 ]
 

-- 
To stop receiving notification emails like this one, please contact
['"commits@openwhisk.apache.org" <commits@openwhisk.apache.org>'].

Mime
View raw message