flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m..@apache.org
Subject flink git commit: [FLINK-4881][docker] remove shared volume for configuration
Date Fri, 21 Oct 2016 12:20:51 GMT
Repository: flink
Updated Branches:
  refs/heads/master a27451d8e -> 58a16c100


[FLINK-4881][docker] remove shared volume for configuration

README.md                  -    Description of the bluemix specific steps
build.sh                   -    modify permissions to make script executable by default
docker-compose-bluemix.yml -    allow to specify image names with path to private BM registry
docker-compose.sh          -    find out path to docker registry, extend build time to 120
secs, then call docker-compose
docker-compose.yml         -    make use of container linking instead of shared volumes
docker-entrypoint.sh       -    use 'jobmanager' entry in /etc/hosts arising from container
linking so that taskmanager can register itself with jobmanager

docker-compose.sh -> bluemix-docker-compose.sh    - renamed on request
docker-entrypoint.sh    - made sure the taskmanager's slots correlate to the number of CPUs

This closes #2667


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/58a16c10
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/58a16c10
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/58a16c10

Branch: refs/heads/master
Commit: 58a16c1007c97809bdd46c93632abd90c0ce5c4b
Parents: a27451d
Author: Markus Müller <markus_mueller@de.ibm.com>
Authored: Thu Oct 20 10:08:34 2016 +0200
Committer: Maximilian Michels <mxm@apache.org>
Committed: Fri Oct 21 14:19:38 2016 +0200

----------------------------------------------------------------------
 flink-contrib/docker-flink/README.md            | 13 ++++++
 .../docker-flink/bluemix-docker-compose.sh      | 22 ++++++++++
 .../docker-flink/docker-compose-bluemix.yml     | 42 ++++++++++++++++++++
 flink-contrib/docker-flink/docker-compose.yml   | 12 ++++--
 flink-contrib/docker-flink/docker-entrypoint.sh | 13 +++++-
 5 files changed, 96 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/58a16c10/flink-contrib/docker-flink/README.md
----------------------------------------------------------------------
diff --git a/flink-contrib/docker-flink/README.md b/flink-contrib/docker-flink/README.md
index 6ff3aea..c7d94bb 100644
--- a/flink-contrib/docker-flink/README.md
+++ b/flink-contrib/docker-flink/README.md
@@ -8,6 +8,19 @@ https://docs.docker.com/installation/
 Install the most recent stable version of docker-compose
 https://docs.docker.com/compose/install/
 
+## Bluemix PaaS
+
+If you want to build the image on Bluemix redirect the docker cli to the remote endpoints.
There is plenty of documentation how
+to obtain a Bluemix account, so I'm not specifying details. Once you have it and log in using
the cloud foundry CLI 'cf' you also
+need the container specific plugin 'ic'. With that you can get the URL to the remote docker
host as well as the path to
+the certificates. If you search for "Logging in to the IBM Containers CLI plug-in" you get
the details. <br>Here an example:<br>
+export DOCKER_HOST=tcp://containers-api.eu-gb.bluemix.net:8443<br>
+export DOCKER_CERT_PATH=/home/markus/.ice/certs/containers-api.eu-gb.bluemix.net/3c63cb44-86d8-4e89-9a40-f8f3f894a09f<br>
+export DOCKER_TLS_VERIFY=1<br>
+
+Now when proceeding to the next paragraph 'build' the docker commands build and run the image
(provided you use ./docker-compose.sh instead of the native docker-compose command.
+Do not forget to allocate and bind a public IP address with the 'cf ic ip' set of commands
afterwards.
+
 # Build
 
 Images are based on the official Java Alpine (OpenJDK 8) image. If you want to

http://git-wip-us.apache.org/repos/asf/flink/blob/58a16c10/flink-contrib/docker-flink/bluemix-docker-compose.sh
----------------------------------------------------------------------
diff --git a/flink-contrib/docker-flink/bluemix-docker-compose.sh b/flink-contrib/docker-flink/bluemix-docker-compose.sh
new file mode 100755
index 0000000..9d090a7
--- /dev/null
+++ b/flink-contrib/docker-flink/bluemix-docker-compose.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+export COMPOSE_HTTP_TIMEOUT=120
+export IMAGENAME="registry.eu-gb.bluemix.net/"`cf ic namespace get`"/flink"
+docker-compose  -f docker-compose-bluemix.yml up -d

http://git-wip-us.apache.org/repos/asf/flink/blob/58a16c10/flink-contrib/docker-flink/docker-compose-bluemix.yml
----------------------------------------------------------------------
diff --git a/flink-contrib/docker-flink/docker-compose-bluemix.yml b/flink-contrib/docker-flink/docker-compose-bluemix.yml
new file mode 100644
index 0000000..0155f01
--- /dev/null
+++ b/flink-contrib/docker-flink/docker-compose-bluemix.yml
@@ -0,0 +1,42 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# image specification requires the full path to your private bluemix image registry
+
+version: "2"
+services:
+  jobmanager:
+    #image example:  registry.eu-gb.bluemix.net/markussaidhiho/flink
+    image: ${IMAGENAME}
+    container_name: "jobmanager"
+    expose:
+      - "6123"
+    ports:
+      - "48081:8081"
+    command: jobmanager
+
+  taskmanager:
+    image: ${IMAGENAME}
+    expose:
+      - "6121"
+      - "6122"
+    depends_on:
+      - jobmanager
+    command: taskmanager
+    links:
+      - "jobmanager:jobmanager"

http://git-wip-us.apache.org/repos/asf/flink/blob/58a16c10/flink-contrib/docker-flink/docker-compose.yml
----------------------------------------------------------------------
diff --git a/flink-contrib/docker-flink/docker-compose.yml b/flink-contrib/docker-flink/docker-compose.yml
index 08956c6..eaa30a6 100644
--- a/flink-contrib/docker-flink/docker-compose.yml
+++ b/flink-contrib/docker-flink/docker-compose.yml
@@ -20,16 +20,20 @@ version: "2"
 services:
   jobmanager:
     image: flink
+    container_name: "jobmanager"
+    expose:
+      - "6123"
     ports:
       - "48081:8081"
     command: jobmanager
-    volumes:
-      - /opt/flink/conf
 
   taskmanager:
     image: flink
+    expose:
+      - "6121"
+      - "6122"
     depends_on:
       - jobmanager
     command: taskmanager
-    volumes_from:
-      - jobmanager:ro
+    links:
+      - "jobmanager:jobmanager"

http://git-wip-us.apache.org/repos/asf/flink/blob/58a16c10/flink-contrib/docker-flink/docker-entrypoint.sh
----------------------------------------------------------------------
diff --git a/flink-contrib/docker-flink/docker-entrypoint.sh b/flink-contrib/docker-flink/docker-entrypoint.sh
index 0da8809..a2ca0e9 100755
--- a/flink-contrib/docker-flink/docker-entrypoint.sh
+++ b/flink-contrib/docker-flink/docker-entrypoint.sh
@@ -20,11 +20,20 @@
 
 if [ "$1" = "jobmanager" ]; then
     echo "Starting Job Manager"
-    sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: `hostname -f`/g"
$FLINK_HOME/conf/flink-conf.yaml
-    sed -i -e "s/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: `grep -c
^processor /proc/cpuinfo`/g" $FLINK_HOME/conf/flink-conf.yaml
+    #sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: `hostname -f`/g"
$FLINK_HOME/conf/flink-conf.yaml
+
+    # make use of container linking and exploit the jobmanager entry in /etc/hosts
+    sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: jobmanager/g"
$FLINK_HOME/conf/flink-conf.yaml
+
     echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml
     $FLINK_HOME/bin/jobmanager.sh start cluster
 elif [ "$1" = "taskmanager" ]; then
+
+    # make use of container linking and exploit the jobmanager entry in /etc/hosts
+    sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: jobmanager/g"
$FLINK_HOME/conf/flink-conf.yaml
+
+    sed -i -e "s/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: `grep -c
^processor /proc/cpuinfo`/g" $FLINK_HOME/conf/flink-conf.yaml
+
     echo "Starting Task Manager"
     echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml
     $FLINK_HOME/bin/taskmanager.sh start


Mime
View raw message