openwhisk-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From houshen...@apache.org
Subject [incubator-openwhisk] branch master updated: Improve the efficiency of Jenkins pipeline (#4391)
Date Tue, 26 Mar 2019 13:08:40 GMT
This is an automated email from the ASF dual-hosted git repository.

houshengbo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git


The following commit(s) were added to refs/heads/master by this push:
     new aad27be  Improve the efficiency of Jenkins pipeline (#4391)
aad27be is described below

commit aad27bed182f709fd7d25bc7a01f7e0fe985b027
Author: Vincent <shou@us.ibm.com>
AuthorDate: Tue Mar 26 09:08:32 2019 -0400

    Improve the efficiency of Jenkins pipeline (#4391)
    
    This PR will utilize three VMs as the edge machines, so that we can run 3
    openwhisk depoyments at the same time instead of one.
---
 Jenkinsfile                                        | 98 +++++++++++++---------
 .../group_vars/{all => openwhisk-vm1-he-de}        |  0
 .../group_vars/{all => openwhisk-vm2-he-de}        |  3 +-
 .../group_vars/{all => openwhisk-vm3-he-de}        |  2 +-
 .../{hosts.j2.ini => openwhisk-vm1-he-de.j2.ini}   |  0
 .../jenkins/openwhisk-vm2-he-de.j2.ini             | 36 ++++++++
 .../jenkins/openwhisk-vm3-he-de.j2.ini             | 36 ++++++++
 7 files changed, 133 insertions(+), 42 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1a7118d..262cd44 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -16,57 +16,75 @@
  * limitations under the License.
  */
 
-timeout(time: 4, unit: 'HOURS') {
+timeout(time: 12, unit: 'HOURS') {
 
-    def domainName = "openwhisk-vm1-he-de.apache.org"
     def port = "444"
     def cert = "domain.crt"
     def key = "domain.key"
 
-    node("openwhisk1") {
-        properties([disableConcurrentBuilds()])
-        deleteDir()
-        stage ('Checkout') {
-            checkout scm
-        }
-
-        stage ('Build') {
-            // Set up a private docker registry service, accessed by all the OpenWhisk VMs.
+    node("openwhisk") {
+        def hostName = sh(returnStdout: true, script: 'hostname').trim()
+        def domainName = hostName+".apache.org"
+        lock("${hostName}") {
             try {
-                sh "docker container stop registry && docker container rm -v registry"
-            } catch (exp) {
-                println("Unable to stop and remove the container registry.")
-            }
+                deleteDir()
+                stage('Checkout') {
+                    checkout scm
+                }
+
+                stage('Build') {
+                    // Set up a private docker registry service, accessed by all the OpenWhisk
VMs.
+                    try {
+                        sh "docker container stop registry && docker container rm
-v registry"
+                    } catch (exp) {
+                        println("Unable to stop and remove the container registry.")
+                    }
 
-            sh "docker run -d --restart=always --name registry -v \"$HOME\"/certs:/certs
\
+                    sh "docker run -d --restart=always --name registry -v \"$HOME\"/certs:/certs
\
                 -e REGISTRY_HTTP_ADDR=0.0.0.0:${port} -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/${cert}
\
                 -e REGISTRY_HTTP_TLS_KEY=/certs/${key} -p ${port}:${port} registry:2"
-            // Build the controller and invoker images.
-            sh "./gradlew distDocker -PdockerRegistry=${domainName}:${port}"
-        }
+                    // Build the controller and invoker images.
+                    sh "./gradlew distDocker -PdockerRegistry=${domainName}:${port}"
+                }
 
-        stage('Deploy') {
-            dir("ansible") {
-                // Copy the jenkins ansible configuration under the directory ansible. This
can make sure the SSH is used to
-                // access the VMs of invokers by the VM of the controller.
-                sh '[ -f "environments/jenkins/ansible_jenkins.cfg" ] && cp environments/jenkins/ansible_jenkins.cfg
ansible.cfg'
-                sh 'ansible-playbook -i environments/jenkins setup.yml'
-                sh 'ansible-playbook -i environments/jenkins openwhisk.yml -e mode=clean'
-                sh 'ansible-playbook -i environments/jenkins apigateway.yml -e mode=clean'
-                sh 'ansible-playbook -i environments/jenkins couchdb.yml -e mode=clean'
-                sh 'ansible-playbook -i environments/jenkins couchdb.yml'
-                sh 'ansible-playbook -i environments/jenkins initdb.yml'
-                sh 'ansible-playbook -i environments/jenkins wipe.yml'
-                sh 'ansible-playbook -i environments/jenkins apigateway.yml'
-                sh 'ansible-playbook -i environments/jenkins openwhisk.yml'
-                sh 'ansible-playbook -i environments/jenkins properties.yml'
-                sh 'ansible-playbook -i environments/jenkins routemgmt.yml'
-                sh 'ansible-playbook -i environments/jenkins postdeploy.yml'
-            }
-        }
+                stage('Deploy') {
+                    dir("ansible") {
+                        // Copy the jenkins ansible configuration under the directory ansible.
This can make sure the SSH is used to
+                        // access the VMs of invokers by the VM of the controller.
+                        sh '[ -f "environments/jenkins/ansible_jenkins.cfg" ] &&
cp environments/jenkins/ansible_jenkins.cfg ansible.cfg'
+                    }
+
+                    dir("ansible/environments/jenkins") {
+                        sh "cp ${hostName}.j2.ini hosts.j2.ini"
+                    }
 
-        stage('Test') {
-            sh './gradlew :tests:test'
+                    dir("ansible/environments/jenkins/group_vars") {
+                        sh "cp ${hostName} all"
+                    }
+
+                    dir("ansible") {
+                        sh 'ansible-playbook -i environments/jenkins setup.yml'
+                        sh 'ansible-playbook -i environments/jenkins openwhisk.yml -e mode=clean'
+                        sh 'ansible-playbook -i environments/jenkins apigateway.yml -e mode=clean'
+                        sh 'ansible-playbook -i environments/jenkins couchdb.yml -e mode=clean'
+                        sh 'ansible-playbook -i environments/jenkins couchdb.yml'
+                        sh 'ansible-playbook -i environments/jenkins initdb.yml'
+                        sh 'ansible-playbook -i environments/jenkins wipe.yml'
+                        sh 'ansible-playbook -i environments/jenkins apigateway.yml'
+                        sh 'ansible-playbook -i environments/jenkins openwhisk.yml'
+                        sh 'ansible-playbook -i environments/jenkins properties.yml'
+                        sh 'ansible-playbook -i environments/jenkins routemgmt.yml'
+                        sh 'ansible-playbook -i environments/jenkins postdeploy.yml'
+                    }
+                }
+                stage('Test') {
+                    sh './gradlew :tests:test'
+                }
+            } catch (exp) {
+                println("Exception:" + exp)
+            } finally {
+                step([$class: 'JUnitResultArchiver', testResults: '**/test-results/**/TEST-*.xml'])
+            }
         }
     }
 }
diff --git a/ansible/environments/jenkins/group_vars/all b/ansible/environments/jenkins/group_vars/openwhisk-vm1-he-de
similarity index 100%
copy from ansible/environments/jenkins/group_vars/all
copy to ansible/environments/jenkins/group_vars/openwhisk-vm1-he-de
diff --git a/ansible/environments/jenkins/group_vars/all b/ansible/environments/jenkins/group_vars/openwhisk-vm2-he-de
similarity index 93%
copy from ansible/environments/jenkins/group_vars/all
copy to ansible/environments/jenkins/group_vars/openwhisk-vm2-he-de
index 659fdc2..814a060 100755
--- a/ansible/environments/jenkins/group_vars/all
+++ b/ansible/environments/jenkins/group_vars/openwhisk-vm2-he-de
@@ -6,7 +6,7 @@ config_root_dir: "{{ openwhisk_tmp_dir }}/wskconf"
 whisk_logs_dir: "{{ openwhisk_tmp_dir }}/wsklogs"
 coverage_enabled: "{{ lookup('env', 'GRADLE_COVERAGE') | default('false', true) | bool}}"
 coverage_logs_dir: "{{ openwhisk_tmp_dir }}/wskcov"
-docker_registry: "openwhisk-vm1-he-de.apache.org:444/"
+docker_registry: "openwhisk-vm2-he-de.apache.org:444/"
 docker_dns: ""
 runtimes_bypass_pull_for_local_images: true
 invoker_use_runc: "{{ ansible_distribution != 'MacOSX' }}"
@@ -34,3 +34,4 @@ env_hosts_dir: "{{ playbook_dir }}/environments/jenkins"
 container_pool_akka_client: true
 runtimes_enable_concurrency: true
 limit_action_concurrency_max: 500
+limit_invocations_per_minute: 120
diff --git a/ansible/environments/jenkins/group_vars/all b/ansible/environments/jenkins/group_vars/openwhisk-vm3-he-de
similarity index 95%
rename from ansible/environments/jenkins/group_vars/all
rename to ansible/environments/jenkins/group_vars/openwhisk-vm3-he-de
index 659fdc2..e5ed3f2 100755
--- a/ansible/environments/jenkins/group_vars/all
+++ b/ansible/environments/jenkins/group_vars/openwhisk-vm3-he-de
@@ -6,7 +6,7 @@ config_root_dir: "{{ openwhisk_tmp_dir }}/wskconf"
 whisk_logs_dir: "{{ openwhisk_tmp_dir }}/wsklogs"
 coverage_enabled: "{{ lookup('env', 'GRADLE_COVERAGE') | default('false', true) | bool}}"
 coverage_logs_dir: "{{ openwhisk_tmp_dir }}/wskcov"
-docker_registry: "openwhisk-vm1-he-de.apache.org:444/"
+docker_registry: "openwhisk-vm3-he-de.apache.org:444/"
 docker_dns: ""
 runtimes_bypass_pull_for_local_images: true
 invoker_use_runc: "{{ ansible_distribution != 'MacOSX' }}"
diff --git a/ansible/environments/jenkins/hosts.j2.ini b/ansible/environments/jenkins/openwhisk-vm1-he-de.j2.ini
similarity index 100%
rename from ansible/environments/jenkins/hosts.j2.ini
rename to ansible/environments/jenkins/openwhisk-vm1-he-de.j2.ini
diff --git a/ansible/environments/jenkins/openwhisk-vm2-he-de.j2.ini b/ansible/environments/jenkins/openwhisk-vm2-he-de.j2.ini
new file mode 100644
index 0000000..b6be1a4
--- /dev/null
+++ b/ansible/environments/jenkins/openwhisk-vm2-he-de.j2.ini
@@ -0,0 +1,36 @@
+; the first parameter in a host is the inventory_hostname
+
+; used for local actions only
+ansible ansible_connection=local
+
+[edge]
+5.9.20.92           ansible_host=5.9.20.92 ansible_connection=local
+
+[controllers]
+controller0         ansible_host=5.9.20.92 ansible_connection=local
+;{% if mode is defined and 'HA' in mode %}
+;controller1         ansible_host=5.9.20.92 ansible_connection=local
+;{% endif %}
+
+[kafkas]
+kafka0              ansible_host=5.9.20.92 ansible_connection=local
+{% if mode is defined and 'HA' in mode %}
+kafka1              ansible_host=5.9.20.92 ansible_connection=local
+{% endif %}
+
+[zookeepers:children]
+kafkas
+
+[invokers]
+invoker0            ansible_host=144.76.27.183 ansible_connection=ssh
+invoker1            ansible_host=144.76.63.94 ansible_connection=ssh
+
+; db group is only used if db.provider is CouchDB
+[db]
+5.9.20.92         ansible_host=5.9.20.92 ansible_connection=local
+
+[redis]
+5.9.20.92          ansible_host=5.9.20.92 ansible_connection=local
+
+[apigateway]
+5.9.20.92          ansible_host=5.9.20.92 ansible_connection=local
diff --git a/ansible/environments/jenkins/openwhisk-vm3-he-de.j2.ini b/ansible/environments/jenkins/openwhisk-vm3-he-de.j2.ini
new file mode 100644
index 0000000..c150271
--- /dev/null
+++ b/ansible/environments/jenkins/openwhisk-vm3-he-de.j2.ini
@@ -0,0 +1,36 @@
+; the first parameter in a host is the inventory_hostname
+
+; used for local actions only
+ansible ansible_connection=local
+
+[edge]
+144.76.27.183          ansible_host=144.76.27.183 ansible_connection=local
+
+[controllers]
+controller0         ansible_host=144.76.27.183 ansible_connection=local
+;{% if mode is defined and 'HA' in mode %}
+;controller1         ansible_host=144.76.27.183 ansible_connection=local
+;{% endif %}
+
+[kafkas]
+kafka0              ansible_host=144.76.27.183 ansible_connection=local
+{% if mode is defined and 'HA' in mode %}
+kafka1              ansible_host=144.76.27.183 ansible_connection=local
+{% endif %}
+
+[zookeepers:children]
+kafkas
+
+[invokers]
+invoker0            ansible_host=144.76.63.94 ansible_connection=ssh
+invoker1            ansible_host=5.9.20.92 ansible_connection=ssh
+
+; db group is only used if db.provider is CouchDB
+[db]
+144.76.27.183         ansible_host=144.76.27.183 ansible_connection=local
+
+[redis]
+144.76.27.183          ansible_host=144.76.27.183 ansible_connection=local
+
+[apigateway]
+144.76.27.183          ansible_host=144.76.27.183 ansible_connection=local


Mime
View raw message