geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbarr...@apache.org
Subject [geode] branch develop updated: GEODE-5476: Migrate pipeline changes to pull request pipeline.
Date Thu, 26 Jul 2018 22:26:25 GMT
This is an automated email from the ASF dual-hosted git repository.

jbarrett pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git


The following commit(s) were added to refs/heads/develop by this push:
     new 4e0bab8  GEODE-5476: Migrate pipeline changes to pull request pipeline.
4e0bab8 is described below

commit 4e0bab82e4b1c457755f876f490e2f9bcab5cb50
Author: FSOUTHERLAND <39743785+FSOUTHERLAND@users.noreply.github.com>
AuthorDate: Thu Jul 26 15:26:21 2018 -0700

    GEODE-5476: Migrate pipeline changes to pull request pipeline.
    
    
    Co-authored-by: Finn Southerland <fsoutherland@pivotal.io>
---
 ci/pipelines/geode-build/test-template.yml      |   1 +
 ci/pipelines/images/deploy_images_pipeline.sh   |   1 -
 ci/pipelines/pull-request/base.yml              | 151 +++-------------------
 ci/pipelines/pull-request/deploy_pr_pipeline.sh |  15 ++-
 ci/pipelines/pull-request/pr-template.yml       | 163 ++++++++++++++++--------
 ci/scripts/start_instance.sh                    |   2 +-
 6 files changed, 140 insertions(+), 193 deletions(-)

diff --git a/ci/pipelines/geode-build/test-template.yml b/ci/pipelines/geode-build/test-template.yml
index ecbb47f..a06121e 100644
--- a/ci/pipelines/geode-build/test-template.yml
+++ b/ci/pipelines/geode-build/test-template.yml
@@ -17,6 +17,7 @@
 ---
 jobs:
 - name: (( grab metadata.job.name ))
+  serial: false
   public: true
   plan:
   - do:
diff --git a/ci/pipelines/images/deploy_images_pipeline.sh b/ci/pipelines/images/deploy_images_pipeline.sh
index 7e9844c..bcbeb50 100755
--- a/ci/pipelines/images/deploy_images_pipeline.sh
+++ b/ci/pipelines/images/deploy_images_pipeline.sh
@@ -47,7 +47,6 @@ if [ "${GEODE_BRANCH}" = "HEAD" ]; then
 fi
 
 SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
-MAX_IN_FLIGHT=5
 
 BIN_DIR=${OUTPUT_DIRECTORY}/bin
 TMP_DIR=${OUTPUT_DIRECTORY}/tmp
diff --git a/ci/pipelines/pull-request/base.yml b/ci/pipelines/pull-request/base.yml
index e4812b1..92f0fbe 100644
--- a/ci/pipelines/pull-request/base.yml
+++ b/ci/pipelines/pull-request/base.yml
@@ -26,6 +26,13 @@ resource_types:
     type: docker-image
     source:
       repository: jtarchie/pr
+  - name: concourse-metadata-resource
+    type: docker-image
+    source:
+      username: ((!docker-username))
+      password: ((!docker-password))
+      repository: gcr.io/apachegeode-ci/((!docker-image-prefix))concourse-metadata-resource
+      tag: latest
 
 resources:
 - name: docker-geode-build-image
@@ -33,9 +40,15 @@ resources:
   source:
     username: ((!docker-username))
     password: ((!docker-password))
-    repository: gcr.io/apachegeode-ci/((!docker-image-name))
+    repository: gcr.io/apachegeode-ci/((!docker-image-prefix))((!docker-image-name))
+    tag: latest
+- name: alpine-tools-image
+  type: docker-image
+  source:
+    username: ((!docker-username))
+    password: ((!docker-password))
+    repository: gcr.io/apachegeode-ci/((!docker-image-prefix))alpine-tools
     tag: latest
-
 - name: geode
   type: pull-request
   source:
@@ -45,6 +58,8 @@ resources:
     ignore_paths:
     - geode-docs/*
     - geode-book/*
+- name: concourse-metadata-resource
+  type: concourse-metadata-resource
 
 groups:
 - name: main
@@ -59,7 +74,6 @@ groups:
 jobs:
 - name: Build
   serial: false
-  max_in_flight: (( grab metadata.max_in_flight ))
   public: true
   plan:
   - aggregate:
@@ -116,134 +130,3 @@ jobs:
             path: geode-unmerged-request
             status: success
             context: $BUILD_JOB_NAME
-
-- name: DistributedTest
-  serial: false
-  max_in_flight: (( grab metadata.max_in_flight ))
-  public: true
-  plan:
-  - aggregate:
-    - get: geode
-      trigger: true
-      version: every
-      params:
-        fetch_merge: true
-        git:
-          depth: 100
-    - get: docker-geode-build-image
-      params:
-        rootfs: true
-  # Unmerged pull request SHA, for writing status into GitHub
-  - get: geode-unmerged-request
-    resource: geode
-    version: every
-    params:
-      fetch_merge: false
-
-  - aggregate:
-    - put: geode
-      params:
-        path: geode-unmerged-request
-        context: $BUILD_JOB_NAME
-        status: pending
-
-    - task: run-distributed-core
-      image: docker-geode-build-image
-      tags: [pr_large]
-      privileged: true
-      timeout: 8h
-      config:
-        inputs:
-          - name: geode
-          - name: docker-geode-build-image
-        platform: linux
-        outputs:
-          - name: built-geode
-        params:
-          MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
-          SERVICE_ACCOUNT: ((!concourse-gcp-account))
-          PUBLIC_BUCKET: ((!public-bucket))
-          PARALLEL_DUNIT: true
-          DUNIT_PARALLEL_FORKS: 7
-          CALL_STACK_TIMEOUT: 25200
-        run:
-          args:
-          - geode-core:distributedTest
-          - distributedtestcore
-          path: geode/ci/scripts/test-run.sh
-
-      ensure:
-        aggregate:
-        - task: archive-results-core
-          image: docker-geode-build-image
-          config:
-            inputs:
-              - name: geode
-              - name: built-geode
-            platform: linux
-            params:
-              MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
-              SERVICE_ACCOUNT: ((!concourse-gcp-account))
-              PUBLIC_BUCKET: ((!public-bucket))
-            run:
-              args:
-              - geode-core:distributedTest
-              - distributedtestcore
-              path: geode/ci/scripts/test-archive.sh
-    - task: run-distributed-everything-else
-      image: docker-geode-build-image
-      tags: [pr_large]
-      privileged: true
-      timeout: 8h
-      config:
-        inputs:
-          - name: geode
-          - name: docker-geode-build-image
-        platform: linux
-        outputs:
-          - name: built-geode
-        params:
-          MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
-          SERVICE_ACCOUNT: ((!concourse-gcp-account))
-          PUBLIC_BUCKET: ((!public-bucket))
-          PARALLEL_DUNIT: true
-          DUNIT_PARALLEL_FORKS: 7
-          CALL_STACK_TIMEOUT: 25200
-          GRADLE_TASK_OPTIONS: -x geode-core:distributedTest
-        run:
-          args:
-          - distributedTest
-          - distributedtesteverythingelse
-          path: geode/ci/scripts/test-run.sh
-      ensure:
-        aggregate:
-        - task: archive-results-everythingelse
-          image: docker-geode-build-image
-          config:
-            inputs:
-              - name: geode
-              - name: built-geode
-            platform: linux
-            params:
-              MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
-              SERVICE_ACCOUNT: ((!concourse-gcp-account))
-              PUBLIC_BUCKET: ((!public-bucket))
-            run:
-              args:
-              - disitributedTest
-              - distributedtesteverythingelse
-              path: geode/ci/scripts/test-archive.sh
-    on_failure:
-      aggregate:
-      - put: geode
-        params:
-          path: geode-unmerged-request
-          status: failure
-          context: $BUILD_JOB_NAME
-    on_success:
-      aggregate:
-      - put: geode
-        params:
-          path: geode-unmerged-request
-          context: $BUILD_JOB_NAME
-          status: success
diff --git a/ci/pipelines/pull-request/deploy_pr_pipeline.sh b/ci/pipelines/pull-request/deploy_pr_pipeline.sh
index da2a2e1..440828e 100755
--- a/ci/pipelines/pull-request/deploy_pr_pipeline.sh
+++ b/ci/pipelines/pull-request/deploy_pr_pipeline.sh
@@ -47,7 +47,6 @@ if [ "${GEODE_BRANCH}" = "HEAD" ]; then
 fi
 
 SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
-MAX_IN_FLIGHT=5
 
 BIN_DIR=${OUTPUT_DIRECTORY}/bin
 TMP_DIR=${OUTPUT_DIRECTORY}/tmp
@@ -64,7 +63,7 @@ for i in ${GEODEBUILDDIR}/test-stubs/*.yml; do
     <(echo "metadata:"; \
       echo "  geode-build-branch: ${GEODE_BRANCH}"; \
       echo "  geode-fork: ${GEODE_FORK}"; \
-      echo "  max_in_flight: ${MAX_IN_FLIGHT}") \
+      echo "  ") \
     ${SCRIPTDIR}/pr-template.yml \
     ${i} > ${TMP_DIR}/${X}
 done
@@ -75,7 +74,6 @@ ${SPRUCE} merge --prune metadata \
   <(echo "metadata:"; \
     echo "  geode-build-branch: ${GEODE_BRANCH}"; \
     echo "  geode-fork: ${GEODE_FORK}"; \
-    echo "  max_in_flight: ${MAX_IN_FLIGHT}"; \
     echo "  ") \
   ${TMP_DIR}/*.yml > ${TMP_DIR}/final.yml
 
@@ -88,10 +86,19 @@ TEAM=${CONCOURSE_TEAM}
 #  TEAM="main"
 #fi
 
+if [[ "${GEODE_FORK}" == "apache" ]]; then
+  PIPELINE_PREFIX=""
+  DOCKER_IMAGE_PREFIX=""
+else
+  PIPELINE_PREFIX="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
+  DOCKER_IMAGE_PREFIX=${PIPELINE_PREFIX}
+fi
+
 fly login -t ${TARGET} -n ${TEAM} -c https://concourse.apachegeode-ci.info -u ${CONCOURSE_USERNAME}
-p ${CONCOURSE_PASSWORD}
 fly -t ${TARGET} set-pipeline \
   --non-interactive \
-  --pipeline pr-${SANITIZED_GEODE_BRANCH} \
+  --pipeline ${PIPELINE_PREFIX}pr \
   --config ${TMP_DIR}/final.yml \
+  --var docker-image-prefix=${DOCKER_IMAGE_PREFIX} \
   --var concourse-team=${TEAM}
 
diff --git a/ci/pipelines/pull-request/pr-template.yml b/ci/pipelines/pull-request/pr-template.yml
index f91f216..919a2f7 100644
--- a/ci/pipelines/pull-request/pr-template.yml
+++ b/ci/pipelines/pull-request/pr-template.yml
@@ -15,14 +15,14 @@
 # limitations under the License.
 
 ---
-
 jobs:
 - name: (( grab metadata.job.name ))
   serial: false
-  max_in_flight: (( grab metadata.max_in_flight ))
   public: true
   plan:
+  - do:
     - aggregate:
+      - put: concourse-metadata-resource
       - get: geode
         trigger: true
         version: every
@@ -30,7 +30,7 @@ jobs:
           fetch_merge: true
           git:
             depth: 100
-      - get: docker-geode-build-image
+      - get: alpine-tools-image
         params:
           rootfs: true
     # Unmerged pull request SHA, for writing status into GitHub
@@ -46,60 +46,117 @@ jobs:
           context: $BUILD_JOB_NAME
           status: pending
 
-      - task: runtests
-        image: docker-geode-build-image
-        tags: [pr]
-        privileged: true
-        timeout: (( grab metadata.job.timeout ))
+    - task: start_instance
+      image: alpine-tools-image
+      timeout: 5m
+      config:
+        inputs:
+          - name: concourse-metadata-resource
+          - name: geode
+        outputs:
+          - name: instance-data
+        platform: linux
+        params:
+          CPUS: (( grab metadata.job.cpus ))
+          RAM: (( grab metadata.job.ram ))
+          GEODE_BRANCH: (( grab metadata.geode-build-branch ))
+          GEODE_FORK: (( grab metadata.geode-fork ))
+        run:
+          path: geode/ci/scripts/start_instance.sh
+    - task: rsync_code_up
+      image: alpine-tools-image
+      timeout: 5m
+      config:
+        inputs:
+          - name: geode
+          - name: instance-data
+        platform: linux
+        run:
+          path: geode/ci/scripts/rsync_code_up.sh
+    - task: prepopulate_build
+      image: alpine-tools-image
+      config:
+        inputs:
+          - name: geode
+          - name: instance-data
+        platform: linux
+        run:
+          path: geode/ci/scripts/prepopulate_build.sh
+    - task: execute_tests
+      image: alpine-tools-image
+      timeout: (( grab metadata.job.timeout ))
+      tags: [pr]
+      config:
+        inputs:
+          - name: geode
+          - name: instance-data
+        platform: linux
+        params:
+          MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
+          SERVICE_ACCOUNT: ((!concourse-gcp-account))
+          PUBLIC_BUCKET: ((!public-bucket))
+          PARALLEL_DUNIT: (( grab metadata.job.dunit.parallel ))
+          DUNIT_PARALLEL_FORKS: (( grab metadata.job.dunit.forks ))
+          CALL_STACK_TIMEOUT: (( grab metadata.job.call_stack_timeout ))
+          GRADLE_TASK: (( grab metadata.job.gradle_task ))
+          GRADLE_TASK_OPTIONS: (( grab metadata.job.gradle_task_options || "" ))
+          ARTIFACT_SLUG: (( grab metadata.job.artifact_slug ))
+        run:
+          path: geode/ci/scripts/execute_tests.sh
+    ensure:
+      do:
+      - task: rsync_code_down
+        image: alpine-tools-image
+        timeout: 5m
         config:
           inputs:
             - name: geode
-            - name: docker-geode-build-image
-          platform: linux
+            - name: instance-data
           outputs:
-            - name: built-geode
-          params:
-            MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
-            SERVICE_ACCOUNT: ((!concourse-gcp-account))
-            PUBLIC_BUCKET: ((!public-bucket))
-            PARALLEL_DUNIT: (( grab metadata.job.dunit.parallel ))
-            DUNIT_PARALLEL_FORKS: (( grab metadata.job.dunit.forks ))
-            CALL_STACK_TIMEOUT: (( grab metadata.job.call_stack_timeout ))
+            - name: geode-results
+          platform: linux
           run:
-            args:
-            - (( grab metadata.job.gradle_task ))
-            - (( grab metadata.job.artifact_slug ))
-            path: geode/ci/scripts/test-run.sh
-        on_failure:
-          aggregate:
-          - put: geode
-            params:
-              path: geode-unmerged-request
-              status: failure
-              context: $BUILD_JOB_NAME
-        on_success:
-          aggregate:
-          - put: geode
+            path: geode/ci/scripts/rsync_code_down.sh
+      ensure:
+        aggregate:
+        - task: archive-results
+          image: alpine-tools-image
+          config:
+            inputs:
+              - name: concourse-metadata-resource
+              - name: geode
+              - name: geode-results
+            platform: linux
             params:
-              path: geode-unmerged-request
-              context: $BUILD_JOB_NAME
-              status: success
+              MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
+              SERVICE_ACCOUNT: ((!concourse-gcp-account))
+              PUBLIC_BUCKET: ((!public-bucket))
+              GRADLE_TASK: (( grab metadata.job.gradle_task ))
+              ARTIFACT_SLUG: (( grab metadata.job.artifact_slug ))
+            run:
+              path: geode/ci/scripts/archive_results.sh
+        - task: stop_instance
+          image: alpine-tools-image
+          timeout: 1h
+          config:
+            inputs:
+              - name: geode
+              - name: instance-data
+            platform: linux
+            run:
+              path: geode/ci/scripts/stop_instance.sh
+    on_failure:
+      aggregate:
+      - put: geode
+        params:
+          path: geode-unmerged-request
+          status: failure
+          context: $BUILD_JOB_NAME
+    on_success:
+      aggregate:
+      - put: geode
+        params:
+          path: geode-unmerged-request
+          context: $BUILD_JOB_NAME
+          status: success
 
-        ensure:
-          aggregate:
-          - task: archive-results
-            image: docker-geode-build-image
-            config:
-              inputs:
-                - name: geode
-                - name: built-geode
-              platform: linux
-              params:
-                MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
-                SERVICE_ACCOUNT: ((!concourse-gcp-account))
-                PUBLIC_BUCKET: ((!public-bucket))
-              run:
-                args:
-                - (( grab metadata.job.gradle_task ))
-                - (( grab metadata.job.artifact_slug ))
-                path: geode/ci/scripts/test-archive.sh
diff --git a/ci/scripts/start_instance.sh b/ci/scripts/start_instance.sh
index 261ab86..9e72c8b 100755
--- a/ci/scripts/start_instance.sh
+++ b/ci/scripts/start_instance.sh
@@ -52,7 +52,7 @@ if [[ "${GEODE_FORK}" != "apache" ]]; then
   IMAGE_FAMILY_PREFIX="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
 fi
 
-INSTANCE_NAME="$(echo "geode-builder-${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}"
| tr '[:upper:]' '[:lower:]')"
+INSTANCE_NAME="$(echo "build-${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" | tr
'[:upper:]' '[:lower:]')"
 PROJECT=apachegeode-ci
 ZONE=us-central1-f
 echo "${INSTANCE_NAME}" > "instance-data/instance-name"


Mime
View raw message