geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mhan...@apache.org
Subject [geode] 02/04: Separate benchmark tasks into jobs. (#4404)
Date Fri, 27 Dec 2019 23:57:19 GMT
This is an automated email from the ASF dual-hosted git repository.

mhanson pushed a commit to branch release/1.11.0
in repository https://gitbox.apache.org/repos/asf/geode.git

commit 3621f68a69ee01b79483783e4aac05daa1bfe3d8
Author: Robert Houghton <rhoughton@pivotal.io>
AuthorDate: Wed Dec 4 12:12:13 2019 -0800

    Separate benchmark tasks into jobs. (#4404)
    
    * Separate benchmark tasks into jobs.
    
    To enable more granular re-run in Concourse, split the Benchmarks into
    independently triggerable jobs in the main pipeline.
    Extract variables from the template into the jinja.variables file.
    Make better use of YML anchors.
    
    Co-authored-by: Robert Houghton <rhoughton@pivotal.io>
    Co-authored-by: Helena Bales <hbales@pivotal.io>
    
    * empty to trigger pr
    
    (cherry picked from commit 1fcadf987706af7515a64869e088e1573695dae0)
---
 ci/pipelines/geode-build/jinja.template.yml | 92 ++++++++++++++++-------------
 ci/pipelines/shared/jinja.variables.yml     | 10 ++++
 2 files changed, 60 insertions(+), 42 deletions(-)

diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
index a400a4a..b33fb8e 100644
--- a/ci/pipelines/geode-build/jinja.template.yml
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -100,7 +100,9 @@ groups:
   {%- if repository.upstream_fork != "apache" or repository.branch == "develop" %}
   - PublishArtifacts
   {%- endif %}
-  - Benchmark
+  {%- for flavor in (benchmarks.flavors) %}
+  - Benchmark{{flavor.title}}
+  {%- endfor %}
 - name: complete
   jobs:
   - {{ build_test.name }}
@@ -113,7 +115,9 @@ groups:
   {%- if repository.upstream_fork != "apache" or repository.branch == "develop" %}
   - PublishArtifacts
   {%- endif %}
-  - Benchmark
+  {%- for flavor in (benchmarks.flavors) %}
+  - Benchmark{{flavor.title}}
+  {%- endfor %}
 - name: linux
   jobs:
   - {{ build_test.name }}
@@ -122,7 +126,9 @@ groups:
   - {{test.name}}Test{{java_test_version.name}}
     {%- endfor -%}
   {%- endfor %}
-  - Benchmark
+  {%- for flavor in (benchmarks.flavors) %}
+  - Benchmark{{flavor.title}}
+  {%- endfor %}
 - name: windows
   jobs:
   - {{ build_test.name }}
@@ -362,7 +368,9 @@ jobs:
 {%- if repository.upstream_fork != "apache" or repository.branch == "develop" %}
       - PublishArtifacts
 {% else %}
-      - Benchmark
+      {% for flavor in (benchmarks.flavors) %}
+      - Benchmark{{flavor.title}}
+      {% endfor %}
 {% endif %}
       trigger: true
     - get: geode-build-version
@@ -381,7 +389,7 @@ jobs:
         path: bash
         args:
         - -cx
-        - |-
+        - |
           pushd geode
             GEODE_SHA=$(git rev-parse HEAD)
           popd
@@ -405,15 +413,16 @@ jobs:
       params:
         file: geode-passing-tokens/passing-build-tokens.json
 
-- name: Benchmark
+{% for run_var in (benchmarks.flavors) %}
+- name: Benchmark{{ run_var.title }}
   public: true
   max_in_flight: 3
   plan:
   - get: geode-ci
     passed:
     {{ all_gating_jobs() | indent(4) }}
+  - get: alpine-tools-image
   - aggregate:
-    - get: alpine-tools-image
     - get: geode
       passed:
       {{ all_gating_jobs() | indent(6) }}
@@ -423,8 +432,7 @@ jobs:
       passed:
       {{ all_gating_jobs() | indent(6) }}
     - put: concourse-metadata-resource
-  - aggregate:
-    {% for run_var in [{'title': '', 'flag':'', 'options': ''}, {'title': '_with_ssl', 'flag':
'-PwithSsl', 'options': '--tests=*GetBenchmark --tests=*PutBenchmark'}, {'title': '_with_security_manager',
'flag': '-PwithSecurityManager', 'options': '--tests=Partitioned*'}] %}
+  - do:
     - task: run_benchmarks{{ run_var.title }}
       image: alpine-tools-image
       config:
@@ -476,50 +484,50 @@ jobs:
             - name: geode-benchmarks
             - name: concourse-metadata-resource
             - name: results
-    {% endfor %}
+{% endfor %}
 {% if repository.upstream_fork != "apache" or repository.branch == "develop" %}
 - name: PublishArtifacts
   public: true
   plan:
   - aggregate:
     - get: geode-ci
-      passed:
-      - Benchmark
+      passed: &publish-passed-inputs
+      {% for flavor in (benchmarks.flavors) %}
+      - Benchmark{{flavor.title}}
+      {% endfor %}
     - get: alpine-tools-image
     - get: geode
-      passed:
-        - Benchmark
+      passed: *publish-passed-inputs
       trigger: true
   - aggregate:
     - get: geode-build-version
-      passed:
-      - Benchmark
-    - do:
-      - put: concourse-metadata-resource
-      {{ init_retry()|indent(6) }}
-      - task: create_instance
-        image: alpine-tools-image
-        config:
-          platform: linux
-          params:
-            {{ common_instance_params(publish_artifacts) | indent(12) }}
-            GEODE_BRANCH: {{repository.branch}}
-            GEODE_FORK: {{repository.fork}}
-            JAVA_BUILD_VERSION: {{ java_build_version.version }}
-          run:
-            path: geode-ci/ci/scripts/create_instance.sh
-          inputs:
-          - name: concourse-metadata-resource
-          - name: geode-ci
-          - name: geode
-          - name: attempts-log
-            path: old
-          outputs:
-          - name: instance-data
-          - name: attempts-log
-            path: new
-        timeout: 15m
-        attempts: 10
+      trigger: true
+      passed: *publish-passed-inputs
+    - put: concourse-metadata-resource
+    {{ init_retry()|indent(4) }}
+  - task: create_instance
+    image: alpine-tools-image
+    config:
+      platform: linux
+      params:
+        {{ common_instance_params(publish_artifacts) | indent(8) }}
+        GEODE_BRANCH: {{repository.branch}}
+        GEODE_FORK: {{repository.fork}}
+        JAVA_BUILD_VERSION: {{ java_build_version.version }}
+      run:
+        path: geode-ci/ci/scripts/create_instance.sh
+      inputs:
+      - name: concourse-metadata-resource
+      - name: geode-ci
+      - name: geode
+      - name: attempts-log
+        path: old
+      outputs:
+      - name: instance-data
+      - name: attempts-log
+        path: new
+    timeout: 15m
+    attempts: 10
   - task: rsync_code_up
     image: alpine-tools-image
     config:
diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml
index 4b956b7..24fbcf3 100644
--- a/ci/pipelines/shared/jinja.variables.yml
+++ b/ci/pipelines/shared/jinja.variables.yml
@@ -19,6 +19,16 @@ benchmarks:
   baseline_branch: ''
   baseline_version: '1.10.0'
   benchmark_branch: develop
+  flavors:
+  - title: '_base'
+    flag: ''
+    options: ''
+  - title: '_with_ssl'
+    flag: '-PwithSsl'
+    options: '--tests=*GetBenchmark --tests=*PutBenchmark'
+  - title: '_with_security_manager'
+    flag: '-PwithSecurityManager'
+    options: '--tests=Partitioned*'
 
 build_test:
   ARTIFACT_SLUG: build


Mime
View raw message