accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [accumulo] branch main updated: Create on-demand build for all ITs as a matrix
Date Thu, 24 Sep 2020 19:27:24 GMT
This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/main by this push:
     new 6986a42  Create on-demand build for all ITs as a matrix
6986a42 is described below

commit 6986a427fd78e82909e89dd079970e27c5922094
Author: Christopher Tubbs <ctubbsii@apache.org>
AuthorDate: Thu Sep 24 15:18:09 2020 -0400

    Create on-demand build for all ITs as a matrix
---
 .../workflows/{maven.yaml => maven-full-its.yaml}  | 42 ++++++++++++++--------
 .github/workflows/maven-on-demand.yaml             |  4 ++-
 .github/workflows/maven.yaml                       |  3 +-
 contrib/ci/it-matrix.sh                            | 38 ++++++++++++++++++++
 4 files changed, 70 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven-full-its.yaml
similarity index 71%
copy from .github/workflows/maven.yaml
copy to .github/workflows/maven-full-its.yaml
index ee5f19c..658f717 100644
--- a/.github/workflows/maven.yaml
+++ b/.github/workflows/maven-full-its.yaml
@@ -18,15 +18,20 @@
 #
 
 # This workflow will build a Java project with Maven
-# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# See also:
+#   https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+#   https://docs.github.com/en/actions/reference/events-that-trigger-workflows#manual-events
+#   https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#fromjson
 
-name: QA
+name: Full ITs
 
 on:
-  push:
-    branches: [ '*' ]
-  pull_request:
-    branches: [ '*' ]
+  workflow_dispatch:
+    inputs:
+      numITsPerTask:
+        description: Number of ITs per task
+        required: true
+        default: 15
 
 jobs:
   # fast build to populate the local maven repository cache
@@ -51,15 +56,22 @@ jobs:
       run: mvn -B -V -e -ntp "-Dstyle.color=always" clean package dependency:resolve -PskipQA
       env:
         MAVEN_OPTS: -Djansi.force=true
-  # more complete builds with tests
-  mvn:
+  creatematrix:
     needs: fastbuild
+    timeout-minutes: 5
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - id: set-matrix
+      name: Create the IT build matrix
+      run: contrib/ci/it-matrix.sh ${{ github.event.inputs.numITsPerTask }}
+  # targeted builds that tests groups of ITs
+  mvn:
+    needs: creatematrix
     strategy:
-      matrix:
-        profile:
-          - {name: 'unit-tests',    args: 'verify -PskipQA -DskipTests=false -DforkCount=1C'}
-          - {name: 'qa-checks',     args: 'verify javadoc:jar -Psec-bugs -DskipTests=true
-Dspotbugs.timeout=3600000'}
-          - {name: 'hadoop-compat', args: 'package -DskipTests -Dhadoop.version=3.0.3'}
+      matrix: ${{ fromJson(needs.creatematrix.outputs.matrix) }}
       fail-fast: false
     timeout-minutes: 60
     runs-on: ubuntu-latest
@@ -77,8 +89,8 @@ jobs:
           !~/.m2/repository/org/apache/accumulo
         key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
         restore-keys: ${{ runner.os }}-m2
-    - name: Build with Maven (${{ matrix.profile.name }})
-      run: mvn -B -V -e -ntp "-Dstyle.color=always" ${{ matrix.profile.args }}
+    - name: Build with Maven (${{ matrix.profile.its }})
+      run: mvn -B -V -e -ntp "-Dstyle.color=always" verify -PskipQA -DskipTests=false -DskipITs=false
-Dtest=nomatchingtest -Dit.test="${{ matrix.profile.its }}"
       env:
         MAVEN_OPTS: -Djansi.force=true
     - name: Upload unit test results
diff --git a/.github/workflows/maven-on-demand.yaml b/.github/workflows/maven-on-demand.yaml
index 34af0df..ef50d43 100644
--- a/.github/workflows/maven-on-demand.yaml
+++ b/.github/workflows/maven-on-demand.yaml
@@ -18,7 +18,9 @@
 #
 
 # This workflow will build a Java project with Maven
-# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# See also:
+#   https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+#   https://docs.github.com/en/actions/reference/events-that-trigger-workflows#manual-events
 
 name: Manual Build
 
diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml
index ee5f19c..b52420f 100644
--- a/.github/workflows/maven.yaml
+++ b/.github/workflows/maven.yaml
@@ -18,7 +18,8 @@
 #
 
 # This workflow will build a Java project with Maven
-# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# See also:
+#   https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
 
 name: QA
 
diff --git a/contrib/ci/it-matrix.sh b/contrib/ci/it-matrix.sh
new file mode 100755
index 0000000..89a887d
--- /dev/null
+++ b/contrib/ci/it-matrix.sh
@@ -0,0 +1,38 @@
+#! /usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+testsPerJob=15
+if [[ -n $1 && $1 =~ ^[0-9]*$ ]]; then
+  testsPerJob=$1
+fi
+echo "Creating matrix (tests per job: $testsPerJob)..."
+
+gitRootDir=$(git rev-parse --show-toplevel)
+# this only works because our test paths don't have spaces; we should keep it that way
+count=0
+echo -n '::set-output name=matrix::{"profile":['
+for x in $(find "$gitRootDir" -name '*IT.java' -exec basename '{}' .java \; | sort -u | xargs
-n "$testsPerJob" | tr ' ' ','); do
+  [[ $count -gt 0 ]] && echo -n ','
+  echo -n "{\"name\":\"task_$count\",\"its\":\"$x\"}"
+  ((count=count+1))
+done
+echo ']}'
+echo "Finished creating matrix ($count tasks)"
+


Mime
View raw message