geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jensde...@apache.org
Subject [geode] branch develop updated: Add files necessary for Concourse CI infrastructure. (#1006)
Date Thu, 02 Nov 2017 17:23:28 GMT
This is an automated email from the ASF dual-hosted git repository.

jensdeppe pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git


The following commit(s) were added to refs/heads/develop by this push:
     new 891274c  Add files necessary for Concourse CI infrastructure. (#1006)
891274c is described below

commit 891274c0b20c795be998889d1483fcb8c8dfc36b
Author: Sean Goller <sean@goller.net>
AuthorDate: Thu Nov 2 10:23:26 2017 -0700

    Add files necessary for Concourse CI infrastructure. (#1006)
    
    * Implements GEODE-3942.
    * Add pipelines for develop, building the docker images that builds geode,
      and updating those those pipelines.
    * Add build script.
    * Add Dockerfile for building geode in Concourse.
---
 ci/docker/Dockerfile            |  37 ++++++++
 ci/docker/cache_dependencies.sh |  39 +++++++++
 ci/pipelines/develop.yml        | 110 +++++++++++++++++++++++
 ci/pipelines/docker-images.yml  |  49 +++++++++++
 ci/pipelines/meta.yml           |  74 ++++++++++++++++
 ci/scripts/build.sh             | 189 ++++++++++++++++++++++++++++++++++++++++
 6 files changed, 498 insertions(+)

diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile
new file mode 100644
index 0000000..70e3b5b
--- /dev/null
+++ b/ci/docker/Dockerfile
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM apachegeode/geode-build
+ENTRYPOINT []
+
+# Setup Google APT repository
+RUN apt-get update && \
+    apt-get install -y lsb-release && \
+    echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" |
tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - &&
\
+    apt-get update && apt-get install -y google-cloud-sdk && \
+    apt-get clean
+
+# For a CI tool, disable updates to gcloud since they'll be thrown away at end of run
+RUN gcloud config set core/disable_usage_reporting true && \
+    gcloud config set component_manager/disable_update_check true && \
+    gcloud config set metrics/environment github_docker_image
+
+ENV GRADLE_USER_HOME /usr/local/maven_files
+WORKDIR /tmp/work
+ADD cache_dependencies.sh cache_dependencies.sh
+RUN chmod +x cache_dependencies.sh
+RUN ./cache_dependencies.sh && rm -rf /tmp/work
+
diff --git a/ci/docker/cache_dependencies.sh b/ci/docker/cache_dependencies.sh
new file mode 100644
index 0000000..ade203e
--- /dev/null
+++ b/ci/docker/cache_dependencies.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+git clone -b develop --depth 1 https://github.com/apache/geode.git geode
+
+pushd geode
+  cat << EOF >> build.gradle
+  subprojects {
+    task getDeps(type: Copy) {
+      from project.sourceSets.main.runtimeClasspath
+      from project.sourceSets.test.runtimeClasspath
+      from configurations.testRuntime
+      into 'runtime/'
+    }
+  }
+EOF
+
+# Include rat to get its runtime dependencies, which are apparently not captured by the 'getDeps'
task above
+./gradlew --no-daemon :rat getDeps
+
+popd
+
+rm -rf geode
diff --git a/ci/pipelines/develop.yml b/ci/pipelines/develop.yml
new file mode 100644
index 0000000..8f11ed0
--- /dev/null
+++ b/ci/pipelines/develop.yml
@@ -0,0 +1,110 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+resource_types:
+  - name: gcs-resource
+    type: docker-image
+    source:
+      repository: frodenas/gcs-resource
+  - name: email
+    type: docker-image
+    source:
+      repository: pcfseceng/email-resource
+
+resources:
+- name: geode
+  type: git
+  source:
+    uri: https://github.com/apache/geode.git
+    branch: develop
+- name: geode-ci
+  type: git
+  source:
+    depth: 1
+    uri: https://github.com/apache/geode.git
+    branch: develop
+    paths:
+    - ci/*
+- name: geode-build-version
+  type: semver
+  source:
+    driver: gcs
+    bucket: ((!concourse-bucket))
+    key: ((!version-key))
+    json_key: ((!concourse-gcp-key))
+    initial_version: 1.3.0
+- name: geode-build-artifact
+  type: gcs-resource
+  source:
+    bucket: ((!public-bucket))
+    json_key: ((!concourse-gcp-key))
+    regexp: artifacts/develop/geodefiles-(.*).tgz
+- name: send-notification-email
+  type: email
+  source:
+    smtp:
+      host: ((!source-email-server))
+      port: "587"
+      username: ((!source-email-username))
+      password: ((!source-email-password))
+    from: ((!source-email-address))
+    to: [ ((!notification-email-address)) ]
+jobs:
+- name: Build
+  serial: true
+  public: true
+  plan:
+    - get: geode
+      trigger: true
+    - get: geode-ci
+    - get: geode-build-version
+      params: {pre: build}
+    - task: build
+      config:
+        platform: linux
+        inputs:
+        - name: geode
+        - name: geode-ci
+        - name: geode-build-version
+        outputs:
+        - name: built-geode
+        - name: results
+        image_resource:
+          type: docker-image
+          source:
+            username: ((!docker-username))
+            password: ((!docker-password))
+            repository: gcr.io/apachegeode-ci/apachegeode-build-concourse
+            tag: latest
+        params:
+          MAINTENANCE_VERSION: ((!maintenance-version))
+          SERVICE_ACCOUNT: ((!concourse-gcp-account))
+        run:
+          path: geode-ci/ci/scripts/build.sh
+      ensure:
+        aggregate:
+          - put: send-notification-email
+            params:
+              subject: results/subject
+              body: results/body
+          - put: geode-build-artifact
+            params:
+              file: built-geode/geodefiles-*.tgz
+          - put: geode-build-version
+            params:
+              file: results/number
diff --git a/ci/pipelines/docker-images.yml b/ci/pipelines/docker-images.yml
new file mode 100644
index 0000000..2b2ed19
--- /dev/null
+++ b/ci/pipelines/docker-images.yml
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+resources:
+# apachegeode-build-concourse resources
+- name: apachegeode-build-concourse-dockerfile
+  type: git
+  source:
+    uri: https://github.com/apache/geode.git
+    branch: create-docker-image
+    paths:
+    - ci/docker/*
+
+- name: apachegeode-build-concourse-docker-image
+  type: docker-image
+  source:
+    username: ((!docker-username))
+    password: ((!docker-password))
+    repository: gcr.io/apachegeode-ci/apachegeode-build-concourse
+
+jobs:
+# apachegeode-build-concourse
+- name: build-apachegeode-build-concourse-docker-image
+  public: true
+  serial: true
+  plan:
+  - aggregate:
+    - get: apachegeode-build-concourse-dockerfile
+      trigger: true
+  - put: apachegeode-build-concourse-docker-image
+    params:
+      build: apachegeode-build-concourse-dockerfile/ci/docker
+      tag_as_latest: true
+
diff --git a/ci/pipelines/meta.yml b/ci/pipelines/meta.yml
new file mode 100644
index 0000000..baad9fc
--- /dev/null
+++ b/ci/pipelines/meta.yml
@@ -0,0 +1,74 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+resource_types:
+- name: concourse-pipeline
+  type: docker-image
+  source:
+    repository: concourse/concourse-pipeline-resource
+
+resources:
+- name: apachegeode-concourse
+  type: concourse-pipeline
+  source:
+    target: https://concourse.apachegeode-ci.info
+    insecure: "true"
+    teams:
+    - name: main
+      username: ((!concourse-username))
+      password: ((!concourse-password))
+- name: geode-develop-pipeline
+  type: git
+  source:
+    uri: https://github.com/apache/geode.git
+    branch: create-docker-image
+    paths:
+    - ci/pipelines/develop.yml
+- name: geode-docker-images-pipeline
+  type: git
+  source:
+    uri: https://github.com/apache/geode.git
+    branch: create-docker-image
+    paths:
+    - ci/pipelines/docker-images.yml
+
+jobs:
+- name: set-develop-pipeline
+  serial: true
+  public: true
+  plan:
+  - get: geode-develop-pipeline
+    trigger: true
+  - put: apachegeode-concourse
+    params:
+      pipelines:
+      - name: develop
+        team: main
+        config_file: geode-develop-pipeline/ci/pipelines/develop.yml
+- name: set-docker-images-pipeline
+  serial: true
+  public: true
+  plan:
+  - get: geode-docker-images-pipeline
+    trigger: true
+  - put: apachegeode-concourse
+    params:
+      pipelines:
+      - name: docker-images
+        team: main
+        config_file: geode-docker-images-pipeline/ci/pipelines/docker-images.yml
diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh
new file mode 100755
index 0000000..595ca25
--- /dev/null
+++ b/ci/scripts/build.sh
@@ -0,0 +1,189 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+set -e
+ROOT_DIR=$(pwd)
+BUILD_DATE=$(date +%s)
+EMAIL_SUBJECT="results/subject"
+EMAIL_BODY="results/body"
+
+GEODE_BUILD_VERSION_FILE=${ROOT_DIR}/geode-build-version/number
+GEODE_RESULTS_VERSION_FILE=${ROOT_DIR}/results/number
+GEODE_BUILD_VERSION_NUMBER=$(grep "versionNumber *=" geode/gradle.properties | awk -F "="
'{print $2}' | tr -d ' ')
+GEODE_BUILD_DIR=/tmp/geode-build
+
+if [ ! -e "${GEODE_BUILD_VERSION_FILE}" ]; then
+  echo "${GEODE_BUILD_VERSION_FILE} file does not exist. Concourse is probably not configured
correctly."
+  exit 1
+fi
+if [ -z ${MAINTENANCE_VERSION+x} ]; then
+  echo "MAINTENANCE_VERSION is unset. Check your pipeline configuration and make sure this
script is called properly."
+  exit 1
+fi
+if [ -z ${SERVICE_ACCOUNT+x} ]; then
+  echo "SERVICE_ACCOUNT is unset. Check your pipeline configuration and make sure this script
is called properly."
+  exit 1
+fi
+
+if [ -z ${GEODE_BUILD_VERSION_NUMBER+x} ]; then
+  echo "gradle.properties does not seem to contain a valid versionNumber. Please check the
source tree."
+  exit 1
+fi
+
+CONCOURSE_VERSION=$(cat ${GEODE_BUILD_VERSION_FILE})
+CONCOURSE_PRODUCT_VERSION=${CONCOURSE_VERSION%%-*}
+GEODE_PRODUCT_VERSION=${GEODE_BUILD_VERSION_NUMBER}
+CONCOURSE_BUILD_SLUG=${CONCOURSE_VERSION##*-}
+BUILD_ID=${CONCOURSE_VERSION##*.}
+FULL_PRODUCT_VERSION=${GEODE_PRODUCT_VERSION}-${CONCOURSE_BUILD_SLUG}
+echo -n "${FULL_PRODUCT_VERSION}" > ${GEODE_RESULTS_VERSION_FILE}
+
+echo "Concourse VERSION is ${CONCOURSE_VERSION}"
+echo "Geode product VERSION is ${GEODE_PRODUCT_VERSION}"
+echo "Build ID is ${BUILD_ID}"
+
+printf "\nUsing the following JDK:"
+java -version
+printf "\n\n"
+
+gcloud config set account ${SERVICE_ACCOUNT}
+
+export TERM=${TERM:-dumb}
+export DEST_DIR=${ROOT_DIR}/built-geode
+export TMPDIR=${DEST_DIR}/tmp
+mkdir -p ${TMPDIR}
+export BUILD_ARTIFACTS_DIR=${DEST_DIR}/test-artifacts
+mkdir -p ${BUILD_ARTIFACTS_DIR}
+
+ln -s ${ROOT_DIR}/geode ${GEODE_BUILD_DIR}
+
+pushd ${GEODE_BUILD_DIR}
+set +e
+./gradlew --no-daemon -PbuildId=${BUILD_ID} --system-prop "java.io.tmpdir=${TMPDIR}" build
+GRADLE_EXIT_STATUS=$?
+set -e
+
+popd
+ARTIFACTS_DESTINATION="files.apachegeode-ci.info/builds/${FULL_PRODUCT_VERSION}"
+TEST_RESULTS_DESTINATION="${ARTIFACTS_DESTINATION}/test-results/"
+FULL_BUILD_ARCHIVE_DESTINATION="${ARTIFACTS_DESTINATION}/geodefiles-${FULL_PRODUCT_VERSION}.tgz"
+BUILD_ARTIFACTS_FILENAME=geode-build-artifacts-${FULL_PRODUCT_VERSION}.tgz
+BUILD_ARTIFACTS_DESTINATION="${ARTIFACTS_DESTINATION}/${BUILD_ARTIFACTS_FILENAME}"
+
+function sendSuccessfulJobEmail {
+  echo "Sending job success email"
+
+  cat <<EOF >${EMAIL_SUBJECT}
+Build for version ${PRODUCT_VERSION} of Apache Geode succeeded.
+EOF
+
+  cat <<EOF >${EMAIL_BODY}
+=================================================================================================
+
+The build job for Apache Geode version ${FULL_PRODUCT_VERSION} has completed successfully.
+
+
+Build artifacts are available at:
+http://${BUILD_ARTIFACTS_DESTINATION}
+
+Test results are available at:
+http://${TEST_RESULTS_DESTINATION}
+
+
+=================================================================================================
+EOF
+
+}
+
+function sendFailureJobEmail {
+  echo "Sending job failure email"
+
+  cat <<EOF >${EMAIL_SUBJECT}
+Build for version ${PRODUCT_VERSION} of Apache Geode failed.
+EOF
+
+  cat <<EOF >${EMAIL_BODY}
+=================================================================================================
+
+The build job for Apache Geode version ${FULL_PRODUCT_VERSION} has failed.
+
+
+Build artifacts are available at:
+http://${BUILD_ARTIFACTS_DESTINATION}
+
+Test results are available at:
+http://${TEST_RESULTS_DESTINATION}
+
+
+Job: \${ATC_EXTERNAL_URL}/teams/\${BUILD_TEAM_NAME}/pipelines/\${BUILD_PIPELINE_NAME}/jobs/\${BUILD_JOB_NAME}/builds/\${BUILD_NAME}
+
+=================================================================================================
+EOF
+
+}
+
+if [ ! -d "geode/build/reports/combined" ]; then
+    echo "No tests exist, compile failed."
+    mkdir -p geode/build/reports/combined
+    echo "<html><head><title>No Test Results Were Captured</title></head><body><h1>No
Test Results Were Captured</h1></body></html>" > geode/build/reports/combined/index.html
+fi
+
+pushd geode/build/reports/combined
+gsutil -q -m cp -r * gs://${TEST_RESULTS_DESTINATION}
+popd
+
+echo ""
+printf "\033[92m=-=-=-=-=-=-=-=-=-=-=-=-=-=  Test Results Website =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\033[0m\n"
+printf "\033[92mhttp://${TEST_RESULTS_DESTINATION}\033[0m\n"
+printf "\033[92m=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\033[0m\n"
+printf "\n"
+
+tar zcf ${DEST_DIR}/geodefiles-${FULL_PRODUCT_VERSION}.tgz geode
+gsutil cp ${DEST_DIR}/geodefiles-${FULL_PRODUCT_VERSION}.tgz gs://${FULL_BUILD_ARCHIVE_DESTINATION}
+cp -r geode/geode-assembly/build/distributions ${BUILD_ARTIFACTS_DIR}/
+cp -r geode/build/reports/rat ${BUILD_ARTIFACTS_DIR}/
+cp -r geode/build/reports/combined ${BUILD_ARTIFACTS_DIR}/
+
+directories_file=${DEST_DIR}/artifact_directories
+
+pushd geode
+find . -name "*-progress*txt" >> ${directories_file}
+echo "Collecting the following artifacts..."
+cat ${directories_file}
+echo ""
+mkdir -p ${BUILD_ARTIFACTS_DIR}/progress
+tar cf - -T ${directories_file} | (cd ${BUILD_ARTIFACTS_DIR}/progress; tar xpf -)
+popd
+
+pushd ${BUILD_ARTIFACTS_DIR}
+tar zcf ${DEST_DIR}/${BUILD_ARTIFACTS_FILENAME} .
+popd
+gsutil -q cp ${DEST_DIR}/${BUILD_ARTIFACTS_FILENAME} gs://${BUILD_ARTIFACTS_DESTINATION}
+printf "\033[92mBuild artifacts from this job are available at:\033[0m\n"
+printf "\n"
+printf "\033[92mhttp://${BUILD_ARTIFACTS_DESTINATION}\033[0m\n"
+
+if [ ${GRADLE_EXIT_STATUS} -eq 0 ]; then
+    sendSuccessfulJobEmail
+else
+    sendFailureJobEmail
+fi
+
+exit ${GRADLE_EXIT_STATUS}

-- 
To stop receiving notification emails like this one, please contact
['"commits@geode.apache.org" <commits@geode.apache.org>'].

Mime
View raw message