geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jensde...@apache.org
Subject [geode] 01/01: initial commit for windows heavy lifting
Date Wed, 29 Aug 2018 19:00:39 GMT
This is an automated email from the ASF dual-hosted git repository.

jensdeppe pushed a commit to branch windows-heavy-lifter
in repository https://gitbox.apache.org/repos/asf/geode.git

commit 2bca4fc2b0e08be24f3f621361f8f008ae1d38e2
Author: Jens Deppe <jdeppe@pivotal.io>
AuthorDate: Wed Aug 22 11:58:01 2018 -0700

    initial commit for windows heavy lifting
---
 ci/images/alpine-tools/Dockerfile           |  5 +-
 ci/pipelines/geode-build/jinja.template.yml | 71 +++--------------------------
 ci/pipelines/shared/jinja.variables.yml     | 53 ++++++++++-----------
 ci/scripts/execute_tests.sh                 |  2 +-
 ci/scripts/rsync_code_down.sh               |  4 +-
 ci/scripts/start_instance.sh                | 30 ++++++++++--
 gradle.properties                           |  2 +-
 7 files changed, 65 insertions(+), 102 deletions(-)

diff --git a/ci/images/alpine-tools/Dockerfile b/ci/images/alpine-tools/Dockerfile
index 1337fd9..a5bf091 100644
--- a/ci/images/alpine-tools/Dockerfile
+++ b/ci/images/alpine-tools/Dockerfile
@@ -38,4 +38,7 @@ RUN apk --no-cache add \
   && (cd winrm-cli; GOPATH=$PWD PATH=$PATH:$PWD/bin make) \
   && cp winrm-cli/bin/winrm /usr/local/bin/ \
   && rm -rf winrm-cli \
-  && gcloud components install -q beta
+  && gcloud components install -q beta \
+  && echo "Host *" >> /etc/ssh/ssh_config \
+  && echo "    ServerAliveInterval 60" >> /etc/ssh/ssh_config \
+  && echo "    ServerAliveCountMax 2" >> /etc/ssh/ssh_config
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
index e94c99d..e72b2ca 100644
--- a/ci/pipelines/geode-build/jinja.template.yml
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -31,7 +31,12 @@
 {% endmacro %}
 
 {% macro common_test_params(test) %}
+  {% if test.CALL_STACK_TIMEOUT %}
 CALL_STACK_TIMEOUT: {{test.CALL_STACK_TIMEOUT}}
+  {% endif %}
+  {% if test.GRADLE_TASK_OPTIONS %}
+GRADLE_TASK_OPTIONS: {{test.GRADLE_TASK_OPTIONS}}
+  {% endif %}
 DUNIT_PARALLEL_FORKS: {{test.DUNIT_PARALLEL_FORKS}}
 MAINTENANCE_VERSION: {{repository.branch}}
 PARALLEL_DUNIT: {{test.PARALLEL_DUNIT}}
@@ -44,7 +49,7 @@ groups:
 - name: main
   jobs:
   - Build
-  {% for test in (tests + windowstests) if not test.name=="StressNew" -%}
+  {% for test in tests if not test.name=="StressNew" -%}
   - {{test.name}}Test
   {% endfor -%}
   - UpdatePassingRef
@@ -321,67 +326,3 @@ jobs:
               - name: instance-data
             timeout: 1h
 {% endfor -%}
-{% for test in windowstests %}
-- name: {{test.name}}Test
-  public: true
-  serial: true
-  plan:
-  {{ plan_resource_gets() |indent(2) }}
-  - task: cleanup-java-processes
-    config:
-      platform: windows
-      run:
-        path: powershell
-        args:
-        - -command
-        - |
-          gwmi win32_process -filter 'name = "java.exe"' | select commandline | format-list
-          kill -name java -force
-          exit 0
-    tags:
-    - windows-{{test.tags}}-tests
-  - task: execute_tests
-    config:
-      platform: windows
-      params:
-        JAVA_HOME: C:\progra~1\java\jdk1.8.0_181
-        SOURCE_REPOSITORY: {{repository.branch}}
-        {{ common_test_params(test) | indent(8) }}
-      run:
-        path: bash
-        args:
-        - geode-ci/ci/scripts/windows/test-run.sh
-        - {{test.target_arg}}
-        - {{test.dir_arg}}
-        - {{test.package_arg}}
-      inputs:
-      - name: geode-ci
-      - name: geode
-      outputs:
-      - name: built-geode
-    tags:
-    - windows-{{test.tags}}-tests
-    ensure:
-      aggregate:
-      - task: archive_results
-        config:
-          platform: windows
-          params:
-            MAINTENANCE_VERSION: {{repository.branch}}
-            PUBLIC_BUCKET: ((!public-bucket))
-            SERVICE_ACCOUNT: ((!concourse-gcp-account))
-          run:
-            path: bash
-            args:
-            - geode-ci/ci/scripts/windows/test-archive.sh
-            - {{test.target_arg}}
-            - {{test.dir_arg}}
-          inputs:
-          - name: geode-ci
-          - name: geode-build-version
-          - name: built-geode
-        tags:
-        - windows-{{test.tags}}-tests
-        timeout: 1h
-    timeout: 6h
-{% endfor %}
diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml
index 85d495a..41ba532 100644
--- a/ci/pipelines/shared/jinja.variables.yml
+++ b/ci/pipelines/shared/jinja.variables.yml
@@ -18,12 +18,6 @@
 repository:
   project: geode
 
-platforms:
-- name: "Linux"
-  prefix: ""
-- name: "Windows"
-  prefix: "Windows"
-
 tests:
 - name: "Acceptance"
   CPUS: "8"
@@ -71,37 +65,40 @@ tests:
   GRADLE_TASK: repeatTest
   execute_test_timeout: 2h15m
   PARALLEL_DUNIT: "true"
-
-windowstests:
 - name: "WindowsAcceptance"
-  tags: "acceptance"
-  CALL_STACK_TIMEOUT: "1800"
+  tags: "windows"
+  CPUS: "16"
+  RAM: "32"
+  ARTIFACT_SLUG: windows-acceptancetestfiles
   DUNIT_PARALLEL_FORKS: "0"
+  GRADLE_TASK: :geode-assembly:acceptanceTest
+  execute_test_timeout: 6h
   PARALLEL_DUNIT: "false"
-  target_arg: :geode-assembly:acceptanceTest
-  dir_arg: windows-acceptancetestfiles
-  package_arg: ""
 - name: "WindowsGfshDistributed"
-  tags: "distributed"
-  CALL_STACK_TIMEOUT: "1800"
+  tags: "windows"
+  CPUS: "16"
+  RAM: "32"
+  ARTIFACT_SLUG: windows-gfshdistributedtest
   DUNIT_PARALLEL_FORKS: "0"
+  GRADLE_TASK: distributedTest
+  GRADLE_TASK_OPTIONS: "-PtestCategory=org.apache.geode.test.junit.categories.GfshTest"
+  execute_test_timeout: 6h
   PARALLEL_DUNIT: "false"
-  target_arg: distributedTest
-  dir_arg: windows-gfshdistributedtest
-  package_arg: org.apache.geode.test.junit.categories.GfshTest
 - name: "WindowsIntegration"
-  tags: "integration"
-  CALL_STACK_TIMEOUT: "1800"
+  tags: "windows"
+  CPUS: "16"
+  RAM: "32"
+  ARTIFACT_SLUG: windows-integrationtestfiles
   DUNIT_PARALLEL_FORKS: "0"
+  GRADLE_TASK: integrationTest
+  execute_test_timeout: 6h
   PARALLEL_DUNIT: "false"
-  target_arg: integrationTest
-  dir_arg: windows-integrationtestfiles
-  package_arg: ""
 - name: "WindowsUnit"
-  tags: "unit"
-  CALL_STACK_TIMEOUT: "1800"
+  tags: "windows"
+  CPUS: "16"
+  RAM: "32"
+  ARTIFACT_SLUG: windows-unittestfiles
   DUNIT_PARALLEL_FORKS: "0"
+  GRADLE_TASK: test
+  execute_test_timeout: 6h
   PARALLEL_DUNIT: "false"
-  target_arg: test
-  dir_arg: windows-unittestfiles
-  package_arg: ""
diff --git a/ci/scripts/execute_tests.sh b/ci/scripts/execute_tests.sh
index d36f85b..c4bd456 100755
--- a/ci/scripts/execute_tests.sh
+++ b/ci/scripts/execute_tests.sh
@@ -84,4 +84,4 @@ GRADLE_COMMAND="./gradlew \
     ${SCM_PROPS}"
 
 echo "${GRADLE_COMMAND}"
-ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "mkdir -p tmp && cd geode &&
${GRADLE_COMMAND}"
+ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "bash -c 'mkdir -p tmp; cd geode; ${GRADLE_COMMAND}'"
diff --git a/ci/scripts/rsync_code_down.sh b/ci/scripts/rsync_code_down.sh
index ce12bcc..a94bb90 100755
--- a/ci/scripts/rsync_code_down.sh
+++ b/ci/scripts/rsync_code_down.sh
@@ -40,7 +40,7 @@ echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
 
 OUTPUT_DIR=${BASE_DIR}/geode-results
 
-ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "cd geode && ./gradlew combineReports"
+ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "bash -c 'cd geode; ./gradlew --no-daemon
combineReports'"
 
-time rsync -e "ssh -i ${SSHKEY_FILE}" -ah geode@${INSTANCE_IP_ADDRESS}:. ${OUTPUT_DIR}/.
+time rsync -e "ssh -i ${SSHKEY_FILE}" -ah geode@${INSTANCE_IP_ADDRESS}:geode ${OUTPUT_DIR}/
 set +x
diff --git a/ci/scripts/start_instance.sh b/ci/scripts/start_instance.sh
index 3bee7f9..88621f8 100755
--- a/ci/scripts/start_instance.sh
+++ b/ci/scripts/start_instance.sh
@@ -18,6 +18,7 @@
 # limitations under the License.
 
 set -e
+set -x
 
 BASE_DIR=$(pwd)
 
@@ -54,11 +55,16 @@ SANITIZED_BUILD_PIPELINE_NAME=$(echo ${BUILD_PIPELINE_NAME} | tr "/" "-"
| tr '[
 SANITIZED_BUILD_JOB_NAME=$(echo ${BUILD_JOB_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
 SANITIZED_BUILD_NAME=$(echo ${BUILD_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
 IMAGE_FAMILY_PREFIX=""
+WINDOWS_PREFIX=""
 
 if [[ "${SANITIZED_GEODE_FORK}" != "apache" ]]; then
   IMAGE_FAMILY_PREFIX="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
 fi
 
+if [[ "${SANITIZED_BUILD_JOB_NAME}" =~ [Ww]indows ]]; then
+  WINDOWS_PREFIX="windows-"
+fi
+
 INSTANCE_NAME="$(echo "${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" | tr '[:upper:]'
'[:lower:]')"
 PROJECT=apachegeode-ci
 ZONE=us-central1-f
@@ -80,7 +86,7 @@ while true; do
       --min-cpu-platform=Intel\ Skylake \
       --network="heavy-lifters" \
       --subnet="heavy-lifters" \
-      --image-family="${IMAGE_FAMILY_PREFIX}geode-builder" \
+      --image-family="${IMAGE_FAMILY_PREFIX}${WINDOWS_PREFIX}geode-builder" \
       --image-project=${PROJECT} \
       --boot-disk-size=100GB \
       --boot-disk-type=pd-ssd \
@@ -103,6 +109,22 @@ echo "${INSTANCE_INFORMATION}" > instance-data/instance-information
 INSTANCE_IP_ADDRESS=$(echo ${INSTANCE_INFORMATION} | jq -r '.[].networkInterfaces[0].accessConfigs[0].natIP')
 echo "${INSTANCE_IP_ADDRESS}" > "instance-data/instance-ip-address"
 
-while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE}
--quiet -- true; do
-  echo -n .
-done
+if [[ -z "${WINDOWS_PREFIX}" ]]; then
+  while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE}
--quiet -- true; do
+    echo -n .
+  done
+else
+  # Set up ssh access for Windows systems
+  while [[ -z "${PASSWORD}" ]]; do
+    PASSWORD=$( yes | gcloud beta compute reset-windows-password ${INSTANCE_NAME} --user=geode
--zone=${ZONE} --format json | jq -r .password )
+    sleep 5
+  done
+
+  ssh-keygen -N "" -f ${SSHKEY_FILE}
+
+  KEY=$( cat ${SSHKEY_FILE}.pub )
+
+  winrm -hostname ${INSTANCE_IP_ADDRESS} -username geode -password "${PASSWORD}" \
+    -https -insecure -port 5986 \
+    "powershell -command \"&{ mkdir c:\users\geode\.ssh -force; set-content -path c:\users\geode\.ssh\authorized_keys
-encoding utf8 -value '${KEY}' }\""
+fi
diff --git a/gradle.properties b/gradle.properties
index 3edf13d..e9c97ec 100755
--- a/gradle.properties
+++ b/gradle.properties
@@ -42,7 +42,7 @@ productName = Apache Geode
 productOrg = Apache Software Foundation (ASF)
 
 org.gradle.daemon = true
-org.gradle.jvmargs = -Xmx3g
+org.gradle.jvmargs = -Xmx4g
 
 minimumGradleVersion = 4.10
 # Set this on the command line with -P or in ~/.gradle/gradle.properties


Mime
View raw message