geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jensde...@apache.org
Subject [geode] 01/06: initial commit for windows heavy lifting
Date Fri, 24 Aug 2018 15:09:50 GMT
This is an automated email from the ASF dual-hosted git repository.

jensdeppe pushed a commit to branch windows-heavy-lifter
in repository https://gitbox.apache.org/repos/asf/geode.git

commit 97a7795da3edf4cfb2460582b63c85332fb0a929
Author: Jens Deppe <jdeppe@pivotal.io>
AuthorDate: Wed Aug 22 11:58:01 2018 -0700

    initial commit for windows heavy lifting
---
 ci/pipelines/geode-build/jinja.template.yml | 71 +++--------------------------
 ci/pipelines/shared/jinja.variables.yml     | 53 ++++++++++-----------
 ci/scripts/execute_tests.sh                 |  2 +-
 ci/scripts/start_instance.sh                | 29 ++++++++++--
 4 files changed, 56 insertions(+), 99 deletions(-)

diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
index 39acf0a..7762377 100644
--- a/ci/pipelines/geode-build/jinja.template.yml
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -31,7 +31,12 @@
 {% endmacro %}
 
 {% macro common_test_params(test) %}
+  {%- if test.CALL_STACK_TIMEOUT -%}
 CALL_STACK_TIMEOUT: {{test.CALL_STACK_TIMEOUT}}
+  {%- endif -%}
+  {%- if test.GRADLE_TASK_OPTIONS -%}
+GRADLE_TASK_OPTIONS: {{test.GRADLE_TASK_OPTIONS}}
+  {%- endif -%}
 DUNIT_PARALLEL_FORKS: {{test.DUNIT_PARALLEL_FORKS}}
 MAINTENANCE_VERSION: {{repository.branch}}
 PARALLEL_DUNIT: {{test.PARALLEL_DUNIT}}
@@ -43,7 +48,7 @@ groups:
 - name: main
   jobs:
   - Build
-  {% for test in (tests + windowstests) if not test.name=="StressNew" -%}
+  {% for test in tests if not test.name=="StressNew" -%}
   - {{test.name}}Test
   {% endfor -%}
   - UpdatePassingRef
@@ -283,67 +288,3 @@ jobs:
               - name: instance-data
             timeout: 1h
 {% endfor -%}
-{% for test in windowstests %}
-- name: {{test.name}}Test
-  public: true
-  serial: true
-  plan:
-  {{ plan_resource_gets() |indent(2) }}
-  - task: cleanup-java-processes
-    config:
-      platform: windows
-      run:
-        path: powershell
-        args:
-        - -command
-        - |
-          gwmi win32_process -filter 'name = "java.exe"' | select commandline | format-list
-          kill -name java -force
-          exit 0
-    tags:
-    - windows-{{test.tags}}-tests
-  - task: execute_tests
-    config:
-      platform: windows
-      params:
-        JAVA_HOME: C:\progra~1\java\jdk1.8.0_181
-        SOURCE_REPOSITORY: {{repository.branch}}
-        {{ common_test_params(test) | indent(8) }}
-      run:
-        path: bash
-        args:
-        - geode-ci/ci/scripts/windows/test-run.sh
-        - {{test.target_arg}}
-        - {{test.dir_arg}}
-        - {{test.package_arg}}
-      inputs:
-      - name: geode-ci
-      - name: geode
-      outputs:
-      - name: built-geode
-    tags:
-    - windows-{{test.tags}}-tests
-    ensure:
-      aggregate:
-      - task: archive_results
-        config:
-          platform: windows
-          params:
-            MAINTENANCE_VERSION: {{repository.branch}}
-            PUBLIC_BUCKET: ((!public-bucket))
-            SERVICE_ACCOUNT: ((!concourse-gcp-account))
-          run:
-            path: bash
-            args:
-            - geode-ci/ci/scripts/windows/test-archive.sh
-            - {{test.target_arg}}
-            - {{test.dir_arg}}
-          inputs:
-          - name: geode-ci
-          - name: geode-build-version
-          - name: built-geode
-        tags:
-        - windows-{{test.tags}}-tests
-        timeout: 1h
-    timeout: 6h
-{% endfor %}
diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml
index fd4789e..d0a28f4 100644
--- a/ci/pipelines/shared/jinja.variables.yml
+++ b/ci/pipelines/shared/jinja.variables.yml
@@ -18,12 +18,6 @@
 repository:
   project: geode
 
-platforms:
-- name: "Linux"
-  prefix: ""
-- name: "Windows"
-  prefix: "Windows"
-
 tests:
 - name: "Acceptance"
   CPUS: "8"
@@ -70,37 +64,40 @@ tests:
   GRADLE_TASK: repeatTest
   execute_test_timeout: 2h15m
   PARALLEL_DUNIT: "true"
-
-windowstests:
 - name: "WindowsAcceptance"
-  tags: "acceptance"
-  CALL_STACK_TIMEOUT: "1800"
+  tags: "windows"
+  CPUS: "16"
+  RAM: "32"
+  ARTIFACT_SLUG: windows-acceptancetestfiles
   DUNIT_PARALLEL_FORKS: "0"
+  GRADLE_TASK: :geode-assembly:acceptanceTest
+  execute_test_timeout: 6h
   PARALLEL_DUNIT: "false"
-  target_arg: :geode-assembly:acceptanceTest
-  dir_arg: windows-acceptancetestfiles
-  package_arg: ""
 - name: "WindowsGfshDistributed"
-  tags: "distributed"
-  CALL_STACK_TIMEOUT: "1800"
+  tags: "windows"
+  CPUS: "16"
+  RAM: "32"
+  ARTIFACT_SLUG: windows-gfshdistributedtest
   DUNIT_PARALLEL_FORKS: "0"
+  GRADLE_TASK: distributedTest
+  GRADLE_TASK_OPTIONS: "-PtestCategory=org.apache.geode.test.junit.categories.GfshTest"
+  execute_test_timeout: 6h
   PARALLEL_DUNIT: "false"
-  target_arg: distributedTest
-  dir_arg: windows-gfshdistributedtest
-  package_arg: org.apache.geode.test.junit.categories.GfshTest
 - name: "WindowsIntegration"
-  tags: "integration"
-  CALL_STACK_TIMEOUT: "1800"
+  tags: "windows"
+  CPUS: "16"
+  RAM: "32"
+  ARTIFACT_SLUG: windows-integrationtestfiles
   DUNIT_PARALLEL_FORKS: "0"
+  GRADLE_TASK: integrationTest
+  execute_test_timeout: 6h
   PARALLEL_DUNIT: "false"
-  target_arg: integrationTest
-  dir_arg: windows-integrationtestfiles
-  package_arg: ""
 - name: "WindowsUnit"
-  tags: "unit"
-  CALL_STACK_TIMEOUT: "1800"
+  tags: "windows"
+  CPUS: "16"
+  RAM: "32"
+  ARTIFACT_SLUG: windows-unittestfiles
   DUNIT_PARALLEL_FORKS: "0"
+  GRADLE_TASK: test
+  execute_test_timeout: 6h
   PARALLEL_DUNIT: "false"
-  target_arg: test
-  dir_arg: windows-unittestfiles
-  package_arg: ""
diff --git a/ci/scripts/execute_tests.sh b/ci/scripts/execute_tests.sh
index e270994..00db410 100755
--- a/ci/scripts/execute_tests.sh
+++ b/ci/scripts/execute_tests.sh
@@ -79,4 +79,4 @@ GRADLE_COMMAND="./gradlew \
     ${SCM_PROPS}"
 
 echo "${GRADLE_COMMAND}"
-ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "mkdir -p tmp && cd geode &&
${GRADLE_COMMAND}"
+ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "bash -c 'mkdir -p tmp; cd geode; ${GRADLE_COMMAND}'"
diff --git a/ci/scripts/start_instance.sh b/ci/scripts/start_instance.sh
index c9a54f2..aa85924 100755
--- a/ci/scripts/start_instance.sh
+++ b/ci/scripts/start_instance.sh
@@ -50,11 +50,16 @@ SANITIZED_BUILD_PIPELINE_NAME=$(echo ${BUILD_PIPELINE_NAME} | tr "/" "-"
| tr '[
 SANITIZED_BUILD_JOB_NAME=$(echo ${BUILD_JOB_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
 SANITIZED_BUILD_NAME=$(echo ${BUILD_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
 IMAGE_FAMILY_PREFIX=""
+WINDOWS_PREFIX=""
 
 if [[ "${GEODE_FORK}" != "apache" ]]; then
   IMAGE_FAMILY_PREFIX="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
 fi
 
+if [[ "${SANITIZED_BUILD_JOB_NAME}" =~ Windows ]]; then
+  WINDOWS_PREFIX="windows-"
+fi
+
 INSTANCE_NAME="$(echo "build-${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" | tr
'[:upper:]' '[:lower:]')"
 PROJECT=apachegeode-ci
 ZONE=us-central1-f
@@ -76,7 +81,7 @@ while true; do
       --min-cpu-platform=Intel\ Skylake \
       --network="heavy-lifters" \
       --subnet="heavy-lifters" \
-      --image-family="${IMAGE_FAMILY_PREFIX}geode-builder" \
+      --image-family="${IMAGE_FAMILY_PREFIX}${WINDOWS_PREFIX}geode-builder" \
       --image-project=${PROJECT} \
       --boot-disk-size=100GB \
       --boot-disk-type=pd-ssd \
@@ -96,9 +101,23 @@ done
 
 echo "${INSTANCE_INFORMATION}" > instance-data/instance-information
 
-INSTANCE_IP_ADDRESS=$(echo ${INSTANCE_INFORMATION} | jq -r '.[].networkInterfaces[0].accessConfigs[0].natIP')
+# This extracts the internal IP address for subsequent use
+INSTANCE_IP_ADDRESS=$(echo ${INSTANCE_INFORMATION} | jq -r '.[].networkInterfaces[0].networkIP')
 echo "${INSTANCE_IP_ADDRESS}" > "instance-data/instance-ip-address"
 
-while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE}
--quiet -- true; do
-  echo -n .
-done
+if [[ -z "${WINDOWS_PREFIX}" ]]; then
+  while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE}
--quiet -- true; do
+    echo -n .
+  done
+else
+  # Set up ssh access for Windows systems
+  while [[ -z "${PASSWORD}" ]]; do
+    PASSWORD=$( yes | gcloud beta compute reset-windows-password ${INSTANCE_NAME} --user=geode
--zone=${ZONE} --format json | jq -r .password )
+    sleep 5
+  done
+
+  KEY=$( cat ${SSHKEY_FILE} )
+
+  winrm -hostname ${INSTANCE_IP_ADDRESS} -username geode -password "${PASSWORD}" \
+    "powershell -command \"&{ mkdir c:\users\geode\.ssh -force; set-content -path c:\users\geode\.ssh\authorized_keys
-encoding utf8 -value '${KEY}' }\""
+fi


Mime
View raw message