From commits-return-60704-archive-asf-public=cust-asf.ponee.io@beam.apache.org Mon Mar 12 03:57:04 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 3371C18060F for ; Mon, 12 Mar 2018 03:57:04 +0100 (CET) Received: (qmail 9274 invoked by uid 500); 12 Mar 2018 02:57:03 -0000 Mailing-List: contact commits-help@beam.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@beam.apache.org Delivered-To: mailing list commits@beam.apache.org Received: (qmail 9265 invoked by uid 99); 12 Mar 2018 02:57:02 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 12 Mar 2018 02:57:02 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 46D7880865; Mon, 12 Mar 2018 02:57:02 +0000 (UTC) Date: Mon, 12 Mar 2018 02:57:01 +0000 To: "commits@beam.apache.org" Subject: [beam] branch master updated: [BEAM-3217] Jenkins job for HadoopInputFormatIOIT (#4758) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <152082342163.21484.1606147063793515972@gitbox.apache.org> From: chamikara@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: beam X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 8739d179738cc6f13bbc46450fea5260d84480d5 X-Git-Newrev: af28351e896b71579bac3f530544fd5c9a7e9a44 X-Git-Rev: af28351e896b71579bac3f530544fd5c9a7e9a44 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. chamikara pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new af28351 [BEAM-3217] Jenkins job for HadoopInputFormatIOIT (#4758) af28351 is described below commit af28351e896b71579bac3f530544fd5c9a7e9a44 Author: Łukasz Gajowy AuthorDate: Mon Mar 12 03:56:57 2018 +0100 [BEAM-3217] Jenkins job for HadoopInputFormatIOIT (#4758) --- .test-infra/jenkins/common_job_properties.groovy | 48 +++++++++++++++++++++ ...job_beam_PerformanceTests_FileBasedIO_IT.groovy | 12 ++---- ...beam_PerformanceTests_HadoopInputFormat.groovy} | 49 ++++++++-------------- .../jenkins/job_beam_PerformanceTests_JDBC.groovy | 33 +++++---------- .../postgres/postgres-service-for-local-dev.yml | 27 ++++++++++++ sdks/java/io/hadoop-input-format/pom.xml | 2 +- sdks/java/io/jdbc/pom.xml | 2 +- 7 files changed, 108 insertions(+), 65 deletions(-) diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index c370dc2..118fe50 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -262,6 +262,31 @@ class common_job_properties { return mapToArgString(joinedArgs) } + static def setupKubernetes(def context, def namespace, def kubeconfigLocation) { + context.steps { + shell('gcloud container clusters get-credentials io-datastores --zone=us-central1-a --verbosity=debug') + shell("cp /home/jenkins/.kube/config ${kubeconfigLocation}") + + shell("kubectl --kubeconfig=${kubeconfigLocation} create namespace ${namespace}") + shell("kubectl --kubeconfig=${kubeconfigLocation} config set-context \$(kubectl config current-context) --namespace=${namespace}") + } + } + + static def cleanupKubernetes(def context, def namespace, def kubeconfigLocation) { + context.steps { + shell("kubectl --kubeconfig=${kubeconfigLocation} delete namespace ${namespace}") + shell("rm ${kubeconfigLocation}") + } + } + + static String getKubernetesNamespace(def testName) { + return "${testName}-${new Date().getTime()}" + } + + static String getKubeconfigLocationForNamespace(def namespace) { + return '"$WORKSPACE/' + "config-${namespace}" + '"' + } + // Adds the standard performance test job steps. static def buildPerformanceTest(def context, def argMap) { def pkbArgs = genPerformanceArgs(argMap) @@ -388,4 +413,27 @@ class common_job_properties { } } } + + /** + * Transforms pipeline options to a string of format like below: + * ["--pipelineOption=123", "--pipelineOption2=abc", ...] + * + * @param pipelineOptions A map of pipeline options. + */ + static String joinPipelineOptions(Map pipelineOptions) { + List pipelineArgList = [] + pipelineOptions.each({ + key, value -> pipelineArgList.add("\"--$key=$value\"") + }) + return "[" + pipelineArgList.join(',') + "]" + } + + + /** + * Returns absolute path to beam project's files. + * @param path A relative path to project resource. + */ + static String makePathAbsolute(String path) { + return '"$WORKSPACE/' + path + '"' + } } diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy index 117ff40..b5558c0 100644 --- a/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy +++ b/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy @@ -108,21 +108,15 @@ private void create_filebasedio_performance_test_job(testConfiguration) { 'commits@beam.apache.org', false) - def pipelineArgs = [ + def pipelineOptions = [ project : 'apache-beam-testing', tempRoot : 'gs://temp-storage-for-perf-tests', filenamePrefix : "gs://temp-storage-for-perf-tests/${testConfiguration.jobName}/\${BUILD_ID}/", ] if (testConfiguration.containsKey('extraPipelineArgs')) { - pipelineArgs << testConfiguration.extraPipelineArgs + pipelineOptions << testConfiguration.extraPipelineArgs } - def pipelineArgList = [] - pipelineArgs.each({ - key, value -> pipelineArgList.add("\"--$key=$value\"") - }) - def pipelineArgsJoined = "[" + pipelineArgList.join(',') + "]" - def argMap = [ benchmarks : 'beam_integration_benchmark', beam_it_timeout : '1200', @@ -131,7 +125,7 @@ private void create_filebasedio_performance_test_job(testConfiguration) { beam_sdk : 'java', beam_it_module : 'sdks/java/io/file-based-io-tests', beam_it_class : testConfiguration.itClass, - beam_it_options : pipelineArgsJoined, + beam_it_options : common_job_properties.joinPipelineOptions(pipelineOptions), beam_extra_mvn_properties: '["filesystem=gcs"]', bigquery_table : testConfiguration.bqTable, ] diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_HadoopInputFormat.groovy similarity index 54% copy from .test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy copy to .test-infra/jenkins/job_beam_PerformanceTests_HadoopInputFormat.groovy index 1e5131f..093390d 100644 --- a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy +++ b/.test-infra/jenkins/job_beam_PerformanceTests_HadoopInputFormat.groovy @@ -18,8 +18,7 @@ import common_job_properties -// This job runs the Beam performance tests on PerfKit Benchmarker. -job('beam_PerformanceTests_JDBC') { +job('beam_PerformanceTests_HadoopInputFormat') { // Set default Beam job properties. common_job_properties.setTopLevelMainJobProperties(delegate) @@ -34,48 +33,36 @@ job('beam_PerformanceTests_JDBC') { common_job_properties.enablePhraseTriggeringFromPullRequest( delegate, - 'Java JdbcIO Performance Test', - 'Run Java JdbcIO Performance Test') + 'Java HadoopInputFormatIO Performance Test', + 'Run Java HadoopInputFormatIO Performance Test') - def pipelineArgs = [ + def pipelineOptions = [ tempRoot : 'gs://temp-storage-for-perf-tests', project : 'apache-beam-testing', postgresPort : '5432', - numberOfRecords: '5000000' + numberOfRecords: '600000' ] + String namespace = common_job_properties.getKubernetesNamespace('hadoopinputformatioit') + String kubeconfig = common_job_properties.getKubeconfigLocationForNamespace(namespace) + def testArgs = [ - kubeconfig : '"$HOME/.kube/config"', - beam_it_timeout : '1800', + kubeconfig : kubeconfig, + beam_it_timeout : '1200', benchmarks : 'beam_integration_benchmark', beam_it_profile : 'io-it', beam_prebuilt : 'true', beam_sdk : 'java', - beam_it_module : 'sdks/java/io/jdbc', - beam_it_class : 'org.apache.beam.sdk.io.jdbc.JdbcIOIT', - beam_it_options : joinPipelineOptions(pipelineArgs), - beam_kubernetes_scripts : makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres.yml') - + ',' + makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'), - beam_options_config_file: makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'), - bigquery_table : 'beam_performance.jdbcioit_pkb_results' + beam_it_module : 'sdks/java/io/hadoop-input-format', + beam_it_class : 'org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIOIT', + beam_it_options : common_job_properties.joinPipelineOptions(pipelineOptions), + beam_kubernetes_scripts : common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'), + beam_options_config_file: common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'), + bigquery_table : 'beam_performance.hadoopinputformatioit_pkb_results' ] - steps { - // create .kube/config file for perfkit (if not exists) - shell('gcloud container clusters get-credentials io-datastores --zone=us-central1-a --verbosity=debug') - } - + common_job_properties.setupKubernetes(delegate, namespace, kubeconfig) common_job_properties.buildPerformanceTest(delegate, testArgs) + common_job_properties.cleanupKubernetes(delegate, namespace, kubeconfig) } -static String joinPipelineOptions(Map pipelineArgs) { - List pipelineArgList = [] - pipelineArgs.each({ - key, value -> pipelineArgList.add("\"--$key=$value\"") - }) - return "[" + pipelineArgList.join(',') + "]" -} - -static String makePathAbsolute(String path) { - return '"$WORKSPACE/' + path + '"' -} \ No newline at end of file diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy index 1e5131f..401a910 100644 --- a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy +++ b/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy @@ -18,7 +18,6 @@ import common_job_properties -// This job runs the Beam performance tests on PerfKit Benchmarker. job('beam_PerformanceTests_JDBC') { // Set default Beam job properties. common_job_properties.setTopLevelMainJobProperties(delegate) @@ -37,15 +36,18 @@ job('beam_PerformanceTests_JDBC') { 'Java JdbcIO Performance Test', 'Run Java JdbcIO Performance Test') - def pipelineArgs = [ + def pipelineOptions = [ tempRoot : 'gs://temp-storage-for-perf-tests', project : 'apache-beam-testing', postgresPort : '5432', numberOfRecords: '5000000' ] + String namespace = common_job_properties.getKubernetesNamespace('jdbcioit') + String kubeconfig = common_job_properties.getKubeconfigLocationForNamespace(namespace) + def testArgs = [ - kubeconfig : '"$HOME/.kube/config"', + kubeconfig : kubeconfig, beam_it_timeout : '1800', benchmarks : 'beam_integration_benchmark', beam_it_profile : 'io-it', @@ -53,29 +55,14 @@ job('beam_PerformanceTests_JDBC') { beam_sdk : 'java', beam_it_module : 'sdks/java/io/jdbc', beam_it_class : 'org.apache.beam.sdk.io.jdbc.JdbcIOIT', - beam_it_options : joinPipelineOptions(pipelineArgs), - beam_kubernetes_scripts : makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres.yml') - + ',' + makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'), - beam_options_config_file: makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'), + beam_it_options : common_job_properties.joinPipelineOptions(pipelineOptions), + beam_kubernetes_scripts : common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'), + beam_options_config_file: common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'), bigquery_table : 'beam_performance.jdbcioit_pkb_results' ] - steps { - // create .kube/config file for perfkit (if not exists) - shell('gcloud container clusters get-credentials io-datastores --zone=us-central1-a --verbosity=debug') - } - + common_job_properties.setupKubernetes(delegate, namespace, kubeconfig) common_job_properties.buildPerformanceTest(delegate, testArgs) + common_job_properties.cleanupKubernetes(delegate, namespace, kubeconfig) } -static String joinPipelineOptions(Map pipelineArgs) { - List pipelineArgList = [] - pipelineArgs.each({ - key, value -> pipelineArgList.add("\"--$key=$value\"") - }) - return "[" + pipelineArgList.join(',') + "]" -} - -static String makePathAbsolute(String path) { - return '"$WORKSPACE/' + path + '"' -} \ No newline at end of file diff --git a/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml b/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml index 5d2c664..7ba106a 100644 --- a/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml +++ b/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml @@ -26,3 +26,30 @@ spec: selector: name: postgres type: LoadBalancer + +--- + +apiVersion: v1 +kind: ReplicationController +metadata: + name: postgres +spec: + replicas: 1 + selector: + name: postgres + template: + metadata: + name: postgres + labels: + name: postgres + spec: + containers: + - name: postgres + image: postgres + env: + - name: POSTGRES_PASSWORD + value: uuinkks + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - containerPort: 5432 diff --git a/sdks/java/io/hadoop-input-format/pom.xml b/sdks/java/io/hadoop-input-format/pom.xml index e8c3565..1907deb 100644 --- a/sdks/java/io/hadoop-input-format/pom.xml +++ b/sdks/java/io/hadoop-input-format/pom.xml @@ -253,7 +253,7 @@ -beam_it_module=sdks/java/io/hadoop-input-format -beam_it_class=org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIOIT -beam_options_config_file=${beamRootProjectDir}/.test-infra/kubernetes/postgres/pkb-config-local.yml - -beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres.yml,${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml + -beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml -beam_it_options=${integrationTestPipelineOptions} diff --git a/sdks/java/io/jdbc/pom.xml b/sdks/java/io/jdbc/pom.xml index 4d52746..9151cfb 100644 --- a/sdks/java/io/jdbc/pom.xml +++ b/sdks/java/io/jdbc/pom.xml @@ -220,7 +220,7 @@ ${pkbBeamRunnerOption} -beam_options_config_file=${beamRootProjectDir}/.test-infra/kubernetes/postgres/pkb-config-local.yml - -beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres.yml,${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml + -beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml -beam_it_module=sdks/java/io/jdbc -beam_it_class=org.apache.beam.sdk.io.jdbc.JdbcIOIT -- To stop receiving notification emails like this one, please contact chamikara@apache.org.