From commits-return-64844-archive-asf-public=cust-asf.ponee.io@beam.apache.org Thu Apr 5 02:50:07 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 39FEA18064F for ; Thu, 5 Apr 2018 02:50:06 +0200 (CEST) Received: (qmail 46681 invoked by uid 500); 5 Apr 2018 00:50:05 -0000 Mailing-List: contact commits-help@beam.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@beam.apache.org Delivered-To: mailing list commits@beam.apache.org Received: (qmail 46668 invoked by uid 99); 5 Apr 2018 00:50:05 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd2-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 05 Apr 2018 00:50:05 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd2-us-west.apache.org (ASF Mail Server at spamd2-us-west.apache.org) with ESMTP id 9EAD01A09B9 for ; Thu, 5 Apr 2018 00:50:04 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd2-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -101.511 X-Spam-Level: X-Spam-Status: No, score=-101.511 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, RCVD_IN_DNSWL_MED=-2.3, SPF_PASS=-0.001, T_RP_MATCHES_RCVD=-0.01, USER_IN_WHITELIST=-100] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd2-us-west.apache.org [10.40.0.9]) (amavisd-new, port 10024) with ESMTP id QkUb4PzwZSvs for ; Thu, 5 Apr 2018 00:50:02 +0000 (UTC) Received: from mailrelay1-us-west.apache.org (mailrelay1-us-west.apache.org [209.188.14.139]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with ESMTP id 50D8A5F180 for ; Thu, 5 Apr 2018 00:50:01 +0000 (UTC) Received: from jira-lw-us.apache.org (unknown [207.244.88.139]) by mailrelay1-us-west.apache.org (ASF Mail Server at mailrelay1-us-west.apache.org) with ESMTP id 78876E00A6 for ; Thu, 5 Apr 2018 00:50:00 +0000 (UTC) Received: from jira-lw-us.apache.org (localhost [127.0.0.1]) by jira-lw-us.apache.org (ASF Mail Server at jira-lw-us.apache.org) with ESMTP id 21B392561E for ; Thu, 5 Apr 2018 00:50:00 +0000 (UTC) Date: Thu, 5 Apr 2018 00:50:00 +0000 (UTC) From: "ASF GitHub Bot (JIRA)" To: commits@beam.apache.org Message-ID: In-Reply-To: References: Subject: [jira] [Work logged] (BEAM-3250) Migrate ValidatesRunner Jenkins PostCommits to Gradle MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 7bit X-JIRA-FingerPrint: 30527f35849b9dde25b450d4833f0394 [ https://issues.apache.org/jira/browse/BEAM-3250?focusedWorklogId=87839&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-87839 ] ASF GitHub Bot logged work on BEAM-3250: ---------------------------------------- Author: ASF GitHub Bot Created on: 05/Apr/18 00:49 Start Date: 05/Apr/18 00:49 Worklog Time Spent: 10m Work Description: aaltay closed pull request #5025: [BEAM-3250] Migrate Apex and Gearpump ValidatesRunner tests to Gradle URL: https://github.com/apache/beam/pull/5025 This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy index c16a1e2f9d0..512cfa976a1 100644 --- a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy +++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy @@ -19,15 +19,22 @@ import common_job_properties // This job runs the suite of ValidatesRunner tests against the Apex runner. -mavenJob('beam_PostCommit_Java_ValidatesRunner_Apex') { +job('beam_PostCommit_Java_ValidatesRunner_Apex_Gradle') { description('Runs the ValidatesRunner suite on the Apex runner.') + previousNames('beam_PostCommit_Java_ValidatesRunner_Apex') previousNames('beam_PostCommit_Java_RunnableOnService_Apex') // Set common parameters. common_job_properties.setTopLevelMainJobProperties(delegate) - // Set maven parameters. - common_job_properties.setMavenConfig(delegate) + def gradle_switches = [ + // Gradle log verbosity enough to diagnose basic build issues + "--info", + // Continue the build even if there is a failure to show as many potential failures as possible. + '--continue', + // Until we verify the build cache is working appropriately, force rerunning all tasks + '--rerun-tasks', + ] // Sets that this is a PostCommit job. common_job_properties.setPostCommit(delegate) @@ -38,11 +45,14 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Apex') { 'Apache Apex Runner ValidatesRunner Tests', 'Run Apex ValidatesRunner') - // Maven goals for this job. - goals('''clean verify --projects runners/apex \ - --also-make \ - --batch-mode \ - --errors \ - --activate-profiles validates-runner-tests \ - --activate-profiles local-validates-runner-tests''') + // Gradle goals for this job. + steps { + gradle { + rootBuildScriptDir(common_job_properties.checkoutDir) + tasks(':runners:apex:validatesRunner') + for (String gradle_switch : gradle_switches) { + switches(gradle_switch) + } + } + } } diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy index e1cbafe6e4b..8ba0a71dc59 100644 --- a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy +++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy @@ -20,9 +20,9 @@ import common_job_properties // This job runs the suite of ValidatesRunner tests against the Gearpump // runner. -mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') { +job('beam_PostCommit_Java_ValidatesRunner_Gearpump_Gradle') { description('Runs the ValidatesRunner suite on the Gearpump runner.') - + previousNames('beam_PostCommit_Java_ValidatesRunner_Gearpump') previousNames('beam_PostCommit_Java_RunnableOnService_Gearpump') // Set common parameters. @@ -30,8 +30,14 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') { delegate, 'gearpump-runner') - // Set maven parameters. - common_job_properties.setMavenConfig(delegate) + def gradle_switches = [ + // Gradle log verbosity enough to diagnose basic build issues + "--info", + // Continue the build even if there is a failure to show as many potential failures as possible. + '--continue', + // Until we verify the build cache is working appropriately, force rerunning all tasks + '--rerun-tasks', + ] // Sets that this is a PostCommit job. // 0 5 31 2 * will run on Feb 31 (i.e. never) according to job properties. @@ -44,6 +50,14 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') { 'Apache Gearpump Runner ValidatesRunner Tests', 'Run Gearpump ValidatesRunner') - // Maven goals for this job. - goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner"]\'') + // Gradle goals for this job. + steps { + gradle { + rootBuildScriptDir(common_job_properties.checkoutDir) + tasks(':runners:gearpump:validatesRunner') + for (String gradle_switch : gradle_switches) { + switches(gradle_switch) + } + } + } } diff --git a/runners/apex/build.gradle b/runners/apex/build.gradle index 800e5947c5e..1ddb855b585 100644 --- a/runners/apex/build.gradle +++ b/runners/apex/build.gradle @@ -16,6 +16,8 @@ * limitations under the License. */ +import groovy.json.JsonOutput + apply from: project(":").file("build_rules.gradle") applyJavaNature(artifactId: "beam-runners-apex") @@ -30,6 +32,11 @@ description = "Apache Beam :: Runners :: Apex" */ evaluationDependsOn(":model:fn-execution") evaluationDependsOn(":runners:core-java") +evaluationDependsOn(":sdks:java:core") + +configurations { + validatesRunner +} dependencies { shadow project(path: ":model:pipeline", configuration: "shadow") @@ -42,13 +49,15 @@ dependencies { shadow library.java.commons_lang3 shadow library.java.findbugs_jsr305 shadow library.java.apex_engine - testCompile project(path: ":sdks:java:core", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") // ApexStateInternalsTest extends abstract StateInternalsTest - testCompile project(":runners:core-java").sourceSets.test.output - testCompile library.java.hamcrest_core - testCompile library.java.junit - testCompile library.java.mockito_core - testCompile library.java.jackson_dataformat_yaml + shadowTest project(":runners:core-java").sourceSets.test.output + shadowTest library.java.hamcrest_core + shadowTest library.java.junit + shadowTest library.java.mockito_core + shadowTest library.java.jackson_dataformat_yaml + validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest") + validatesRunner project(path: project.path, configuration: "shadow") } // TODO: Update this so that the generated file is added to the explicitly added instead of @@ -73,5 +82,31 @@ task buildDependencyTree(type: DependencyReportTask) { } compileJava.dependsOn buildDependencyTree +task validatesRunnerBatch(type: Test) { + group = "Verification" + systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ + "--runner=TestApexRunner", + ]) + + classpath = configurations.validatesRunner + testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs) + useJUnit { + includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' + excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders' + excludeCategories 'org.apache.beam.sdk.testing.UsesTimersInParDo' + excludeCategories 'org.apache.beam.sdk.testing.UsesSplittableParDo' + excludeCategories 'org.apache.beam.sdk.testing.UsesAttemptedMetrics' + excludeCategories 'org.apache.beam.sdk.testing.UsesCommittedMetrics' + excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream' + excludeCategories 'org.apache.beam.sdk.testing.UsesParDoLifecycle' + } +} + +task validatesRunner { + group = "Verification" + description "Validates Apex runner" + dependsOn validatesRunnerBatch +} + // Generates :runners:apex:runQuickstartJavaApex createJavaQuickstartValidationTask(name: 'Apex') diff --git a/runners/flink/build.gradle b/runners/flink/build.gradle index d6673ae246d..3df88913153 100644 --- a/runners/flink/build.gradle +++ b/runners/flink/build.gradle @@ -124,7 +124,7 @@ createValidatesRunnerTask(name: "validatesRunnerStreaming", streaming: true) task validatesRunner { group = "Verification" - description "Validates batch and streaming runners" + description "Validates Flink runner" dependsOn validatesRunnerBatch dependsOn validatesRunnerStreaming } diff --git a/runners/gearpump/build.gradle b/runners/gearpump/build.gradle index cd96dc4cea0..b9181c0c9a7 100644 --- a/runners/gearpump/build.gradle +++ b/runners/gearpump/build.gradle @@ -16,13 +16,21 @@ * limitations under the License. */ +import groovy.json.JsonOutput + apply from: project(":").file("build_rules.gradle") applyJavaNature(artifactId: "beam-runners-gearpump") description = "Apache Beam :: Runners :: Gearpump" +evaluationDependsOn(":sdks:java:core") + def gearpump_version = "0.8.4" +configurations { + validatesRunner +} + dependencies { compile library.java.guava compileOnly "com.typesafe:config:1.3.0" @@ -35,10 +43,41 @@ dependencies { shadow library.java.joda_time shadow library.java.jackson_annotations shadow library.java.findbugs_jsr305 - testCompile project(path: ":sdks:java:core", configuration: "shadowTest") - testCompile library.java.junit - testCompile library.java.hamcrest_core - testCompile library.java.jackson_databind - testCompile library.java.jackson_dataformat_yaml - testCompile library.java.mockito_core + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") + shadowTest library.java.junit + shadowTest library.java.hamcrest_core + shadowTest library.java.jackson_databind + shadowTest library.java.jackson_dataformat_yaml + shadowTest library.java.mockito_core + validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest") + validatesRunner project(path: project.path, configuration: "shadow") +} + +task validatesRunnerStreaming(type: Test) { + group = "Verification" + systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ + "--runner=TestGearpumpRunner", + "--streaming=true", + ]) + + classpath = configurations.validatesRunner + testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs) + useJUnit { + includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' + excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders' + excludeCategories 'org.apache.beam.sdk.testing.UsesStatefulParDo' + excludeCategories 'org.apache.beam.sdk.testing.UsesTimersInParDo' + excludeCategories 'org.apache.beam.sdk.testing.UsesSplittableParDo' + excludeCategories 'org.apache.beam.sdk.testing.UsesAttemptedMetrics' + excludeCategories 'org.apache.beam.sdk.testing.UsesCommittedMetrics' + excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream' + excludeCategories 'org.apache.beam.sdk.testing.UsesCustomWindowMerging' + excludeCategories 'org.apache.beam.sdk.testing.UsesParDoLifecycle' + } +} + +task validatesRunner { + group = "Verification" + description "Validates Gearpump runner" + dependsOn validatesRunnerStreaming } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesParDoLifecycle.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesParDoLifecycle.java new file mode 100644 index 00000000000..961616565da --- /dev/null +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesParDoLifecycle.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.beam.sdk.testing; + +/** + * Category tag for the ParDoLifecycleTest for exclusion (BEAM-3241). + */ +public interface UsesParDoLifecycle {} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java index 9ecc31d4a22..3ba516e521f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java @@ -31,6 +31,7 @@ import org.apache.beam.sdk.state.StateSpecs; import org.apache.beam.sdk.state.ValueState; import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.UsesParDoLifecycle; import org.apache.beam.sdk.testing.UsesStatefulParDo; import org.apache.beam.sdk.testing.ValidatesRunner; import org.apache.beam.sdk.values.KV; @@ -53,7 +54,7 @@ public final transient TestPipeline p = TestPipeline.create(); @Test - @Category(ValidatesRunner.class) + @Category({ValidatesRunner.class, UsesParDoLifecycle.class}) public void testOldFnCallSequence() { PCollectionList.of(p.apply("Impolite", Create.of(1, 2, 4))) .and(p.apply("Polite", Create.of(3, 5, 6, 7))) @@ -64,7 +65,7 @@ public void testOldFnCallSequence() { } @Test - @Category(ValidatesRunner.class) + @Category({ValidatesRunner.class, UsesParDoLifecycle.class}) public void testOldFnCallSequenceMulti() { PCollectionList.of(p.apply("Impolite", Create.of(1, 2, 4))) .and(p.apply("Polite", Create.of(3, 5, 6, 7))) @@ -133,7 +134,7 @@ public void teardown() { } @Test - @Category(ValidatesRunner.class) + @Category({ValidatesRunner.class, UsesParDoLifecycle.class}) public void testFnCallSequence() { PCollectionList.of(p.apply("Impolite", Create.of(1, 2, 4))) .and(p.apply("Polite", Create.of(3, 5, 6, 7))) @@ -144,7 +145,7 @@ public void testFnCallSequence() { } @Test - @Category(ValidatesRunner.class) + @Category({ValidatesRunner.class, UsesParDoLifecycle.class}) public void testFnCallSequenceMulti() { PCollectionList.of(p.apply("Impolite", Create.of(1, 2, 4))) .and(p.apply("Polite", Create.of(3, 5, 6, 7))) @@ -157,7 +158,7 @@ public void testFnCallSequenceMulti() { } @Test - @Category({ValidatesRunner.class, UsesStatefulParDo.class}) + @Category({ValidatesRunner.class, UsesStatefulParDo.class, UsesParDoLifecycle.class}) public void testFnCallSequenceStateful() { PCollectionList.of(p.apply("Impolite", Create.of(KV.of("a", 1), KV.of("b", 2), KV.of("a", 4)))) .and( @@ -233,7 +234,7 @@ public void after() { } @Test - @Category(ValidatesRunner.class) + @Category({ValidatesRunner.class, UsesParDoLifecycle.class}) public void testTeardownCalledAfterExceptionInStartBundle() { ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.START_BUNDLE); p @@ -251,7 +252,7 @@ public void testTeardownCalledAfterExceptionInStartBundle() { } @Test - @Category(ValidatesRunner.class) + @Category({ValidatesRunner.class, UsesParDoLifecycle.class}) public void testTeardownCalledAfterExceptionInProcessElement() { ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.PROCESS_ELEMENT); p @@ -269,7 +270,7 @@ public void testTeardownCalledAfterExceptionInProcessElement() { } @Test - @Category(ValidatesRunner.class) + @Category({ValidatesRunner.class, UsesParDoLifecycle.class}) public void testTeardownCalledAfterExceptionInFinishBundle() { ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.FINISH_BUNDLE); p @@ -287,7 +288,7 @@ public void testTeardownCalledAfterExceptionInFinishBundle() { } @Test - @Category(ValidatesRunner.class) + @Category({ValidatesRunner.class, UsesParDoLifecycle.class}) public void testWithContextTeardownCalledAfterExceptionInSetup() { ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.SETUP); p.apply(Create.of(1, 2, 3)).apply(ParDo.of(fn)); @@ -302,7 +303,7 @@ public void testWithContextTeardownCalledAfterExceptionInSetup() { } @Test - @Category(ValidatesRunner.class) + @Category({ValidatesRunner.class, UsesParDoLifecycle.class}) public void testWithContextTeardownCalledAfterExceptionInStartBundle() { ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.START_BUNDLE); p.apply(Create.of(1, 2, 3)).apply(ParDo.of(fn)); @@ -317,7 +318,7 @@ public void testWithContextTeardownCalledAfterExceptionInStartBundle() { } @Test - @Category(ValidatesRunner.class) + @Category({ValidatesRunner.class, UsesParDoLifecycle.class}) public void testWithContextTeardownCalledAfterExceptionInProcessElement() { ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.PROCESS_ELEMENT); p.apply(Create.of(1, 2, 3)).apply(ParDo.of(fn)); @@ -332,7 +333,7 @@ public void testWithContextTeardownCalledAfterExceptionInProcessElement() { } @Test - @Category(ValidatesRunner.class) + @Category({ValidatesRunner.class, UsesParDoLifecycle.class}) public void testWithContextTeardownCalledAfterExceptionInFinishBundle() { ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.FINISH_BUNDLE); p.apply(Create.of(1, 2, 3)).apply(ParDo.of(fn)); ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: users@infra.apache.org Issue Time Tracking ------------------- Worklog Id: (was: 87839) Time Spent: 2h (was: 1h 50m) > Migrate ValidatesRunner Jenkins PostCommits to Gradle > ----------------------------------------------------- > > Key: BEAM-3250 > URL: https://issues.apache.org/jira/browse/BEAM-3250 > Project: Beam > Issue Type: Sub-task > Components: build-system, testing > Reporter: Luke Cwik > Assignee: Henning Rohde > Priority: Major > Time Spent: 2h > Remaining Estimate: 0h > > Update these targets to execute ValidatesRunner tests: https://github.com/apache/beam/search?l=Groovy&q=ValidatesRunner&type=&utf8=%E2%9C%93 -- This message was sent by Atlassian JIRA (v7.6.3#76005)