beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Work logged] (BEAM-3250) Migrate ValidatesRunner Jenkins PostCommits to Gradle
Date Thu, 05 Apr 2018 00:50:00 GMT

     [ https://issues.apache.org/jira/browse/BEAM-3250?focusedWorklogId=87839&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-87839
]

ASF GitHub Bot logged work on BEAM-3250:
----------------------------------------

                Author: ASF GitHub Bot
            Created on: 05/Apr/18 00:49
            Start Date: 05/Apr/18 00:49
    Worklog Time Spent: 10m 
      Work Description: aaltay closed pull request #5025: [BEAM-3250] Migrate Apex and Gearpump
ValidatesRunner tests to Gradle
URL: https://github.com/apache/beam/pull/5025
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy
index c16a1e2f9d0..512cfa976a1 100644
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy
@@ -19,15 +19,22 @@
 import common_job_properties
 
 // This job runs the suite of ValidatesRunner tests against the Apex runner.
-mavenJob('beam_PostCommit_Java_ValidatesRunner_Apex') {
+job('beam_PostCommit_Java_ValidatesRunner_Apex_Gradle') {
   description('Runs the ValidatesRunner suite on the Apex runner.')
+  previousNames('beam_PostCommit_Java_ValidatesRunner_Apex')
   previousNames('beam_PostCommit_Java_RunnableOnService_Apex')
 
   // Set common parameters.
   common_job_properties.setTopLevelMainJobProperties(delegate)
 
-  // Set maven parameters.
-  common_job_properties.setMavenConfig(delegate)
+  def gradle_switches = [
+    // Gradle log verbosity enough to diagnose basic build issues
+    "--info",
+    // Continue the build even if there is a failure to show as many potential failures as
possible.
+    '--continue',
+    // Until we verify the build cache is working appropriately, force rerunning all tasks
+    '--rerun-tasks',
+  ]
 
   // Sets that this is a PostCommit job.
   common_job_properties.setPostCommit(delegate)
@@ -38,11 +45,14 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Apex') {
     'Apache Apex Runner ValidatesRunner Tests',
     'Run Apex ValidatesRunner')
 
-  // Maven goals for this job.
-  goals('''clean verify --projects runners/apex \
-      --also-make \
-      --batch-mode \
-      --errors \
-      --activate-profiles validates-runner-tests \
-      --activate-profiles local-validates-runner-tests''')
+  // Gradle goals for this job.
+  steps {
+    gradle {
+      rootBuildScriptDir(common_job_properties.checkoutDir)
+      tasks(':runners:apex:validatesRunner')
+      for (String gradle_switch : gradle_switches) {
+        switches(gradle_switch)
+      }
+    }
+  }
 }
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
index e1cbafe6e4b..8ba0a71dc59 100644
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
@@ -20,9 +20,9 @@ import common_job_properties
 
 // This job runs the suite of ValidatesRunner tests against the Gearpump
 // runner.
-mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') {
+job('beam_PostCommit_Java_ValidatesRunner_Gearpump_Gradle') {
   description('Runs the ValidatesRunner suite on the Gearpump runner.')
-
+  previousNames('beam_PostCommit_Java_ValidatesRunner_Gearpump')
   previousNames('beam_PostCommit_Java_RunnableOnService_Gearpump')
 
   // Set common parameters.
@@ -30,8 +30,14 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') {
       delegate,
       'gearpump-runner')
 
-  // Set maven parameters.
-  common_job_properties.setMavenConfig(delegate)
+  def gradle_switches = [
+    // Gradle log verbosity enough to diagnose basic build issues
+    "--info",
+    // Continue the build even if there is a failure to show as many potential failures as
possible.
+    '--continue',
+    // Until we verify the build cache is working appropriately, force rerunning all tasks
+    '--rerun-tasks',
+  ]
 
   // Sets that this is a PostCommit job.
   // 0 5 31 2 * will run on Feb 31 (i.e. never) according to job properties.
@@ -44,6 +50,14 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') {
     'Apache Gearpump Runner ValidatesRunner Tests',
     'Run Gearpump ValidatesRunner')
 
-  // Maven goals for this job.
-  goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[
"--runner=TestGearpumpRunner"]\'')
+  // Gradle goals for this job.
+  steps {
+    gradle {
+      rootBuildScriptDir(common_job_properties.checkoutDir)
+      tasks(':runners:gearpump:validatesRunner')
+      for (String gradle_switch : gradle_switches) {
+        switches(gradle_switch)
+      }
+    }
+  }
 }
diff --git a/runners/apex/build.gradle b/runners/apex/build.gradle
index 800e5947c5e..1ddb855b585 100644
--- a/runners/apex/build.gradle
+++ b/runners/apex/build.gradle
@@ -16,6 +16,8 @@
  * limitations under the License.
  */
 
+import groovy.json.JsonOutput
+
 apply from: project(":").file("build_rules.gradle")
 applyJavaNature(artifactId: "beam-runners-apex")
 
@@ -30,6 +32,11 @@ description = "Apache Beam :: Runners :: Apex"
  */
 evaluationDependsOn(":model:fn-execution")
 evaluationDependsOn(":runners:core-java")
+evaluationDependsOn(":sdks:java:core")
+
+configurations {
+  validatesRunner
+}
 
 dependencies {
   shadow project(path: ":model:pipeline", configuration: "shadow")
@@ -42,13 +49,15 @@ dependencies {
   shadow library.java.commons_lang3
   shadow library.java.findbugs_jsr305
   shadow library.java.apex_engine
-  testCompile project(path: ":sdks:java:core", configuration: "shadowTest")
+  shadowTest project(path: ":sdks:java:core", configuration: "shadowTest")
   // ApexStateInternalsTest extends abstract StateInternalsTest
-  testCompile project(":runners:core-java").sourceSets.test.output
-  testCompile library.java.hamcrest_core
-  testCompile library.java.junit
-  testCompile library.java.mockito_core
-  testCompile library.java.jackson_dataformat_yaml
+  shadowTest project(":runners:core-java").sourceSets.test.output
+  shadowTest library.java.hamcrest_core
+  shadowTest library.java.junit
+  shadowTest library.java.mockito_core
+  shadowTest library.java.jackson_dataformat_yaml
+  validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+  validatesRunner project(path: project.path, configuration: "shadow")
 }
 
 // TODO: Update this so that the generated file is added to the explicitly added instead
of
@@ -73,5 +82,31 @@ task buildDependencyTree(type: DependencyReportTask) {
 }
 compileJava.dependsOn buildDependencyTree
 
+task validatesRunnerBatch(type: Test) {
+  group = "Verification"
+  systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+          "--runner=TestApexRunner",
+  ])
+
+  classpath = configurations.validatesRunner
+  testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs)
+  useJUnit {
+    includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+    excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesTimersInParDo'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesSplittableParDo'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesAttemptedMetrics'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesCommittedMetrics'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesParDoLifecycle'
+  }
+}
+
+task validatesRunner {
+  group = "Verification"
+  description "Validates Apex runner"
+  dependsOn validatesRunnerBatch
+}
+
 // Generates :runners:apex:runQuickstartJavaApex
 createJavaQuickstartValidationTask(name: 'Apex')
diff --git a/runners/flink/build.gradle b/runners/flink/build.gradle
index d6673ae246d..3df88913153 100644
--- a/runners/flink/build.gradle
+++ b/runners/flink/build.gradle
@@ -124,7 +124,7 @@ createValidatesRunnerTask(name: "validatesRunnerStreaming", streaming:
true)
 
 task validatesRunner {
   group = "Verification"
-  description "Validates batch and streaming runners"
+  description "Validates Flink runner"
   dependsOn validatesRunnerBatch
   dependsOn validatesRunnerStreaming
 }
diff --git a/runners/gearpump/build.gradle b/runners/gearpump/build.gradle
index cd96dc4cea0..b9181c0c9a7 100644
--- a/runners/gearpump/build.gradle
+++ b/runners/gearpump/build.gradle
@@ -16,13 +16,21 @@
  * limitations under the License.
  */
 
+import groovy.json.JsonOutput
+
 apply from: project(":").file("build_rules.gradle")
 applyJavaNature(artifactId: "beam-runners-gearpump")
 
 description = "Apache Beam :: Runners :: Gearpump"
 
+evaluationDependsOn(":sdks:java:core")
+
 def gearpump_version = "0.8.4"
 
+configurations {
+  validatesRunner
+}
+
 dependencies {
   compile library.java.guava
   compileOnly "com.typesafe:config:1.3.0"
@@ -35,10 +43,41 @@ dependencies {
   shadow library.java.joda_time
   shadow library.java.jackson_annotations
   shadow library.java.findbugs_jsr305
-  testCompile project(path: ":sdks:java:core", configuration: "shadowTest")
-  testCompile library.java.junit
-  testCompile library.java.hamcrest_core
-  testCompile library.java.jackson_databind
-  testCompile library.java.jackson_dataformat_yaml
-  testCompile library.java.mockito_core
+  shadowTest project(path: ":sdks:java:core", configuration: "shadowTest")
+  shadowTest library.java.junit
+  shadowTest library.java.hamcrest_core
+  shadowTest library.java.jackson_databind
+  shadowTest library.java.jackson_dataformat_yaml
+  shadowTest library.java.mockito_core
+  validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+  validatesRunner project(path: project.path, configuration: "shadow")
+}
+
+task validatesRunnerStreaming(type: Test) {
+  group = "Verification"
+  systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+          "--runner=TestGearpumpRunner",
+          "--streaming=true",
+  ])
+
+  classpath = configurations.validatesRunner
+  testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs)
+  useJUnit {
+    includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+    excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesStatefulParDo'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesTimersInParDo'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesSplittableParDo'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesAttemptedMetrics'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesCommittedMetrics'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesCustomWindowMerging'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesParDoLifecycle'
+  }
+}
+
+task validatesRunner {
+  group = "Verification"
+  description "Validates Gearpump runner"
+  dependsOn validatesRunnerStreaming
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesParDoLifecycle.java
b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesParDoLifecycle.java
new file mode 100644
index 00000000000..961616565da
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesParDoLifecycle.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.testing;
+
+/**
+ * Category tag for the ParDoLifecycleTest for exclusion (BEAM-3241).
+ */
+public interface UsesParDoLifecycle {}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java
b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java
index 9ecc31d4a22..3ba516e521f 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java
@@ -31,6 +31,7 @@
 import org.apache.beam.sdk.state.StateSpecs;
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.UsesParDoLifecycle;
 import org.apache.beam.sdk.testing.UsesStatefulParDo;
 import org.apache.beam.sdk.testing.ValidatesRunner;
 import org.apache.beam.sdk.values.KV;
@@ -53,7 +54,7 @@
   public final transient TestPipeline p = TestPipeline.create();
 
   @Test
-  @Category(ValidatesRunner.class)
+  @Category({ValidatesRunner.class, UsesParDoLifecycle.class})
   public void testOldFnCallSequence() {
     PCollectionList.of(p.apply("Impolite", Create.of(1, 2, 4)))
         .and(p.apply("Polite", Create.of(3, 5, 6, 7)))
@@ -64,7 +65,7 @@ public void testOldFnCallSequence() {
   }
 
   @Test
-  @Category(ValidatesRunner.class)
+  @Category({ValidatesRunner.class, UsesParDoLifecycle.class})
   public void testOldFnCallSequenceMulti() {
     PCollectionList.of(p.apply("Impolite", Create.of(1, 2, 4)))
         .and(p.apply("Polite", Create.of(3, 5, 6, 7)))
@@ -133,7 +134,7 @@ public void teardown() {
   }
 
   @Test
-  @Category(ValidatesRunner.class)
+  @Category({ValidatesRunner.class, UsesParDoLifecycle.class})
   public void testFnCallSequence() {
     PCollectionList.of(p.apply("Impolite", Create.of(1, 2, 4)))
         .and(p.apply("Polite", Create.of(3, 5, 6, 7)))
@@ -144,7 +145,7 @@ public void testFnCallSequence() {
   }
 
   @Test
-  @Category(ValidatesRunner.class)
+  @Category({ValidatesRunner.class, UsesParDoLifecycle.class})
   public void testFnCallSequenceMulti() {
     PCollectionList.of(p.apply("Impolite", Create.of(1, 2, 4)))
         .and(p.apply("Polite", Create.of(3, 5, 6, 7)))
@@ -157,7 +158,7 @@ public void testFnCallSequenceMulti() {
   }
 
   @Test
-  @Category({ValidatesRunner.class, UsesStatefulParDo.class})
+  @Category({ValidatesRunner.class, UsesStatefulParDo.class, UsesParDoLifecycle.class})
   public void testFnCallSequenceStateful() {
     PCollectionList.of(p.apply("Impolite", Create.of(KV.of("a", 1), KV.of("b", 2), KV.of("a",
4))))
         .and(
@@ -233,7 +234,7 @@ public void after() {
   }
 
   @Test
-  @Category(ValidatesRunner.class)
+  @Category({ValidatesRunner.class, UsesParDoLifecycle.class})
   public void testTeardownCalledAfterExceptionInStartBundle() {
     ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.START_BUNDLE);
     p
@@ -251,7 +252,7 @@ public void testTeardownCalledAfterExceptionInStartBundle() {
   }
 
   @Test
-  @Category(ValidatesRunner.class)
+  @Category({ValidatesRunner.class, UsesParDoLifecycle.class})
   public void testTeardownCalledAfterExceptionInProcessElement() {
     ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.PROCESS_ELEMENT);
     p
@@ -269,7 +270,7 @@ public void testTeardownCalledAfterExceptionInProcessElement() {
   }
 
   @Test
-  @Category(ValidatesRunner.class)
+  @Category({ValidatesRunner.class, UsesParDoLifecycle.class})
   public void testTeardownCalledAfterExceptionInFinishBundle() {
     ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.FINISH_BUNDLE);
     p
@@ -287,7 +288,7 @@ public void testTeardownCalledAfterExceptionInFinishBundle() {
   }
 
   @Test
-  @Category(ValidatesRunner.class)
+  @Category({ValidatesRunner.class, UsesParDoLifecycle.class})
   public void testWithContextTeardownCalledAfterExceptionInSetup() {
     ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.SETUP);
     p.apply(Create.of(1, 2, 3)).apply(ParDo.of(fn));
@@ -302,7 +303,7 @@ public void testWithContextTeardownCalledAfterExceptionInSetup() {
   }
 
   @Test
-  @Category(ValidatesRunner.class)
+  @Category({ValidatesRunner.class, UsesParDoLifecycle.class})
   public void testWithContextTeardownCalledAfterExceptionInStartBundle() {
     ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.START_BUNDLE);
     p.apply(Create.of(1, 2, 3)).apply(ParDo.of(fn));
@@ -317,7 +318,7 @@ public void testWithContextTeardownCalledAfterExceptionInStartBundle()
{
   }
 
   @Test
-  @Category(ValidatesRunner.class)
+  @Category({ValidatesRunner.class, UsesParDoLifecycle.class})
   public void testWithContextTeardownCalledAfterExceptionInProcessElement() {
     ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.PROCESS_ELEMENT);
     p.apply(Create.of(1, 2, 3)).apply(ParDo.of(fn));
@@ -332,7 +333,7 @@ public void testWithContextTeardownCalledAfterExceptionInProcessElement()
{
   }
 
   @Test
-  @Category(ValidatesRunner.class)
+  @Category({ValidatesRunner.class, UsesParDoLifecycle.class})
   public void testWithContextTeardownCalledAfterExceptionInFinishBundle() {
     ExceptionThrowingOldFn fn = new ExceptionThrowingOldFn(MethodForException.FINISH_BUNDLE);
     p.apply(Create.of(1, 2, 3)).apply(ParDo.of(fn));


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Issue Time Tracking
-------------------

    Worklog Id:     (was: 87839)
    Time Spent: 2h  (was: 1h 50m)

> Migrate ValidatesRunner Jenkins PostCommits to Gradle
> -----------------------------------------------------
>
>                 Key: BEAM-3250
>                 URL: https://issues.apache.org/jira/browse/BEAM-3250
>             Project: Beam
>          Issue Type: Sub-task
>          Components: build-system, testing
>            Reporter: Luke Cwik
>            Assignee: Henning Rohde
>            Priority: Major
>          Time Spent: 2h
>  Remaining Estimate: 0h
>
> Update these targets to execute ValidatesRunner tests: https://github.com/apache/beam/search?l=Groovy&q=ValidatesRunner&type=&utf8=%E2%9C%93



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message