beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Work logged] (BEAM-5068) Get Java mobile-gaming auto validations run on any local environments.
Date Mon, 06 Aug 2018 16:45:00 GMT

     [ https://issues.apache.org/jira/browse/BEAM-5068?focusedWorklogId=131474&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-131474
]

ASF GitHub Bot logged work on BEAM-5068:
----------------------------------------

                Author: ASF GitHub Bot
            Created on: 06/Aug/18 16:44
            Start Date: 06/Aug/18 16:44
    Worklog Time Spent: 10m 
      Work Description: pabloem closed pull request #6135: [BEAM-5068] switch mobile gaming
input file to a public storage
URL: https://github.com/apache/beam/pull/6135
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy b/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
index 4679229659b..8d3a7fb1279 100644
--- a/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
+++ b/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
@@ -23,9 +23,6 @@ import CommonJobProperties as commonJobProperties
 job('beam_PostRelease_NightlySnapshot') {
   description('Runs post release verification of the nightly snapshot.')
 
-  // Execute concurrent builds if necessary.
-  concurrentBuild()
-
   // Set common parameters.
   commonJobProperties.setTopLevelMainJobProperties(delegate)
 
diff --git a/release/src/main/groovy/MobileGamingCommands.groovy b/release/src/main/groovy/MobileGamingCommands.groovy
index bf8d04982fa..13874d10469 100644
--- a/release/src/main/groovy/MobileGamingCommands.groovy
+++ b/release/src/main/groovy/MobileGamingCommands.groovy
@@ -22,6 +22,8 @@ class MobileGamingCommands {
 
   private TestScripts testScripts
 
+  private static final INPUT_GAMING_DATA = "gs://dataflow-samples/game/5000_gaming_data.csv"
+
   public static final RUNNERS = [DirectRunner: "direct-runner",
     DataflowRunner: "dataflow-runner",
     SparkRunner: "spark-runner",
@@ -64,10 +66,10 @@ class MobileGamingCommands {
     return "java-hourlyteamscore-result-${RUNNERS[runner]}.txt"
   }
 
-  public String createPipelineCommand(String exampleName, String runner){
+  public String createPipelineCommand(String exampleName, String runner, String jobName=''){
     return """mvn compile exec:java -q \
       -Dexec.mainClass=org.apache.beam.examples.complete.game.${exampleName} \
-      -Dexec.args=\"${getArgs(exampleName, runner)}\" \
+      -Dexec.args=\"${getArgs(exampleName, runner, jobName)}\" \
       -P${RUNNERS[runner]}"""
   }
 
@@ -78,7 +80,7 @@ class MobileGamingCommands {
   }
 
 
-  private String getArgs(String exampleName, String runner){
+  private String getArgs(String exampleName, String runner, String jobName){
     def args
     switch (exampleName) {
       case "UserScore":
@@ -88,10 +90,10 @@ class MobileGamingCommands {
         args = getHourlyTeamScoreArgs(runner)
         break
       case "LeaderBoard":
-        args = getLeaderBoardArgs(runner)
+        args = getLeaderBoardArgs(runner, jobName)
         break
       case "GameStats":
-        args = getGameStatsArgs(runner)
+        args = getGameStatsArgs(runner, jobName)
         break
       default:
         testScripts.error("Cannot find example ${exampleName} in archetypes.")
@@ -104,39 +106,41 @@ class MobileGamingCommands {
 
   private Map getUserScoreArgs(String runner){
     if(runner == "DataflowRunner"){
-      return [input: "gs://${testScripts.gcsBucket()}/5000_gaming_data.csv",
+      return [input: INPUT_GAMING_DATA,
         project: testScripts.gcpProject(),
         output: "gs://${testScripts.gcsBucket()}/${getUserScoreOutputName(runner)}"]
     }
-    return [input: "gs://${testScripts.gcsBucket()}/5000_gaming_data.csv",
+    return [input: INPUT_GAMING_DATA,
       output: "${getUserScoreOutputName(runner)}"]
   }
 
   private Map getHourlyTeamScoreArgs(String runner){
     if(runner == "DataflowRunner"){
-      return [input: "gs://${testScripts.gcsBucket()}/5000_gaming_data.csv",
+      return [input: INPUT_GAMING_DATA,
         project: testScripts.gcpProject(),
         output: "gs://${testScripts.gcsBucket()}/${getHourlyTeamScoreOutputName(runner)}"]
     }
-    return [input: "gs://${testScripts.gcsBucket()}/5000_gaming_data.csv",
+    return [input: INPUT_GAMING_DATA,
       output: "${getHourlyTeamScoreOutputName(runner)}"]
   }
 
-  private Map getLeaderBoardArgs(String runner){
+  private Map getLeaderBoardArgs(String runner, String jobName){
     return [project: testScripts.gcpProject(),
       dataset: testScripts.bqDataset(),
       topic: "projects/${testScripts.gcpProject()}/topics/${testScripts.pubsubTopic()}",
       leaderBoardTableName: "leaderboard_${runner}",
-      teamWindowDuration: 5]
+      teamWindowDuration: 5,
+      jobName: jobName]
   }
 
-  private Map getGameStatsArgs(String runner){
+  private Map getGameStatsArgs(String runner, String jobName){
     return [project: testScripts.gcpProject(),
       dataset: testScripts.bqDataset(),
       topic: "projects/${testScripts.gcpProject()}/topics/${testScripts.pubsubTopic()}",
       fixedWindowDuration: 5,
       userActivityWindowDuration: 5,
       sessionGap: 1,
-      gameStatsTablePrefix: "gamestats_${runner}"]
+      gameStatsTablePrefix: "gamestats_${runner}",
+      jobName: jobName]
   }
 }
diff --git a/release/src/main/groovy/mobilegaming-java-dataflow.groovy b/release/src/main/groovy/mobilegaming-java-dataflow.groovy
index 1d721c86ef5..704870deafe 100644
--- a/release/src/main/groovy/mobilegaming-java-dataflow.groovy
+++ b/release/src/main/groovy/mobilegaming-java-dataflow.groovy
@@ -76,8 +76,9 @@ def InjectorThread = Thread.start() {
   t.run(mobileGamingCommands.createInjectorCommand())
 }
 
+jobName = "leaderboard-validation-" + new Date().getTime() + "-" + new Random().nextInt(1000)
 def LeaderBoardThread = Thread.start() {
-  t.run(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner))
+  t.run(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner, jobName))
 }
 
 t.run("gcloud dataflow jobs list | grep pyflow-wordstream-candidate | grep Running | cut
-d' ' -f1")
@@ -101,7 +102,7 @@ while((System.currentTimeMillis() - startTime)/60000 < mobileGamingCommands.EXEC
 }
 InjectorThread.stop()
 LeaderBoardThread.stop()
-t.run("gcloud dataflow jobs cancel \$(gcloud dataflow jobs list | grep leaderboard-jenkins
| grep Running | cut -d' ' -f1)")
+t.run("gcloud dataflow jobs cancel \$(gcloud dataflow jobs list | grep ${jobName} | grep
Running | cut -d' ' -f1)")
 
 if(!isSuccess){
   t.error("FAILED: Failed running LeaderBoard on DataflowRunner")
diff --git a/release/src/main/groovy/mobilegaming-java-direct.groovy b/release/src/main/groovy/mobilegaming-java-direct.groovy
index b402ae9163a..9ff24cefaf9 100644
--- a/release/src/main/groovy/mobilegaming-java-direct.groovy
+++ b/release/src/main/groovy/mobilegaming-java-direct.groovy
@@ -74,8 +74,9 @@ def InjectorThread = Thread.start() {
   t.run(mobileGamingCommands.createInjectorCommand())
 }
 
+jobName = "leaderboard-validation-" + new Date().getTime() + "-" + new Random().nextInt(1000)
 def LeaderBoardThread = Thread.start() {
-  t.run(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner))
+  t.run(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner, jobName))
 }
 
 // verify outputs in BQ tables


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Issue Time Tracking
-------------------

    Worklog Id:     (was: 131474)
    Time Spent: 3h 40m  (was: 3.5h)

> Get Java mobile-gaming auto validations run on any local environments.
> ----------------------------------------------------------------------
>
>                 Key: BEAM-5068
>                 URL: https://issues.apache.org/jira/browse/BEAM-5068
>             Project: Beam
>          Issue Type: Sub-task
>          Components: testing
>            Reporter: yifan zou
>            Assignee: yifan zou
>            Priority: Major
>          Time Spent: 3h 40m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message