beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject [1/2] beam git commit: Move website jobs to main Beam repo to reduce duplication.
Date Fri, 03 Feb 2017 00:39:18 GMT
Repository: beam
Updated Branches:
  refs/heads/master c0910fdf4 -> 1533d7013


Move website jobs to main Beam repo to reduce duplication.

Signed-off-by: Jason Kuster <jasonkuster@google.com>


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/5dc83f57
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/5dc83f57
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/5dc83f57

Branch: refs/heads/master
Commit: 5dc83f577e7b4275475556bf3bd0212bf69e3d35
Parents: c0910fd
Author: Jason Kuster <jasonkuster@google.com>
Authored: Wed Feb 1 16:39:40 2017 -0800
Committer: Davor Bonaci <davor@google.com>
Committed: Thu Feb 2 16:38:47 2017 -0800

----------------------------------------------------------------------
 .jenkins/common_job_properties.groovy           | 40 +++++++++----
 ...job_beam_PostCommit_Java_MavenInstall.groovy |  2 +-
 ...ostCommit_Java_RunnableOnService_Apex.groovy |  2 +-
 ...ommit_Java_RunnableOnService_Dataflow.groovy |  2 +-
 ...stCommit_Java_RunnableOnService_Flink.groovy |  2 +-
 ...ommit_Java_RunnableOnService_Gearpump.groovy |  4 +-
 ...stCommit_Java_RunnableOnService_Spark.groovy |  2 +-
 .../job_beam_PostCommit_Python_Verify.groovy    |  2 +-
 .../job_beam_PreCommit_Java_MavenInstall.groovy |  2 +-
 .../job_beam_PreCommit_Website_Stage.groovy     | 62 ++++++++++++++++++++
 .jenkins/job_beam_PreCommit_Website_Test.groovy | 47 +++++++++++++++
 .../job_beam_Release_NightlySnapshot.groovy     |  2 +-
 .jenkins/job_seed.groovy                        | 11 ++--
 13 files changed, 154 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/common_job_properties.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/common_job_properties.groovy b/.jenkins/common_job_properties.groovy
index f1f2b7f..eee1cda 100644
--- a/.jenkins/common_job_properties.groovy
+++ b/.jenkins/common_job_properties.groovy
@@ -20,14 +20,27 @@
 // common properties that are shared among all Jenkins projects.
 class common_job_properties {
 
-  // Sets common top-level job properties.
-  static def setTopLevelJobProperties(def context,
-                                      def default_branch = 'master',
-                                      def default_timeout = 100) {
+  // Sets common top-level job properties for website repository jobs.
+  static def setTopLevelWebsiteJobProperties(def context) {
+    setTopLevelJobProperties(context, 'beam-site', 'asf-site', 30)
+  }
+
+  // Sets common top-level job properties for main repository jobs.
+  static def setTopLevelMainJobProperties(def context,
+                                          def default_branch = 'master') {
+    setTopLevelJobProperties(context, 'beam', default_branch, 100)
+  }
+
+  // Sets common top-level job properties. Should be accessed through one of the
+  // above methods to protect jobs from internal details of param defaults.
+  private static def setTopLevelJobProperties(def context,
+                                              def repository_name,
+                                              def default_branch,
+                                              def default_timeout) {
 
     // GitHub project.
     context.properties {
-      githubProjectUrl('https://github.com/apache/beam/')
+      githubProjectUrl('https://github.com/apache/' + repository_name + '/')
     }
 
     // Set JDK version.
@@ -46,7 +59,7 @@ class common_job_properties {
     context.scm {
       git {
         remote {
-          url('https://github.com/apache/beam.git')
+          url('https://github.com/apache/' + repository_name + '.git')
           refspec('+refs/heads/*:refs/remotes/origin/* ' +
                   '+refs/pull/*:refs/remotes/origin/pr/*')
         }
@@ -83,10 +96,11 @@ class common_job_properties {
     }
   }
 
-  // Sets the pull request build trigger.
-  static def setPullRequestBuildTrigger(def context,
-                                        def commitStatusContext,
-                                        def successComment = '--none--') {
+  // Sets the pull request build trigger. Accessed through precommit methods
+  // below to insulate callers from internal parameter defaults.
+  private static def setPullRequestBuildTrigger(def context,
+                                                def commitStatusContext,
+                                                def successComment = '--none--') {
     context.triggers {
       githubPullRequest {
         admins(['asfbot'])
@@ -150,9 +164,11 @@ class common_job_properties {
   }
 
   // Sets common config for PreCommit jobs.
-  static def setPreCommit(def context, comment) {
+  static def setPreCommit(def context,
+                          def commitStatusName,
+                          def successComment = '--none--') {
     // Set pull request build trigger.
-    setPullRequestBuildTrigger(context, comment)
+    setPullRequestBuildTrigger(context, commitStatusName, successComment)
   }
 
   // Sets common config for PostCommit jobs.

http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy b/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
index 31257e2..a288a84 100644
--- a/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
+++ b/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
@@ -29,7 +29,7 @@ mavenJob('beam_PostCommit_Java_MavenInstall') {
   concurrentBuild()
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set maven parameters.
   common_job_properties.setMavenConfig(delegate)

http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy
index 232c94e..f657596 100644
--- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy
+++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy
@@ -23,7 +23,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Apex') {
   description('Runs the RunnableOnService suite on the Apex runner.')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set maven parameters.
   common_job_properties.setMavenConfig(delegate)

http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy
index 44136ad..f849c83 100644
--- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy
+++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy
@@ -26,7 +26,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Dataflow') {
   previousNames('beam_PostCommit_RunnableOnService_GoogleCloudDataflow')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set maven parameters.
   common_job_properties.setMavenConfig(delegate)

http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy
index 5a5863f..c0caa51 100644
--- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy
+++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy
@@ -25,7 +25,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Flink') {
   previousNames('beam_PostCommit_RunnableOnService_FlinkLocal')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set maven parameters.
   common_job_properties.setMavenConfig(delegate)

http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy
index f07bbc6..1e35017 100644
--- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy
+++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy
@@ -26,7 +26,9 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Gearpump') {
   previousNames('beam_PostCommit_RunnableOnService_GearpumpLocal')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate, 'gearpump-runner')
+  common_job_properties.setTopLevelMainJobProperties(
+      delegate,
+      'gearpump-runner')
 
   // Set maven parameters.
   common_job_properties.setMavenConfig(delegate)

http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy
index 875c6e6..97f6a70 100644
--- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy
+++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy
@@ -25,7 +25,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Spark') {
   previousNames('beam_PostCommit_RunnableOnService_SparkLocal')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set maven parameters.
   common_job_properties.setMavenConfig(delegate)

http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Python_Verify.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Python_Verify.groovy b/.jenkins/job_beam_PostCommit_Python_Verify.groovy
index 7f40a15..d92958c 100644
--- a/.jenkins/job_beam_PostCommit_Python_Verify.groovy
+++ b/.jenkins/job_beam_PostCommit_Python_Verify.groovy
@@ -25,7 +25,7 @@ job('beam_PostCommit_Python_Verify') {
   previousNames('beam_PostCommit_PythonVerify')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Sets that this is a PostCommit job.
   common_job_properties.setPostCommit(delegate, '0 3-22/6 * * *')

http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
index 153df14..ff2e476 100644
--- a/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
+++ b/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
@@ -29,7 +29,7 @@ mavenJob('beam_PreCommit_Java_MavenInstall') {
   concurrentBuild()
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set Maven parameters.
   common_job_properties.setMavenConfig(delegate)

http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PreCommit_Website_Stage.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PreCommit_Website_Stage.groovy b/.jenkins/job_beam_PreCommit_Website_Stage.groovy
new file mode 100644
index 0000000..b63d40a
--- /dev/null
+++ b/.jenkins/job_beam_PreCommit_Website_Stage.groovy
@@ -0,0 +1,62 @@
+import common_job_properties
+
+// Defines a job.
+job('beam_PreCommit_Website_Stage') {
+  description('Stages the pull requests proposed for the Apache Beam ' +
+              'website to a temporary location to ease reviews.')
+
+  // Set common parameters.
+  common_job_properties.setTopLevelWebsiteJobProperties(delegate)
+
+  // Set pull request build trigger.
+  common_job_properties.setPreCommit(
+      delegate,
+      'Jenkins: automatic staging of pull requests',
+      '\nJenkins built the site at commit id ${ghprbActualCommit} with ' +
+      'Jekyll and staged it [here](http://apache-beam-website-pull-' +
+      'requests.storage.googleapis.com/${ghprbPullId}/index.html). ' +
+      'Happy reviewing.\n\nNote that any previous site has been deleted. ' +
+      'This staged site will be automatically deleted after its TTL ' +
+      'expires. Push any commit to the pull request branch or re-trigger ' +
+      'the build to get it staged again.')
+
+  steps {
+    // Run the following shell script as a build step.
+    shell '''
+        # Install RVM.
+        gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
+            409B6B1796C275462A1703113804BB82D39DC0E3
+        \\curl -sSL https://get.rvm.io | bash
+        source /home/jenkins/.rvm/scripts/rvm
+
+        # Install Ruby.
+        RUBY_VERSION_NUM=2.3.0
+        rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
+
+        # Install Bundler gem
+        PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
+        GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
+        gem install bundler --user-install
+
+        # Install all needed gems.
+        bundle install --path ~/.gem/
+
+        # Remove current site if it exists.
+        GCS_PATH="gs://apache-beam-website-pull-requests/${ghprbPullId}/"
+        gsutil -m rm -r -f ${GCS_PATH} || true
+
+        # Build the new site with the baseurl specified.
+        rm -fr ./content/
+        bundle exec jekyll build --baseurl=/${ghprbPullId}
+
+        # Install BeautifulSoup HTML Parser for python.
+        pip install --user beautifulsoup4
+
+        # Fix links on staged website.
+        python .jenkins/append_index_html_to_internal_links.py
+
+        # Upload the new site.
+        gsutil -m cp -R ./content/* ${GCS_PATH}
+    '''.stripIndent().trim()
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PreCommit_Website_Test.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PreCommit_Website_Test.groovy b/.jenkins/job_beam_PreCommit_Website_Test.groovy
new file mode 100644
index 0000000..ada089f
--- /dev/null
+++ b/.jenkins/job_beam_PreCommit_Website_Test.groovy
@@ -0,0 +1,47 @@
+import common_job_properties
+
+// Defines a job.
+job('beam_PreCommit_Website_Test') {
+  description('Runs tests on the pull requests proposed for the Apache Beam ' +
+              'website.')
+
+  // Set common parameters.
+  common_job_properties.setTopLevelWebsiteJobProperties(delegate)
+
+  // Execute concurrent builds. Multiple builds of this project may be executed
+  // in parallel. This is safe because this build does not require exclusive
+  // access to any shared resources.
+  concurrentBuild()
+
+  // Set pull request build trigger.
+  common_job_properties.setPreCommit(
+      delegate,
+      'Jenkins: test website (dead links, etc.)')
+
+  steps {
+    // Run the following shell script as a build step.
+    shell '''
+        # Install RVM.
+        gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
+            409B6B1796C275462A1703113804BB82D39DC0E3
+        \\curl -sSL https://get.rvm.io | bash
+        source /home/jenkins/.rvm/scripts/rvm
+
+        # Install Ruby.
+        RUBY_VERSION_NUM=2.3.0
+        rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
+
+        # Install Bundler gem
+        PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
+        GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
+        gem install bundler --user-install
+
+        # Install all needed gems.
+        bundle install --path ~/.gem/
+
+        # Build the new site and test it.
+        rm -fr ./content/
+        bundle exec rake test
+    '''.stripIndent().trim()
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_Release_NightlySnapshot.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_Release_NightlySnapshot.groovy b/.jenkins/job_beam_Release_NightlySnapshot.groovy
index a19f8ff..f2c3ff0 100644
--- a/.jenkins/job_beam_Release_NightlySnapshot.groovy
+++ b/.jenkins/job_beam_Release_NightlySnapshot.groovy
@@ -28,7 +28,7 @@ mavenJob('beam_Release_NightlySnapshot') {
   concurrentBuild()
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set maven paramaters.
   common_job_properties.setMavenConfig(delegate)

http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_seed.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_seed.groovy b/.jenkins/job_seed.groovy
index fcdd564..b830f88 100644
--- a/.jenkins/job_seed.groovy
+++ b/.jenkins/job_seed.groovy
@@ -19,13 +19,14 @@
 import common_job_properties
 
 // Defines the seed job, which creates or updates all other Jenkins projects.
-job('beam_SeedJob_Main') {
-  description('Automatically configures all Apache Beam main repo Jenkins ' +
-              'projects based on Jenkins DSL groovy files checked into the ' +
-              'code repository.')
+job('beam_SeedJob') {
+  description('Automatically configures all Apache Beam Jenkins projects based' +
+              ' on Jenkins DSL groovy files checked into the code repository.')
+
+  previousNames('beam_SeedJob_Main')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // This is a post-commit job that runs once per day, not for every push.
   common_job_properties.setPostCommit(


Mime
View raw message