beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lc...@apache.org
Subject [beam] 01/04: Initial post-release snapshot test
Date Thu, 25 Jan 2018 00:10:43 GMT
This is an automated email from the ASF dual-hosted git repository.

lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git

commit 93f4bd4560a6f14c9aa88c9903d60bd327f37c42
Author: Alan Myrvold <alan.myrvold@comcast.net>
AuthorDate: Wed Dec 13 01:26:42 2017 +0000

    Initial post-release snapshot test
---
 .../job_beam_PostRelease_NightlySnapshot.groovy    |  43 +++++++
 release/TestHelper.groovy                          | 139 +++++++++++++++++++++
 release/quickstart-java-direct.groovy              |  64 ++++++++++
 3 files changed, 246 insertions(+)

diff --git a/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy b/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
new file mode 100644
index 0000000..57fa631
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This verifies the nightly snapshot build.
+// From https://repository.apache.org/content/groups/snapshots/org/apache/beam.
+job('beam_PostRelease_NightlySnapshot') {
+  description('Runs post release verification of the nightly snapshot.')
+
+  // Execute concurrent builds if necessary.
+  concurrentBuild()
+
+  // Set common parameters.
+  common_job_properties.setTopLevelMainJobProperties(delegate)
+
+  // This is a post-commit job that runs once per day, not for every push.
+  common_job_properties.setPostCommit(
+      delegate,
+      '0 11 * * *',
+      false,
+      'dev@beam.apache.org')
+
+  steps {
+    // Run a quickstart from https://beam.apache.org/get-started/quickstart-java/
+    shell('cd ' + common_job_properties.checkoutDir + '/release && groovy quickstart-java-direct.groovy')
+  }
+}
diff --git a/release/TestHelper.groovy b/release/TestHelper.groovy
new file mode 100644
index 0000000..2a44791
--- /dev/null
+++ b/release/TestHelper.groovy
@@ -0,0 +1,139 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Helper functions to make writing a test similar to the quickstart
+ * instructions from https://beam.apache.org/get-started/quickstart-java/
+ */
+class TestHelper {
+     
+   // Global state to maintain when running the steps
+   class var {
+     static File startDir 
+     static File curDir
+     static String lastText
+   }
+   
+   // Both documents the overal scenario and creates a clean temp directory
+   def describe(String desc) {
+     var.startDir = File.createTempDir()
+     var.startDir.deleteOnExit()
+     var.curDir = var.startDir
+     print "*****\n* Scenario: ${desc}\n*****\n"
+   }
+   
+   // Just document the intention of a set of steps
+   def it(String desc) { 
+     print "\n*****\n* Test: ${desc}\n*****\n\n"
+   }
+   
+     
+   // Run a command
+   public void run(String cmd) {
+     if (cmd.startsWith("cd ")) {
+       _chdir(cmd.substring(3))
+     } else if (cmd.startsWith("mvn ")) {
+       _mvn(cmd.substring(4))
+     } else {
+       _execute(cmd)
+     }
+   }
+   
+   // Check for expected results in stdout of the last command
+   public void see(String expected) {
+     if (!var.lastText.contains(expected)) {
+       var.startDir.deleteDir()
+       println "Cannot find ${expected} in ${var.lastText}"
+       _error("Cannot find expected text")
+     }
+     println "Verified $expected"
+   }
+   
+   // Cleanup and print success
+   public void done() {
+     var.startDir.deleteDir()
+     println "[SUCCESS]"
+     System.exit(0)
+   }
+
+   // Run a single command, capture output, verify return code is 0
+   private void _execute(String cmd) {
+     def shell = "sh -c cmd".split(' ')
+     shell[2] = cmd
+     def pb = new ProcessBuilder(shell)
+     pb.directory(var.curDir)
+     def proc = pb.start()
+     var.lastText = ""
+     proc.inputStream.eachLine {
+       println it
+       var.lastText += it + "\n";
+     }
+     var.lastText = var.lastText.trim()
+     proc.waitFor()
+     if (proc.exitValue() != 0) {
+       println var.lastText
+       _error("Failed command")
+     }
+   }
+   
+   // Change directory
+   private void _chdir(String subdir) {
+     var.curDir = new File(var.curDir.absolutePath, subdir)
+     if (!var.curDir.exists()) {
+       _error("Directory ${var.curDir} not found")
+     }
+     _execute("pwd")
+     if (var.lastText != var.curDir.absolutePath) {
+       _error("Directory mismatch, ${var.lastText} != ${var.curDir.absolutePath}")
+   
+     }
+   }
+   
+   // Run a maven command, setting up a new local repository and a settings.xml with the
snapshot repository
+   private void _mvn(String args) {
+     def m2 = new File(var.startDir, ".m2/repository")
+     m2.mkdirs()
+     def settings = new File(var.startDir, "settings.xml")
+     settings.write """
+       <settings>
+         <localRepository>${m2.absolutePath}</localRepository>
+           <profiles>
+             <profile>
+               <id>snapshot</id>
+                 <repositories>
+                   <repository>
+                     <id>apache.snapshots</id>
+                     <url>https://repository.apache.org/content/repositories/snapshots</url>
+                   </repository>
+                 </repositories>
+               </profile>
+             </profiles>
+        </settings>
+        """
+       def cmd = "mvn ${args} -s${settings.absolutePath} -Psnapshot -B"
+       _execute(cmd)
+   }
+
+   // Clean up and report error
+   private void _error(String text) {
+     var.startDir.deleteDir()
+     println "[ERROR] $text"
+     System.exit(1)
+   }
+}
diff --git a/release/quickstart-java-direct.groovy b/release/quickstart-java-direct.groovy
new file mode 100644
index 0000000..e00ae2e
--- /dev/null
+++ b/release/quickstart-java-direct.groovy
@@ -0,0 +1,64 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+t = new TestHelper()  
+
+/*
+ * Run the direct quickstart from https://beam.apache.org/get-started/quickstart-java/
+ */
+
+t.describe 'Run Apache Beam Java SDK Quickstart - Direct'
+
+  t.it 'Gets the WordCount Code'
+    ver = "2.3.0-SNAPSHOT"
+
+    // Generate a maven project from the snapshot repository
+    t.run """mvn archetype:generate \
+      -DarchetypeGroupId=org.apache.beam \
+      -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \
+      -DarchetypeVersion=$ver \
+      -DgroupId=org.example \
+      -DartifactId=word-count-beam \
+      -Dversion="0.1" \
+      -Dpackage=org.apache.beam.examples \
+      -DinteractiveMode=false"""
+
+    // Check if it was generated
+    t.see "[INFO] BUILD SUCCESS"
+    t.run "cd word-count-beam"
+    t.run "ls"
+    t.see "pom.xml"
+    t.see "src"
+    t.run "ls src/main/java/org/apache/beam/examples/"
+    t.see "WordCount.java"
+
+  t.it 'Runs the WordCount Code with Direct runner'
+
+    // Run the workcount example with the direct runner
+    t.run """mvn compile exec:java \
+      -Dexec.mainClass=org.apache.beam.examples.WordCount \
+      -Dexec.args="--inputFile=pom.xml --output=counts" \
+      -Pdirect-runner"""
+
+    // Verify text from the pom.xml input file
+    t.run "grep Foundation counts*"
+    t.see "Foundation: 1"
+
+    // Clean up
+    t.done()

-- 
To stop receiving notification emails like this one, please contact
lcwik@apache.org.

Mime
View raw message