beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tg...@apache.org
Subject [5/6] beam git commit: Move travis/jenkins folders in a test-infra folder
Date Wed, 12 Apr 2017 20:52:45 GMT
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy b/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy
new file mode 100644
index 0000000..7c64f11
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// Defines a job.
+job('beam_PreCommit_Website_Stage') {
+  description('Stages the pull requests proposed for the Apache Beam ' +
+              'website to a temporary location to ease reviews.')
+
+  // Set common parameters.
+  common_job_properties.setTopLevelWebsiteJobProperties(delegate)
+
+  // Set pull request build trigger.
+  common_job_properties.setPreCommit(
+      delegate,
+      'Automatic staging of pull requests',
+      '\nJenkins built the site at commit id ${ghprbActualCommit} with ' +
+      'Jekyll and staged it [here](http://apache-beam-website-pull-' +
+      'requests.storage.googleapis.com/${ghprbPullId}/index.html). ' +
+      'Happy reviewing.\n\nNote that any previous site has been deleted. ' +
+      'This staged site will be automatically deleted after its TTL ' +
+      'expires. Push any commit to the pull request branch or re-trigger ' +
+      'the build to get it staged again.')
+
+  steps {
+    // Run the following shell script as a build step.
+    shell '''
+        # Install RVM.
+        gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
+            409B6B1796C275462A1703113804BB82D39DC0E3
+        \\curl -sSL https://get.rvm.io | bash
+        source /home/jenkins/.rvm/scripts/rvm
+
+        # Install Ruby.
+        RUBY_VERSION_NUM=2.3.0
+        rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
+
+        # Install Bundler gem
+        PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
+        GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
+        gem install bundler --user-install
+
+        # Install all needed gems.
+        bundle install --path ~/.gem/
+
+        # Remove current site if it exists.
+        GCS_PATH="gs://apache-beam-website-pull-requests/${ghprbPullId}/"
+        gsutil -m rm -r -f ${GCS_PATH} || true
+
+        # Build the new site with the baseurl specified.
+        rm -fr ./content/
+        bundle exec jekyll build --baseurl=/${ghprbPullId}
+
+        # Install BeautifulSoup HTML Parser for python.
+        pip install --user beautifulsoup4
+
+        # Fix links on staged website.
+        python .jenkins/append_index_html_to_internal_links.py
+
+        # Upload the new site.
+        gsutil -m cp -R ./content/* ${GCS_PATH}
+    '''.stripIndent().trim()
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy b/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy
new file mode 100644
index 0000000..421b58a
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// Defines a job.
+job('beam_PreCommit_Website_Test') {
+  description('Runs tests on the pull requests proposed for the Apache Beam ' +
+              'website.')
+
+  // Set common parameters.
+  common_job_properties.setTopLevelWebsiteJobProperties(delegate)
+
+  // Execute concurrent builds. Multiple builds of this project may be executed
+  // in parallel. This is safe because this build does not require exclusive
+  // access to any shared resources.
+  concurrentBuild()
+
+  // Set pull request build trigger.
+  common_job_properties.setPreCommit(
+      delegate,
+      'Test website (dead links, etc.)')
+
+  steps {
+    // Run the following shell script as a build step.
+    shell '''
+        # Install RVM.
+        gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
+            409B6B1796C275462A1703113804BB82D39DC0E3
+        \\curl -sSL https://get.rvm.io | bash
+        source /home/jenkins/.rvm/scripts/rvm
+
+        # Install Ruby.
+        RUBY_VERSION_NUM=2.3.0
+        rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
+
+        # Install Bundler gem
+        PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
+        GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
+        gem install bundler --user-install
+
+        # Install all needed gems.
+        bundle install --path ~/.gem/
+
+        # Build the new site and test it.
+        rm -fr ./content/
+        bundle exec rake test
+    '''.stripIndent().trim()
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy b/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy
new file mode 100644
index 0000000..f2c3ff0
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the nightly snapshot build -- we use this to deploy a daily snapshot
+// to https://repository.apache.org/content/groups/snapshots/org/apache/beam.
+// Runs the postsubmit suite before deploying.
+mavenJob('beam_Release_NightlySnapshot') {
+  description('Runs a mvn clean deploy of the nightly snapshot.')
+
+  // Execute concurrent builds if necessary.
+  concurrentBuild()
+
+  // Set common parameters.
+  common_job_properties.setTopLevelMainJobProperties(delegate)
+
+  // Set maven paramaters.
+  common_job_properties.setMavenConfig(delegate)
+
+  // This is a post-commit job that runs once per day, not for every push.
+  common_job_properties.setPostCommit(
+      delegate,
+      '0 7 * * *',
+      false,
+      'dev@beam.apache.org')
+
+  // Maven goals for this job.
+  goals('-B -e clean deploy -P release,dataflow-runner -DskipITs=false -DintegrationTestPipelineOptions=\'[
"--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner"
]\'')
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_seed.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_seed.groovy b/.test-infra/jenkins/job_seed.groovy
new file mode 100644
index 0000000..2d1b07c
--- /dev/null
+++ b/.test-infra/jenkins/job_seed.groovy
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// Defines the seed job, which creates or updates all other Jenkins projects.
+job('beam_SeedJob') {
+  description('Automatically configures all Apache Beam Jenkins projects based' +
+              ' on Jenkins DSL groovy files checked into the code repository.')
+
+  previousNames('beam_SeedJob_Main')
+
+  // Set common parameters.
+  common_job_properties.setTopLevelMainJobProperties(delegate)
+
+  // This is a post-commit job that runs once per day, not for every push.
+  common_job_properties.setPostCommit(
+      delegate,
+      '0 6 * * *',
+      false,
+      'dev@beam.apache.org')
+
+  // Allows triggering this build against pull requests.
+  common_job_properties.enablePhraseTriggeringFromPullRequest(
+    delegate,
+    'Seed Job',
+    'Run Seed Job')
+
+  steps {
+    dsl {
+      // A list or a glob of other groovy files to process.
+      external('.test-infra/jenkins/job_*.groovy')
+
+      // If a job is removed from the script, disable it (rather than deleting).
+      removeAction('DISABLE')
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/travis/README.md
----------------------------------------------------------------------
diff --git a/.test-infra/travis/README.md b/.test-infra/travis/README.md
new file mode 100644
index 0000000..526995a
--- /dev/null
+++ b/.test-infra/travis/README.md
@@ -0,0 +1,23 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+-->
+
+# Travis Scripts
+
+This directory contains scripts used for [Travis CI](https://travis-ci.org/apache/beam/)
+testing.

http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/travis/settings.xml
----------------------------------------------------------------------
diff --git a/.test-infra/travis/settings.xml b/.test-infra/travis/settings.xml
new file mode 100644
index 0000000..e086aec
--- /dev/null
+++ b/.test-infra/travis/settings.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<settings>
+  <servers>
+    <server>
+      <id>central</id>
+      <configuration>
+        <httpConfiguration>
+          <all>
+            <connectionTimeout>1000</connectionTimeout>
+          </all>
+        </httpConfiguration>
+        <timeout>5000</timeout>
+      </configuration>
+    </server>
+  </servers>
+</settings>

http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/travis/test_wordcount.sh
----------------------------------------------------------------------
diff --git a/.test-infra/travis/test_wordcount.sh b/.test-infra/travis/test_wordcount.sh
new file mode 100755
index 0000000..e059a35
--- /dev/null
+++ b/.test-infra/travis/test_wordcount.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+#
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# This script runs WordCount example locally in a few different ways.
+# Specifically, all combinations of:
+#  a) using mvn exec, or java -cp with a bundled jar file;
+#  b) input filename with no directory component, with a relative directory, or
+#     with an absolute directory; AND
+#  c) input filename containing wildcards or not.
+#
+# The one optional parameter is a path from the directory containing the script
+# to the directory containing the top-level (parent) pom.xml.  If no parameter
+# is provided, the script assumes that directory is equal to the directory
+# containing the script itself.
+#
+# The exit-code of the script indicates success or a failure.
+
+set -e
+set -o pipefail
+
+PASS=1
+VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version
| grep -v '\[')
+JAR_FILE=examples/java/target/beam-examples-java-bundled-${VERSION}.jar
+
+function check_result_hash {
+  local name=$1
+  local outfile_prefix=$2
+  local expected=$3
+
+  local actual=$(LC_ALL=C sort $outfile_prefix-* | md5sum | awk '{print $1}' \
+    || LC_ALL=C sort $outfile_prefix-* | md5 -q) || exit 2  # OSX
+  if [[ "$actual" != "$expected" ]]
+  then
+    echo "FAIL $name: Output hash mismatch.  Got $actual, expected $expected."
+    PASS=""
+    echo "head hexdump of actual:"
+    head $outfile_prefix-* | hexdump -c
+  else
+    echo "pass $name"
+    # Output files are left behind in /tmp
+  fi
+}
+
+function get_outfile_prefix {
+  local name=$1
+  # NOTE: mktemp on OSX doesn't support --tmpdir
+  mktemp -u "/tmp/$name.out.XXXXXXXXXX"
+}
+
+function run_via_mvn {
+  local name=$1
+  local input=$2
+  local expected_hash=$3
+
+  local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
+  local cmd='mvn exec:java -f pom.xml -pl examples/java \
+    -Dexec.mainClass=org.apache.beam.examples.WordCount \
+    -Dexec.args="--runner=DirectRunner --inputFile='"$input"' --output='"$outfile_prefix"'"'
+  echo "$name: Running $cmd" >&2
+  sh -c "$cmd"
+  check_result_hash "$name" "$outfile_prefix" "$expected_hash"
+}
+
+function run_bundled {
+  local name=$1
+  local input=$2
+  local expected_hash=$3
+
+  local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
+  local cmd='java -cp '"$JAR_FILE"' \
+    org.apache.beam.examples.WordCount \
+    --runner=DirectRunner \
+    --inputFile='"'$input'"' \
+    --output='"$outfile_prefix"
+  echo "$name: Running $cmd" >&2
+  sh -c "$cmd"
+  check_result_hash "$name" "$outfile_prefix" "$expected_hash"
+}
+
+function run_all_ways {
+  local name=$1
+  local input=$2
+  local expected_hash=$3
+
+  run_via_mvn ${name}a "$input" $expected_hash
+  check_for_jar_file
+  run_bundled ${name}b "$input" $expected_hash
+}
+
+function check_for_jar_file {
+  if [[ ! -f $JAR_FILE ]]
+  then
+    echo "Jar file $JAR_FILE not created" >&2
+    exit 2
+  fi
+}
+
+run_all_ways wordcount1 "LICENSE" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount2 "./LICENSE" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount3 "$PWD/LICENSE" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount4 "L*N?E*" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount5 "./LICE*N?E" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount6 "$PWD/*LIC?NSE" c5350a5ad4bb51e3e018612b4b044097
+
+if [[ ! "$PASS" ]]
+then
+  echo "One or more tests FAILED."
+  exit 1
+fi
+echo "All tests PASS"

http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 2a75ebc..0b5d700 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -34,7 +34,7 @@ addons:
     - python2.7
 env:
   global:
-   - MAVEN_OVERRIDE="--settings=.travis/settings.xml"
+   - MAVEN_OVERRIDE="--settings=.test-infra/travis/settings.xml"
    - MAVEN_CONTAINER_OVERRIDE="-DbeamSurefireArgline='-Xmx512m'"
 
 matrix:
@@ -80,7 +80,7 @@ install:
 
 script:
   - if [ "$TEST_PYTHON" ]; then travis_retry $TOX_HOME/tox -c sdks/python/tox.ini; fi
-  - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates
--threads 1C $MAVEN_OVERRIDE install && travis_retry bash -ex .travis/test_wordcount.sh;
fi
+  - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates
--threads 1C $MAVEN_OVERRIDE install && travis_retry bash -ex .test-infra/travis/test_wordcount.sh;
fi
 
 cache:
   directories:

http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.travis/README.md
----------------------------------------------------------------------
diff --git a/.travis/README.md b/.travis/README.md
deleted file mode 100644
index 526995a..0000000
--- a/.travis/README.md
+++ /dev/null
@@ -1,23 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
--->
-
-# Travis Scripts
-
-This directory contains scripts used for [Travis CI](https://travis-ci.org/apache/beam/)
-testing.

http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.travis/settings.xml
----------------------------------------------------------------------
diff --git a/.travis/settings.xml b/.travis/settings.xml
deleted file mode 100644
index e086aec..0000000
--- a/.travis/settings.xml
+++ /dev/null
@@ -1,33 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
--->
-<settings>
-  <servers>
-    <server>
-      <id>central</id>
-      <configuration>
-        <httpConfiguration>
-          <all>
-            <connectionTimeout>1000</connectionTimeout>
-          </all>
-        </httpConfiguration>
-        <timeout>5000</timeout>
-      </configuration>
-    </server>
-  </servers>
-</settings>

http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.travis/test_wordcount.sh
----------------------------------------------------------------------
diff --git a/.travis/test_wordcount.sh b/.travis/test_wordcount.sh
deleted file mode 100755
index e059a35..0000000
--- a/.travis/test_wordcount.sh
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/bin/bash
-#
-#    Licensed to the Apache Software Foundation (ASF) under one or more
-#    contributor license agreements.  See the NOTICE file distributed with
-#    this work for additional information regarding copyright ownership.
-#    The ASF licenses this file to You under the Apache License, Version 2.0
-#    (the "License"); you may not use this file except in compliance with
-#    the License.  You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-# This script runs WordCount example locally in a few different ways.
-# Specifically, all combinations of:
-#  a) using mvn exec, or java -cp with a bundled jar file;
-#  b) input filename with no directory component, with a relative directory, or
-#     with an absolute directory; AND
-#  c) input filename containing wildcards or not.
-#
-# The one optional parameter is a path from the directory containing the script
-# to the directory containing the top-level (parent) pom.xml.  If no parameter
-# is provided, the script assumes that directory is equal to the directory
-# containing the script itself.
-#
-# The exit-code of the script indicates success or a failure.
-
-set -e
-set -o pipefail
-
-PASS=1
-VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version
| grep -v '\[')
-JAR_FILE=examples/java/target/beam-examples-java-bundled-${VERSION}.jar
-
-function check_result_hash {
-  local name=$1
-  local outfile_prefix=$2
-  local expected=$3
-
-  local actual=$(LC_ALL=C sort $outfile_prefix-* | md5sum | awk '{print $1}' \
-    || LC_ALL=C sort $outfile_prefix-* | md5 -q) || exit 2  # OSX
-  if [[ "$actual" != "$expected" ]]
-  then
-    echo "FAIL $name: Output hash mismatch.  Got $actual, expected $expected."
-    PASS=""
-    echo "head hexdump of actual:"
-    head $outfile_prefix-* | hexdump -c
-  else
-    echo "pass $name"
-    # Output files are left behind in /tmp
-  fi
-}
-
-function get_outfile_prefix {
-  local name=$1
-  # NOTE: mktemp on OSX doesn't support --tmpdir
-  mktemp -u "/tmp/$name.out.XXXXXXXXXX"
-}
-
-function run_via_mvn {
-  local name=$1
-  local input=$2
-  local expected_hash=$3
-
-  local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
-  local cmd='mvn exec:java -f pom.xml -pl examples/java \
-    -Dexec.mainClass=org.apache.beam.examples.WordCount \
-    -Dexec.args="--runner=DirectRunner --inputFile='"$input"' --output='"$outfile_prefix"'"'
-  echo "$name: Running $cmd" >&2
-  sh -c "$cmd"
-  check_result_hash "$name" "$outfile_prefix" "$expected_hash"
-}
-
-function run_bundled {
-  local name=$1
-  local input=$2
-  local expected_hash=$3
-
-  local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
-  local cmd='java -cp '"$JAR_FILE"' \
-    org.apache.beam.examples.WordCount \
-    --runner=DirectRunner \
-    --inputFile='"'$input'"' \
-    --output='"$outfile_prefix"
-  echo "$name: Running $cmd" >&2
-  sh -c "$cmd"
-  check_result_hash "$name" "$outfile_prefix" "$expected_hash"
-}
-
-function run_all_ways {
-  local name=$1
-  local input=$2
-  local expected_hash=$3
-
-  run_via_mvn ${name}a "$input" $expected_hash
-  check_for_jar_file
-  run_bundled ${name}b "$input" $expected_hash
-}
-
-function check_for_jar_file {
-  if [[ ! -f $JAR_FILE ]]
-  then
-    echo "Jar file $JAR_FILE not created" >&2
-    exit 2
-  fi
-}
-
-run_all_ways wordcount1 "LICENSE" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount2 "./LICENSE" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount3 "$PWD/LICENSE" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount4 "L*N?E*" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount5 "./LICE*N?E" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount6 "$PWD/*LIC?NSE" c5350a5ad4bb51e3e018612b4b044097
-
-if [[ ! "$PASS" ]]
-then
-  echo "One or more tests FAILED."
-  exit 1
-fi
-echo "All tests PASS"


Mime
View raw message