beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhalp...@apache.org
Subject [1/2] beam git commit: [BEAM-1221] Run Wordcount IT in Postcommit Using Test Framework
Date Wed, 28 Dec 2016 19:12:14 GMT
Repository: beam
Updated Branches:
  refs/heads/python-sdk 5107dfad8 -> eff50118b


[BEAM-1221] Run Wordcount IT in Postcommit Using Test Framework


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d58b6495
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d58b6495
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d58b6495

Branch: refs/heads/python-sdk
Commit: d58b6495a929f4b86617214dab11ab467ebb53c1
Parents: 5107dfa
Author: Mark Liu <markliu@google.com>
Authored: Tue Dec 27 16:33:37 2016 -0800
Committer: Dan Halperin <dhalperi@google.com>
Committed: Wed Dec 28 11:11:52 2016 -0800

----------------------------------------------------------------------
 sdks/python/run_postcommit.sh | 35 ++++++++++++++---------------------
 1 file changed, 14 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/d58b6495/sdks/python/run_postcommit.sh
----------------------------------------------------------------------
diff --git a/sdks/python/run_postcommit.sh b/sdks/python/run_postcommit.sh
index 968d4f9..67a257e 100755
--- a/sdks/python/run_postcommit.sh
+++ b/sdks/python/run_postcommit.sh
@@ -60,7 +60,7 @@ python -m apache_beam.examples.wordcount --output /tmp/py-wordcount-direct
 GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
 
 # Job name needs to be unique
-JOBNAME_E2E=py-wordcount-`date +%s`
+JOBNAME_E2E_WC=py-wordcount-`date +%s`
 JOBNAME_VR_TEST=py-validatesrunner-`date +%s`
 
 PROJECT=apache-beam-testing
@@ -71,6 +71,7 @@ python setup.py sdist
 SDK_LOCATION=$(find dist/apache-beam-sdk-*.tar.gz)
 
 # Run ValidatesRunner tests on Google Cloud Dataflow service
+echo ">>> RUNNING DATAFLOW RUNNER VALIDATESRUNNER TESTS"
 python setup.py nosetests \
   -a ValidatesRunner --test-pipeline-options=" \
     --runner=BlockingDataflowRunner \
@@ -82,23 +83,15 @@ python setup.py nosetests \
     --num_workers=1"
 
 # Run wordcount on the Google Cloud Dataflow service
-python -m apache_beam.examples.wordcount \
-  --output $GCS_LOCATION/py-wordcount-cloud \
-  --staging_location $GCS_LOCATION/staging-wordcount \
-  --temp_location $GCS_LOCATION/temp-wordcount \
-  --runner BlockingDataflowRunner \
-  --job_name $JOBNAME_E2E \
-  --project $PROJECT \
-  --sdk_location $SDK_LOCATION \
-  --num_workers 1 >> job_output 2>&1 || true;
-
-# Print full job output, validate correct, then remove it.
-echo ">>> JOB OUTPUT FOLLOWS"
-cat job_output
-
-# Grep will exit with status 1 if success message was not found.
-echo ">>> CHECKING JOB SUCCESS"
-grep JOB_STATE_DONE job_output
-
-# Remove the job output.
-rm job_output
+# and validate job that finishes successfully.
+echo ">>> RUNNING TEST DATAFLOW RUNNER py-wordcount"
+python setup.py nosetests \
+  -a IT --test-pipeline-options=" \
+    --runner=TestDataflowRunner \
+    --project=$PROJECT \
+    --staging_location=$GCS_LOCATION/staging-wordcount \
+    --temp_location=$GCS_LOCATION/temp-wordcount \
+    --output=$GCS_LOCATION/py-wordcount-cloud/output \
+    --sdk_location=$SDK_LOCATION \
+    --job_name=$JOBNAME_E2E_WC \
+    --num_workers=1"


Mime
View raw message