beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rober...@apache.org
Subject [2/2] incubator-beam git commit: [BEAM-1077] @ValidatesRunner Test in Python Postcommit
Date Wed, 07 Dec 2016 20:14:55 GMT
[BEAM-1077] @ValidatesRunner Test in Python Postcommit


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c7626ad2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c7626ad2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c7626ad2

Branch: refs/heads/python-sdk
Commit: c7626ad2f616402dc23809f628d515bcd50f3247
Parents: d5e8c79
Author: Mark Liu <markliu@markliu-macbookpro.roam.corp.google.com>
Authored: Fri Dec 2 13:58:39 2016 -0800
Committer: Robert Bradshaw <robertwb@google.com>
Committed: Wed Dec 7 12:14:38 2016 -0800

----------------------------------------------------------------------
 sdks/python/run_postcommit.sh | 36 ++++++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c7626ad2/sdks/python/run_postcommit.sh
----------------------------------------------------------------------
diff --git a/sdks/python/run_postcommit.sh b/sdks/python/run_postcommit.sh
index 2cd40da..4da078f 100755
--- a/sdks/python/run_postcommit.sh
+++ b/sdks/python/run_postcommit.sh
@@ -54,29 +54,45 @@ echo ">>> RUNNING DIRECT RUNNER py-wordcount"
 python -m apache_beam.examples.wordcount --output /tmp/py-wordcount-direct
 # TODO: check that output file is generated for Direct Runner.
 
-# Run wordcount on the service.
+# Run tests on the service.
 
 # Where to store wordcount output.
 GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
 
 # Job name needs to be unique
-JOBNAME=py-wordcount-`date +%s`
+JOBNAME_E2E=py-wordcount-`date +%s`
+JOBNAME_VR_TEST=py-validatesrunner-`date +%s`
 
 PROJECT=apache-beam-testing
 
 # Create a tarball
 python setup.py sdist
 
+SDK_LOCATION=$(find dist/apache-beam-sdk-*.tar.gz)
+
+# Run ValidatesRunner tests on Google Cloud Dataflow service
+# processes       -> number of processes to run tests in parallel
+# process-timeout -> test timeout in seconds
+python setup.py nosetests \
+  -a ValidatesRunner --processes=4 --process-timeout=360 \
+  --test-pipeline-options=" \
+    --runner=BlockingDataflowPipelineRunner \
+    --project=$PROJECT \
+    --staging_location=$GCS_LOCATION/staging-validatesrunner-test \
+    --sdk_location=$SDK_LOCATION \
+    --job_name=$JOBNAME_VR_TEST \
+    --num_workers=1"
+
 # Run wordcount on the Google Cloud Dataflow service
 python -m apache_beam.examples.wordcount \
---output $GCS_LOCATION/py-wordcount-cloud \
---staging_location $GCS_LOCATION/staging-wordcount \
---temp_location $GCS_LOCATION/temp-wordcount \
---runner BlockingDataflowPipelineRunner \
---job_name $JOBNAME \
---project $PROJECT \
---sdk_location dist/apache-beam-sdk-*.tar.gz \
---num_workers 1 >> job_output 2>&1 || true;
+  --output $GCS_LOCATION/py-wordcount-cloud \
+  --staging_location $GCS_LOCATION/staging-wordcount \
+  --temp_location $GCS_LOCATION/temp-wordcount \
+  --runner BlockingDataflowPipelineRunner \
+  --job_name $JOBNAME_E2E \
+  --project $PROJECT \
+  --sdk_location $SDK_LOCATION \
+  --num_workers 1 >> job_output 2>&1 || true;
 
 # Print full job output, validate correct, then remove it.
 echo ">>> JOB OUTPUT FOLLOWS"


Mime
View raw message