beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chamik...@apache.org
Subject [beam] 01/01: Merge pull request #4210 from szewi/reshuffle-fix
Date Fri, 08 Dec 2017 04:51:00 GMT
This is an automated email from the ASF dual-hosted git repository.

chamikara pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git

commit 7cdbb1cbe559216a9e1b4fb831ab2790f18cafb0
Merge: d4bf90f 84ac804
Author: Chamikara Jayalath <chamikara@google.com>
AuthorDate: Thu Dec 7 20:50:56 2017 -0800

    Merge pull request #4210 from szewi/reshuffle-fix
    
    [BEAM-3060] Temporary fix for failing tests on dataflow runner.

 .../src/test/java/org/apache/beam/sdk/io/text/TextIOIT.java           | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --cc sdks/java/io/file-based-io-tests/src/test/java/org/apache/beam/sdk/io/text/TextIOIT.java
index f9fad80,5f3f540..1a4eccc
--- a/sdks/java/io/file-based-io-tests/src/test/java/org/apache/beam/sdk/io/text/TextIOIT.java
+++ b/sdks/java/io/file-based-io-tests/src/test/java/org/apache/beam/sdk/io/text/TextIOIT.java
@@@ -33,7 -41,12 +33,8 @@@ import org.apache.beam.sdk.io.common.IO
  import org.apache.beam.sdk.testing.PAssert;
  import org.apache.beam.sdk.testing.TestPipeline;
  import org.apache.beam.sdk.transforms.Combine;
 -import org.apache.beam.sdk.transforms.DoFn;
  import org.apache.beam.sdk.transforms.ParDo;
+ import org.apache.beam.sdk.transforms.Reshuffle;
  import org.apache.beam.sdk.transforms.Values;
  import org.apache.beam.sdk.transforms.View;
  import org.apache.beam.sdk.values.PCollection;
@@@ -89,10 -117,10 +90,11 @@@ public class TextIOIT 
  
      PCollection<String> testFilenames = pipeline
          .apply("Generate sequence", GenerateSequence.from(0).to(numberOfTextLines))
 -        .apply("Produce text lines", ParDo.of(new DeterministicallyConstructTestTextLineFn()))
 +        .apply("Produce text lines",
 +            ParDo.of(new FileBasedIOITHelper.DeterministicallyConstructTestTextLineFn()))
          .apply("Write content to files", write)
-         .getPerDestinationOutputFilenames().apply(Values.<String>create());
+         .getPerDestinationOutputFilenames().apply(Values.<String>create())
+         .apply(Reshuffle.<String>viaRandomKey());
  
      PCollection<String> consolidatedHashcode = testFilenames
          .apply("Read all files", TextIO.readAll().withCompression(AUTO))

-- 
To stop receiving notification emails like this one, please contact
"commits@beam.apache.org" <commits@beam.apache.org>.

Mime
View raw message