beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From k...@apache.org
Subject [1/2] beam git commit: Uniquify application nodes in TextIOReadTest and only run pipeline once
Date Fri, 21 Jul 2017 19:45:23 GMT
Repository: beam
Updated Branches:
  refs/heads/master 4b355844a -> 81c2e90f3


Uniquify application nodes in TextIOReadTest and only run pipeline once

 - These nodes have never had stable unique names, but we have always mutated
   the pipeline before running it so they got lucky.
 - A pipeline can also be mutated when run() so it should be considered dead
   after run() is called.

This fixes both issues by uniquifying the names and running a given pipeline
only once.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/cfadecb5
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/cfadecb5
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/cfadecb5

Branch: refs/heads/master
Commit: cfadecb56c64aa155c8b5fd0d8a6654ceb918eba
Parents: 1d9160f
Author: Kenneth Knowles <klk@google.com>
Authored: Mon Jul 17 22:03:09 2017 -0700
Committer: Kenneth Knowles <klk@google.com>
Committed: Thu Jul 20 20:06:54 2017 -0700

----------------------------------------------------------------------
 .../org/apache/beam/sdk/io/TextIOReadTest.java  | 22 ++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/cfadecb5/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java
index 8b53111..e733010 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java
@@ -94,6 +94,7 @@ public class TextIOReadTest {
   private static final List<String> TINY =
       Arrays.asList("Irritable eagle", "Optimistic jay", "Fanciful hawk");
   private static final List<String> LARGE = makeLines(1000);
+  private static int uniquifier = 0;
 
   private static Path tempFolder;
   private static File emptyTxt;
@@ -296,17 +297,19 @@ public class TextIOReadTest {
   private void assertReadingCompressedFileMatchesExpected(
       File file, CompressionType compressionType, List<String> expected) {
 
+    int thisUniquifier = ++uniquifier;
+
     TextIO.Read read = TextIO.read().from(file.getPath()).withCompressionType(compressionType);
-    PAssert.that(p.apply("Read_" + file + "_" + compressionType.toString(), read))
+    PAssert.that(
+            p.apply("Read_" + file + "_" + compressionType.toString() + "_" + thisUniquifier,
read))
         .containsInAnyOrder(expected);
 
     TextIO.ReadAll readAll =
         TextIO.readAll().withCompressionType(compressionType).withDesiredBundleSizeBytes(10);
     PAssert.that(
-            p.apply("Create_" + file, Create.of(file.getPath()))
-                .apply("Read_" + compressionType.toString(), readAll))
+            p.apply("Create_" + file + "_" + thisUniquifier, Create.of(file.getPath()))
+                .apply("Read_" + compressionType.toString() + "_" + thisUniquifier, readAll))
         .containsInAnyOrder(expected);
-    p.run();
   }
 
   /** Helper to make an array of compressible strings. Returns ["word"i] for i in range(0,n).
*/
@@ -324,6 +327,7 @@ public class TextIOReadTest {
   public void testSmallCompressedGzipReadNoExtension() throws Exception {
     File smallGzNoExtension = writeToFile(TINY, "tiny_gz_no_extension", GZIP);
     assertReadingCompressedFileMatchesExpected(smallGzNoExtension, GZIP, TINY);
+    p.run();
   }
 
   /**
@@ -340,6 +344,7 @@ public class TextIOReadTest {
     assertReadingCompressedFileMatchesExpected(smallGzNotCompressed, GZIP, TINY);
     // Should also work with AUTO mode set.
     assertReadingCompressedFileMatchesExpected(smallGzNotCompressed, AUTO, TINY);
+    p.run();
   }
 
   /** Tests reading from a small, bzip2ed file with no .bz2 extension but BZIP2 compression
set. */
@@ -348,6 +353,7 @@ public class TextIOReadTest {
   public void testSmallCompressedBzip2ReadNoExtension() throws Exception {
     File smallBz2NoExtension = writeToFile(TINY, "tiny_bz2_no_extension", BZIP2);
     assertReadingCompressedFileMatchesExpected(smallBz2NoExtension, BZIP2, TINY);
+    p.run();
   }
 
   /**
@@ -393,6 +399,7 @@ public class TextIOReadTest {
       assertReadingCompressedFileMatchesExpected(tinyTxt, type, TINY);
       assertReadingCompressedFileMatchesExpected(largeTxt, type, LARGE);
     }
+    p.run();
   }
 
   @Test
@@ -411,6 +418,7 @@ public class TextIOReadTest {
     // GZIP files with non-gz extension should work in GZIP mode.
     File gzFile = writeToFile(TINY, "tiny_gz_no_extension", GZIP);
     assertReadingCompressedFileMatchesExpected(gzFile, GZIP, TINY);
+    p.run();
   }
 
   @Test
@@ -429,6 +437,7 @@ public class TextIOReadTest {
     // BZ2 files with non-bz2 extension should work in BZIP2 mode.
     File bz2File = writeToFile(TINY, "tiny_bz2_no_extension", BZIP2);
     assertReadingCompressedFileMatchesExpected(bz2File, BZIP2, TINY);
+    p.run();
   }
 
   @Test
@@ -447,6 +456,7 @@ public class TextIOReadTest {
     // Zip files with non-zip extension should work in ZIP mode.
     File zipFile = writeToFile(TINY, "tiny_zip_no_extension", ZIP);
     assertReadingCompressedFileMatchesExpected(zipFile, ZIP, TINY);
+    p.run();
   }
 
   @Test
@@ -465,6 +475,7 @@ public class TextIOReadTest {
     // Deflate files with non-deflate extension should work in DEFLATE mode.
     File deflateFile = writeToFile(TINY, "tiny_deflate_no_extension", DEFLATE);
     assertReadingCompressedFileMatchesExpected(deflateFile, DEFLATE, TINY);
+    p.run();
   }
 
   /**
@@ -476,6 +487,7 @@ public class TextIOReadTest {
   public void testZipCompressedReadWithNoEntries() throws Exception {
     String filename = createZipFile(new ArrayList<String>(), "empty zip file");
     assertReadingCompressedFileMatchesExpected(new File(filename), CompressionType.ZIP, EMPTY);
+    p.run();
   }
 
   /**
@@ -493,6 +505,7 @@ public class TextIOReadTest {
 
     String filename = createZipFile(expected, "multiple entries", entry0, entry1, entry2);
     assertReadingCompressedFileMatchesExpected(new File(filename), CompressionType.ZIP, expected);
+    p.run();
   }
 
   /**
@@ -513,6 +526,7 @@ public class TextIOReadTest {
 
     assertReadingCompressedFileMatchesExpected(
         new File(filename), CompressionType.ZIP, Arrays.asList("cat", "dog"));
+    p.run();
   }
 
   @Test


Mime
View raw message