beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhalp...@apache.org
Subject [1/2] beam git commit: [BEAM-59] Delete old restrictions on output file paths
Date Tue, 02 May 2017 21:53:19 GMT
Repository: beam
Updated Branches:
  refs/heads/master ccbb00e38 -> b5561f718


[BEAM-59] Delete old restrictions on output file paths

These predate Apache Beam and are no longer relevant now that Text and Avro are implemented
in the SDK


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/00bee9b7
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/00bee9b7
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/00bee9b7

Branch: refs/heads/master
Commit: 00bee9b70f6fabbb6bfe0655188f8d2c0a7239f9
Parents: ccbb00e
Author: Dan Halperin <dhalperi@google.com>
Authored: Mon May 1 23:36:57 2017 -0700
Committer: Dan Halperin <dhalperi@google.com>
Committed: Tue May 2 14:48:21 2017 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/beam/sdk/io/AvroIO.java | 14 --------------
 .../main/java/org/apache/beam/sdk/io/TFRecordIO.java | 14 --------------
 .../src/main/java/org/apache/beam/sdk/io/TextIO.java | 14 --------------
 .../test/java/org/apache/beam/sdk/io/TextIOTest.java | 15 ---------------
 4 files changed, 57 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/00bee9b7/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
index 755cdb9..3bb61a2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
@@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import com.google.common.io.BaseEncoding;
 import java.util.Map;
-import java.util.regex.Pattern;
 import javax.annotation.Nullable;
 import org.apache.avro.Schema;
 import org.apache.avro.file.CodecFactory;
@@ -302,7 +301,6 @@ public class AvroIO {
      * in a common extension, if given by {@link #withSuffix}.
      */
     public Write<T> to(String filenamePrefix) {
-      validateOutputComponent(filenamePrefix);
       return toBuilder().setFilenamePrefix(filenamePrefix).build();
     }
 
@@ -317,7 +315,6 @@ public class AvroIO {
      * <p>See {@link ShardNameTemplate} for a description of shard templates.
      */
     public Write<T> withSuffix(String filenameSuffix) {
-      validateOutputComponent(filenameSuffix);
       return toBuilder().setFilenameSuffix(filenameSuffix).build();
     }
 
@@ -474,17 +471,6 @@ public class AvroIO {
     }
   }
 
-  // Pattern which matches old-style shard output patterns, which are now
-  // disallowed.
-  private static final Pattern SHARD_OUTPUT_PATTERN = Pattern.compile("@([0-9]+|\\*)");
-
-  private static void validateOutputComponent(String partialFilePattern) {
-    checkArgument(
-        !SHARD_OUTPUT_PATTERN.matcher(partialFilePattern).find(),
-        "Output name components are not allowed to contain @* or @N patterns: "
-        + partialFilePattern);
-  }
-
   /////////////////////////////////////////////////////////////////////////////
 
   /** Disallow construction of utility class. */

http://git-wip-us.apache.org/repos/asf/beam/blob/00bee9b7/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
index 8a1870e..fe0b97d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
@@ -30,7 +30,6 @@ import java.nio.ByteOrder;
 import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.WritableByteChannel;
 import java.util.NoSuchElementException;
-import java.util.regex.Pattern;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.Coder;
@@ -268,7 +267,6 @@ public class TFRecordIO {
      * in a common extension, if given by {@link #withSuffix(String)}.
      */
     public Write to(String filenamePrefix) {
-      validateOutputComponent(filenamePrefix);
       return to(StaticValueProvider.of(filenamePrefix));
     }
 
@@ -285,7 +283,6 @@ public class TFRecordIO {
      * @see ShardNameTemplate
      */
     public Write withSuffix(String nameExtension) {
-      validateOutputComponent(nameExtension);
       return toBuilder().setFilenameSuffix(nameExtension).build();
     }
 
@@ -422,17 +419,6 @@ public class TFRecordIO {
     }
   }
 
-  // Pattern which matches old-style shard output patterns, which are now
-  // disallowed.
-  private static final Pattern SHARD_OUTPUT_PATTERN = Pattern.compile("@([0-9]+|\\*)");
-
-  private static void validateOutputComponent(String partialFilePattern) {
-    checkArgument(
-        !SHARD_OUTPUT_PATTERN.matcher(partialFilePattern).find(),
-        "Output name components are not allowed to contain @* or @N patterns: "
-            + partialFilePattern);
-  }
-
   //////////////////////////////////////////////////////////////////////////////
 
   /** Disable construction of utility class. */

http://git-wip-us.apache.org/repos/asf/beam/blob/00bee9b7/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
index 1f9b7a0..6b08e1f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
@@ -21,7 +21,6 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.auto.value.AutoValue;
-import java.util.regex.Pattern;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -290,7 +289,6 @@ public class TextIO {
      * in a common extension, if given by {@link #withSuffix(String)}.
      */
     public Write to(String filenamePrefix) {
-      validateOutputComponent(filenamePrefix);
       return to(StaticValueProvider.of(filenamePrefix));
     }
 
@@ -310,7 +308,6 @@ public class TextIO {
      * @see ShardNameTemplate
      */
     public Write withSuffix(String nameExtension) {
-      validateOutputComponent(nameExtension);
       return toBuilder().setFilenameSuffix(nameExtension).build();
     }
 
@@ -505,17 +502,6 @@ public class TextIO {
     }
   }
 
-  // Pattern which matches old-style shard output patterns, which are now
-  // disallowed.
-  private static final Pattern SHARD_OUTPUT_PATTERN = Pattern.compile("@([0-9]+|\\*)");
-
-  private static void validateOutputComponent(String partialFilePattern) {
-    checkArgument(
-        !SHARD_OUTPUT_PATTERN.matcher(partialFilePattern).find(),
-        "Output name components are not allowed to contain @* or @N patterns: "
-        + partialFilePattern);
-  }
-
   //////////////////////////////////////////////////////////////////////////////
 
   /** Disable construction of utility class. */

http://git-wip-us.apache.org/repos/asf/beam/blob/00bee9b7/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
index f30b52f..66b605f 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
@@ -540,21 +540,6 @@ public class TextIOTest {
         displayData, hasItem(hasDisplayItem(hasValue(startsWith(outputPath)))));
   }
 
-  @Test
-  public void testUnsupportedFilePattern() throws IOException {
-    p.enableAbandonedNodeEnforcement(false);
-    // Windows doesn't like resolving paths with * in them.
-    String filename = tempFolder.resolve("output@5").toString();
-
-    PCollection<String> input =
-        p.apply(Create.of(Arrays.asList(LINES_ARRAY))
-            .withCoder(StringUtf8Coder.of()));
-
-    expectedException.expect(IllegalArgumentException.class);
-    expectedException.expectMessage("Output name components are not allowed to contain");
-    input.apply(TextIO.write().to(filename));
-  }
-
   /** Options for testing. */
   public interface RuntimeTestOptions extends PipelineOptions {
     ValueProvider<String> getInput();


Mime
View raw message