beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhalp...@apache.org
Subject [12/50] beam git commit: [BEAM-59] Beam FileSystem.setDefaultConfig: remove scheme from the signature.
Date Sun, 29 Jan 2017 16:21:58 GMT
[BEAM-59] Beam FileSystem.setDefaultConfig: remove scheme from the signature.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4cdd8771
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4cdd8771
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4cdd8771

Branch: refs/heads/python-sdk
Commit: 4cdd87718c3d0719b7c0e421b9cbaf4eb902672e
Parents: 1148be6
Author: Pei He <peihe@google.com>
Authored: Mon Jan 23 18:08:44 2017 -0800
Committer: Dan Halperin <dhalperi@google.com>
Committed: Tue Jan 24 15:54:53 2017 -0800

----------------------------------------------------------------------
 .../org/apache/beam/sdk/io/FileSystems.java     | 32 +++++++------------
 .../org/apache/beam/sdk/io/FileSystemsTest.java | 33 +++-----------------
 2 files changed, 15 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/4cdd8771/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
index d086ec6..e19c1e4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
@@ -17,8 +17,8 @@
  */
 package org.apache.beam.sdk.io;
 
-import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
@@ -53,6 +53,8 @@ public class FileSystems {
   private static final Map<String, FileSystemRegistrar> SCHEME_TO_REGISTRAR =
       new ConcurrentHashMap<>();
 
+  private static PipelineOptions defaultConfig;
+
   private static final Map<String, PipelineOptions> SCHEME_TO_DEFAULT_CONFIG =
       new ConcurrentHashMap<>();
 
@@ -78,27 +80,12 @@ public class FileSystems {
   }
 
   /**
-   * Sets the default configuration to be used with a {@link FileSystemRegistrar} for the
provided
-   * {@code scheme}.
+   * Sets the default configuration in workers.
    *
-   * <p>Syntax: <pre>scheme = alpha *( alpha | digit | "+" | "-" | "." )</pre>
-   * Upper case letters are treated as the same as lower case letters.
+   * <p>It will be used in {@link FileSystemRegistrar FileSystemRegistrars} for all
schemes.
    */
-  public static void setDefaultConfig(String scheme, PipelineOptions options) {
-    String lowerCaseScheme = checkNotNull(scheme, "scheme").toLowerCase();
-    checkArgument(
-        URI_SCHEME_PATTERN.matcher(lowerCaseScheme).matches(),
-        String.format("Scheme: [%s] doesn't match URI syntax: %s",
-            lowerCaseScheme, URI_SCHEME_PATTERN.pattern()));
-    checkArgument(
-        SCHEME_TO_REGISTRAR.containsKey(lowerCaseScheme),
-        String.format("No FileSystemRegistrar found for scheme: [%s].", lowerCaseScheme));
-    SCHEME_TO_DEFAULT_CONFIG.put(lowerCaseScheme, checkNotNull(options, "options"));
-  }
-
-  @VisibleForTesting
-  static PipelineOptions getDefaultConfig(String scheme) {
-    return SCHEME_TO_DEFAULT_CONFIG.get(scheme.toLowerCase());
+  public static void setDefaultConfigInWorkers(PipelineOptions options) {
+    defaultConfig = checkNotNull(options, "options");
   }
 
   /**
@@ -106,9 +93,12 @@ public class FileSystems {
    */
   @VisibleForTesting
   static FileSystem getFileSystemInternal(URI uri) {
+    checkState(
+        defaultConfig != null,
+        "Expect the runner have called setDefaultConfigInWorkers().");
     String lowerCaseScheme = (uri.getScheme() != null
         ? uri.getScheme().toLowerCase() : LocalFileSystemRegistrar.LOCAL_FILE_SCHEME);
-    return getRegistrarInternal(lowerCaseScheme).fromOptions(getDefaultConfig(lowerCaseScheme));
+    return getRegistrarInternal(lowerCaseScheme).fromOptions(defaultConfig);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/beam/blob/4cdd8771/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
index 9b41b98..113a562 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
@@ -17,8 +17,6 @@
  */
 package org.apache.beam.sdk.io;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 
 import com.google.common.collect.Sets;
@@ -26,6 +24,7 @@ import java.net.URI;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -41,33 +40,9 @@ public class FileSystemsTest {
   @Rule
   public ExpectedException thrown = ExpectedException.none();
 
-  @Test
-  public void testSetDefaultConfig() throws Exception {
-    PipelineOptions first = PipelineOptionsFactory.create();
-    PipelineOptions second = PipelineOptionsFactory.create();
-    FileSystems.setDefaultConfig("file", first);
-    assertEquals(first, FileSystems.getDefaultConfig("file"));
-    assertEquals(first, FileSystems.getDefaultConfig("FILE"));
-
-    FileSystems.setDefaultConfig("FILE", second);
-    assertNotEquals(first, FileSystems.getDefaultConfig("file"));
-    assertNotEquals(first, FileSystems.getDefaultConfig("FILE"));
-    assertEquals(second, FileSystems.getDefaultConfig("file"));
-    assertEquals(second, FileSystems.getDefaultConfig("FILE"));
-  }
-
-  @Test
-  public void testSetDefaultConfigNotFound() throws Exception {
-    thrown.expect(IllegalArgumentException.class);
-    thrown.expectMessage("No FileSystemRegistrar found for scheme: [gs-s3].");
-    FileSystems.setDefaultConfig("gs-s3", PipelineOptionsFactory.create());
-  }
-
-  @Test
-  public void testSetDefaultConfigInvalidScheme() throws Exception {
-    thrown.expect(IllegalArgumentException.class);
-    thrown.expectMessage("Scheme: [gs:] doesn't match URI syntax");
-    FileSystems.setDefaultConfig("gs:", PipelineOptionsFactory.create());
+  @Before
+  public void setup() {
+    FileSystems.setDefaultConfigInWorkers(PipelineOptionsFactory.create());
   }
 
   @Test


Mime
View raw message