[BEAM-59] Beam FileSystem.setDefaultConfig: remove scheme from the signature.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4cdd8771 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4cdd8771 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4cdd8771 Branch: refs/heads/python-sdk Commit: 4cdd87718c3d0719b7c0e421b9cbaf4eb902672e Parents: 1148be6 Author: Pei He <pe...@google.com> Authored: Mon Jan 23 18:08:44 2017 -0800 Committer: Dan Halperin <dhalp...@google.com> Committed: Tue Jan 24 15:54:53 2017 -0800 ---------------------------------------------------------------------- .../org/apache/beam/sdk/io/FileSystems.java | 32 +++++++------------ .../org/apache/beam/sdk/io/FileSystemsTest.java | 33 +++----------------- 2 files changed, 15 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/4cdd8771/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java ---------------------------------------------------------------------- diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java index d086ec6..e19c1e4 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; @@ -53,6 +53,8 @@ public class FileSystems { private static final Map<String, FileSystemRegistrar> SCHEME_TO_REGISTRAR = new ConcurrentHashMap<>(); + private static PipelineOptions defaultConfig; + private static final Map<String, PipelineOptions> SCHEME_TO_DEFAULT_CONFIG = new ConcurrentHashMap<>(); @@ -78,27 +80,12 @@ public class FileSystems { } /** - * Sets the default configuration to be used with a {@link FileSystemRegistrar} for the provided - * {@code scheme}. + * Sets the default configuration in workers. * - * <p>Syntax: <pre>scheme = alpha *( alpha | digit | "+" | "-" | "." )</pre> - * Upper case letters are treated as the same as lower case letters. + * <p>It will be used in {@link FileSystemRegistrar FileSystemRegistrars} for all schemes. */ - public static void setDefaultConfig(String scheme, PipelineOptions options) { - String lowerCaseScheme = checkNotNull(scheme, "scheme").toLowerCase(); - checkArgument( - URI_SCHEME_PATTERN.matcher(lowerCaseScheme).matches(), - String.format("Scheme: [%s] doesn't match URI syntax: %s", - lowerCaseScheme, URI_SCHEME_PATTERN.pattern())); - checkArgument( - SCHEME_TO_REGISTRAR.containsKey(lowerCaseScheme), - String.format("No FileSystemRegistrar found for scheme: [%s].", lowerCaseScheme)); - SCHEME_TO_DEFAULT_CONFIG.put(lowerCaseScheme, checkNotNull(options, "options")); - } - - @VisibleForTesting - static PipelineOptions getDefaultConfig(String scheme) { - return SCHEME_TO_DEFAULT_CONFIG.get(scheme.toLowerCase()); + public static void setDefaultConfigInWorkers(PipelineOptions options) { + defaultConfig = checkNotNull(options, "options"); } /** @@ -106,9 +93,12 @@ public class FileSystems { */ @VisibleForTesting static FileSystem getFileSystemInternal(URI uri) { + checkState( + defaultConfig != null, + "Expect the runner have called setDefaultConfigInWorkers()."); String lowerCaseScheme = (uri.getScheme() != null ? uri.getScheme().toLowerCase() : LocalFileSystemRegistrar.LOCAL_FILE_SCHEME); - return getRegistrarInternal(lowerCaseScheme).fromOptions(getDefaultConfig(lowerCaseScheme)); + return getRegistrarInternal(lowerCaseScheme).fromOptions(defaultConfig); } /** http://git-wip-us.apache.org/repos/asf/beam/blob/4cdd8771/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java index 9b41b98..113a562 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java @@ -17,8 +17,6 @@ */ package org.apache.beam.sdk.io; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; import com.google.common.collect.Sets; @@ -26,6 +24,7 @@ import java.net.URI; import javax.annotation.Nullable; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -41,33 +40,9 @@ public class FileSystemsTest { @Rule public ExpectedException thrown = ExpectedException.none(); - @Test - public void testSetDefaultConfig() throws Exception { - PipelineOptions first = PipelineOptionsFactory.create(); - PipelineOptions second = PipelineOptionsFactory.create(); - FileSystems.setDefaultConfig("file", first); - assertEquals(first, FileSystems.getDefaultConfig("file")); - assertEquals(first, FileSystems.getDefaultConfig("FILE")); - - FileSystems.setDefaultConfig("FILE", second); - assertNotEquals(first, FileSystems.getDefaultConfig("file")); - assertNotEquals(first, FileSystems.getDefaultConfig("FILE")); - assertEquals(second, FileSystems.getDefaultConfig("file")); - assertEquals(second, FileSystems.getDefaultConfig("FILE")); - } - - @Test - public void testSetDefaultConfigNotFound() throws Exception { - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("No FileSystemRegistrar found for scheme: [gs-s3]."); - FileSystems.setDefaultConfig("gs-s3", PipelineOptionsFactory.create()); - } - - @Test - public void testSetDefaultConfigInvalidScheme() throws Exception { - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("Scheme: [gs:] doesn't match URI syntax"); - FileSystems.setDefaultConfig("gs:", PipelineOptionsFactory.create()); + @Before + public void setup() { + FileSystems.setDefaultConfigInWorkers(PipelineOptionsFactory.create()); } @Test
