[
https://issues.apache.org/jira/browse/FLINK-2380?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15108553#comment-15108553
]
ASF GitHub Bot commented on FLINK-2380:
---------------------------------------
Github user uce commented on a diff in the pull request:
https://github.com/apache/flink/pull/1524#discussion_r50253604
--- Diff: flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java
---
@@ -159,56 +164,91 @@ public int hashCode() {
/**
* Returns a reference to the {@link FileSystem} instance for accessing
the
* local file system.
- *
+ *
* @return a reference to the {@link FileSystem} instance for accessing
the
- * local file system.
+ * local file system.
*/
public static FileSystem getLocalFileSystem() {
// this should really never fail.
try {
URI localUri = OperatingSystem.isWindows() ? new
URI("file:/") : new URI("file:///");
return get(localUri);
- }
- catch (Exception e) {
+ } catch (Exception e) {
throw new RuntimeException("Cannot create URI for local
file system");
}
}
/**
+ * The default filesystem scheme to be used. This can be specified by
the parameter
+ * <code>fs.default-scheme</code> in <code>flink-conf.yaml</code>. By
default this is
+ * set to <code>file:///</code> and uses the local filesystem.
+ * */
+ private static URI defaultScheme;
+
+ /**
+ * Sets the default filesystem scheme based on the user-specified
configuration parameter
+ * <code>fs.default-scheme</code>.
+ * <li>
+ * As an example, if set to <code>hdfs://localhost:9000/</code>, then
an HDFS deployment
+ * with the namenode being on the local node and listening to port 9000
is going to be used.
+ * In this case, a file path specified as
<code>/user/USERNAME/in.txt</code>
+ * is going to be transformed into
<code>hdfs://localhost:9000/user/USERNAME/in.txt</code>. By
+ * default this is set to <code>file:///</code> which points to the
local filesystem.
+ * @param config the configuration from where to fetch the parameter.
+ * */
+ public static void setDefaultScheme(Configuration config) throws
IOException {
+ if (defaultScheme == null) {
+ String stringifiedUri =
config.getString(ConfigConstants.FILESYSTEM_SCHEME,
+ ConfigConstants.DEFAULT_FILESYSTEM_SCHEME);
+ try {
+ defaultScheme = new URI(stringifiedUri);
+ } catch (URISyntaxException e) {
+ throw new IOException("The URI used to set the
default filesystem " +
+ "scheme ('" + stringifiedUri + "') is
not valid.");
+ }
+ }
+ }
+
+ /**
* Returns a reference to the {@link FileSystem} instance for accessing
the
* file system identified by the given {@link URI}.
- *
- * @param uri
- * the {@link URI} identifying the file system
+ *
+ * @param uri the {@link URI} identifying the file system
* @return a reference to the {@link FileSystem} instance for accessing
the file system identified by the given
- * {@link URI}.
- * @throws IOException
- * thrown if a reference to the file system instance could not
be obtained
+ * {@link URI}.
+ * @throws IOException thrown if a reference to the file system
instance could not be obtained
*/
public static FileSystem get(URI uri) throws IOException {
FileSystem fs;
synchronized (SYNCHRONIZATION_OBJECT) {
-
if (uri.getScheme() == null) {
try {
- uri = new URI("file", null,
uri.getPath(), null);
- }
- catch (URISyntaxException e) {
+ if(defaultScheme == null) {
--- End diff --
formatting: we use `if (...)` (with whitespace after the keyword)
> Allow to configure default FS for file inputs
> ---------------------------------------------
>
> Key: FLINK-2380
> URL: https://issues.apache.org/jira/browse/FLINK-2380
> Project: Flink
> Issue Type: Improvement
> Components: JobManager
> Affects Versions: 0.9, 0.10.0
> Reporter: Ufuk Celebi
> Assignee: Kostas
> Priority: Minor
> Labels: starter
> Fix For: 1.0.0
>
>
> File inputs use "file://" as default prefix. A user asked to make this
> configurable, e.g. "hdfs://" as default.
> (I'm not sure whether this is already possible or not. I will check and
> either close or implement this for the user.)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
