KYLIN-2648 make kylin.env.hdfs-working-dir qualified and absolute
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e3a79c82 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e3a79c82 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e3a79c82 Branch: refs/heads/master Commit: e3a79c8276123bb9fa3a9de9bfc653a81607d392 Parents: 6bfc6d2 Author: Li Yang <liy...@apache.org> Authored: Sat May 27 17:03:09 2017 +0800 Committer: hongbin ma <m...@kyligence.io> Committed: Sat May 27 17:08:06 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 40 +++++++++++++------- .../apache/kylin/common/KylinConfigTest.java | 7 ++++ .../java/org/apache/kylin/rest/DebugTomcat.java | 10 ++--- 3 files changed, 39 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e3a79c82/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index ad08108..854ffbd 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -29,6 +29,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.kylin.common.lock.DistributedLockFactory; import org.apache.kylin.common.util.ClassUtil; import org.apache.kylin.common.util.CliCommandExecutor; @@ -184,21 +187,33 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.env", "DEV"); } + private String cachedHdfsWorkingDirectory; + public String getHdfsWorkingDirectory() { + if (cachedHdfsWorkingDirectory != null) + return cachedHdfsWorkingDirectory; + String root = getRequired("kylin.env.hdfs-working-dir"); - if (!root.endsWith("/")) { - root += "/"; - } - - // make sure path qualified - if (!root.contains("://")) { - if (!root.startsWith("/")) - root = "hdfs:///" + root; - else - root = "hdfs://" + root; + Path path = new Path(root); + if (path.isAbsolute() == false) + throw new IllegalArgumentException("kylin.env.hdfs-working-dir must be absolute, but got " + root); + + // make sure path is qualified + try { + FileSystem fs = path.getFileSystem(new Configuration()); + path = fs.makeQualified(path); + } catch (IOException e) { + throw new RuntimeException(e); } - - return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString(); + + // append metadata-url prefix + root = new Path(path, StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).toString(); + + if (root.endsWith("/") == false) + root += "/"; + + cachedHdfsWorkingDirectory = root; + return cachedHdfsWorkingDirectory; } // ============================================================================ @@ -1028,5 +1043,4 @@ abstract public class KylinConfigBase implements Serializable { public boolean isWebCrossDomainEnabled() { return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true")); } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/e3a79c82/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java index 6027af3..fcbdd36 100644 --- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java @@ -124,4 +124,11 @@ public class KylinConfigTest extends HotLoadKylinPropertiesTestCase { } }).start(); } + + @Test + public void testHdfsWorkingDir() { + KylinConfig conf = KylinConfig.getInstanceFromEnv(); + String hdfsWorkingDirectory = conf.getHdfsWorkingDirectory(); + assertTrue(hdfsWorkingDirectory.startsWith("file:/")); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/e3a79c82/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java index cd6768e..d87f1f9 100644 --- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java +++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java @@ -18,6 +18,10 @@ package org.apache.kylin.rest; +import java.io.File; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + import org.apache.catalina.Context; import org.apache.catalina.core.AprLifecycleListener; import org.apache.catalina.core.StandardServer; @@ -27,10 +31,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.util.Shell; import org.apache.kylin.common.KylinConfig; -import java.io.File; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; - public class DebugTomcat { public static void setupDebugEnv() { @@ -101,7 +101,7 @@ public class DebugTomcat { public static void main(String[] args) throws Exception { setupDebugEnv(); - + int port = 7070; if (args.length >= 1) { port = Integer.parseInt(args[0]);