This is an automated email from the ASF dual-hosted git repository. sunzesong pushed a commit to branch hdfs_0.9 in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 9285f6eab0ccf100fe735c3cac84f258d71433e3 Author: samperson1997 <[email protected]> AuthorDate: Thu Apr 9 12:12:01 2020 +0800 Fix some HDFS config issues in rel/0.9 --- .../iotdb/hadoop/fileSystem/HDFSConfUtil.java | 6 ++++- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 8 ++++++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 29 ++++++++++++++-------- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java b/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java index df0189b..49765d4 100644 --- a/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java +++ b/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java @@ -19,7 +19,6 @@ package org.apache.iotdb.hadoop.fileSystem; - import java.io.File; import java.io.IOException; import java.net.MalformedURLException; @@ -27,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.fileSystem.FSType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +36,10 @@ class HDFSConfUtil { private static final Logger logger = LoggerFactory.getLogger(HDFSConfUtil.class); static Configuration setConf(Configuration conf) { + if (!tsFileConfig.getTSFileStorageFs().equals(FSType.HDFS)) { + return conf; + } + try { conf.addResource(new File(tsFileConfig.getCoreSitePath()).toURI().toURL()); conf.addResource(new File(tsFileConfig.getHdfsSitePath()).toURI().toURL()); diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index d13387e..4154566 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1187,6 +1187,10 @@ public class IoTDBConfig { return hdfsIp.split(","); } + String getRawHDFSIp() { + return hdfsIp; + } + public void setHdfsIp(String[] hdfsIp) { this.hdfsIp = String.join(",", hdfsIp); } @@ -1219,6 +1223,10 @@ public class IoTDBConfig { return dfsHaNamenodes.split(","); } + String getRawDfsHaNamenodes() { + return dfsHaNamenodes; + } + public void setDfsHaNamenodes(String[] dfsHaNamenodes) { this.dfsHaNamenodes = String.join(",", dfsHaNamenodes); } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 13cc3ec..90a38f4 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -313,11 +313,12 @@ public class IoTDBDescriptor { properties.getProperty("core_site_path", conf.getCoreSitePath())); conf.setHdfsSitePath( properties.getProperty("hdfs_site_path", conf.getHdfsSitePath())); - conf.setHdfsIp(properties.getProperty("hdfs_ip").split(",")); + conf.setHdfsIp(properties.getProperty("hdfs_ip", conf.getRawHDFSIp()).split(",")); conf.setHdfsPort(properties.getProperty("hdfs_port", conf.getHdfsPort())); conf.setDfsNameServices( properties.getProperty("dfs_nameservices", conf.getDfsNameServices())); - conf.setDfsHaNamenodes(properties.getProperty("dfs_ha_namenodes").split(",")); + conf.setDfsHaNamenodes( + properties.getProperty("dfs_ha_namenodes", conf.getRawDfsHaNamenodes()).split(",")); conf.setDfsHaAutomaticFailoverEnabled( Boolean.parseBoolean(properties.getProperty("dfs_ha_automatic_failover_enabled", String.valueOf(conf.isDfsHaAutomaticFailoverEnabled())))); @@ -336,22 +337,28 @@ public class IoTDBDescriptor { // At the same time, set TSFileConfig TSFileDescriptor.getInstance().getConfig() - .setTSFileStorageFs(properties.getProperty("tsfile_storage_fs")); - TSFileDescriptor.getInstance().getConfig().setKerberosKeytabFilePath( + .setTSFileStorageFs( + properties.getProperty("tsfile_storage_fs", conf.getTsFileStorageFs().name())); + TSFileDescriptor.getInstance().getConfig().setCoreSitePath( properties.getProperty("core_site_path", conf.getCoreSitePath())); - TSFileDescriptor.getInstance().getConfig().setKerberosPrincipal( + TSFileDescriptor.getInstance().getConfig().setHdfsSitePath( properties.getProperty("hdfs_site_path", conf.getHdfsSitePath())); TSFileDescriptor.getInstance().getConfig() - .setHdfsIp(properties.getProperty("hdfs_ip").split(",")); - TSFileDescriptor.getInstance().getConfig().setHdfsPort(properties.getProperty("hdfs_port")); + .setHdfsIp(properties.getProperty("hdfs_ip", conf.getRawHDFSIp()).split(",")); TSFileDescriptor.getInstance().getConfig() - .setDfsNameServices(properties.getProperty("dfs_nameservices")); + .setHdfsPort(properties.getProperty("hdfs_port", conf.getHdfsPort())); TSFileDescriptor.getInstance().getConfig() - .setDfsHaNamenodes(properties.getProperty("dfs_ha_namenodes").split(",")); + .setDfsNameServices( + properties.getProperty("dfs_nameservices", conf.getDfsNameServices())); + TSFileDescriptor.getInstance().getConfig() + .setDfsHaNamenodes( + properties.getProperty("dfs_ha_namenodes", conf.getRawDfsHaNamenodes()).split(",")); TSFileDescriptor.getInstance().getConfig().setDfsHaAutomaticFailoverEnabled( - Boolean.parseBoolean(properties.getProperty("dfs_ha_automatic_failover_enabled"))); + Boolean.parseBoolean(properties.getProperty("dfs_ha_automatic_failover_enabled", + String.valueOf(conf.isDfsHaAutomaticFailoverEnabled())))); TSFileDescriptor.getInstance().getConfig().setDfsClientFailoverProxyProvider( - properties.getProperty("dfs_client_failover_proxy_provider")); + properties.getProperty("dfs_client_failover_proxy_provider", + conf.getDfsClientFailoverProxyProvider())); TSFileDescriptor.getInstance().getConfig().setUseKerberos(Boolean.parseBoolean( properties.getProperty("hdfs_use_kerberos", String.valueOf(conf.isUseKerberos())))); TSFileDescriptor.getInstance().getConfig().setKerberosKeytabFilePath(
