This is an automated email from the ASF dual-hosted git repository. tanjian pushed a commit to branch support_insecure_hosts in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 00c77e235c8350d529a5a1cb760c031e659b8038 Author: JaredTan95 <jian....@daocloud.io> AuthorDate: Tue Nov 30 19:04:19 2021 +0800 Support disables the verification of server's TLS certificate chain for specific hosts --- CHANGES.md | 2 +- docs/en/setup/backend/configuration-vocabulary.md | 1 + .../library/client/elasticsearch/ElasticSearchClient.java | 11 +++++++++-- .../library/elasticsearch/bulk/ITElasticSearch.java | 2 +- .../library/elasticsearch/ElasticSearchBuilder.java | 14 ++++++++++++-- .../server-starter/src/main/resources/application.yml | 3 ++- .../elasticsearch/StorageModuleElasticsearchConfig.java | 1 + .../elasticsearch/StorageModuleElasticsearchProvider.java | 4 ++-- 8 files changed, 29 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b8aa70d..867b152 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -63,7 +63,7 @@ Release Notes. * Add customized envoy ALS protocol receiver for satellite transmit batch data. * Remove `logback` dependencies in IoTDB plugin. * Fix `StorageModuleElasticsearchProvider` doesn't watch on `trustStorePath`. - +* Support disables the verification of server's TLS certificate chain for specific hosts by `SW_STORAGE_ES_SSL_INSECURE_HOSTS` env. #### UI * Optimize endpoint dependency. diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 891517f..bbc5c92 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -90,6 +90,7 @@ core|default|role|Option values: `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | password | Password of ElasticSearch cluster. | SW_ES_PASSWORD | - | | - | - | trustStorePath | Trust JKS file path. Only works when username and password are enabled. | SW_STORAGE_ES_SSL_JKS_PATH | - | | - | - | trustStorePass | Trust JKS file password. Only works when username and password are enabled. | SW_STORAGE_ES_SSL_JKS_PASS | - | +| - | - | insecureHosts | Disables the verification of server's TLS certificate chain for specific hosts. **NOTE**: You should never use this in production but only for a testing purpose. | SW_STORAGE_ES_SSL_INSECURE_HOSTS | - | | - | - | secretsManagementFile| Secrets management file in the properties format, including username and password, which are managed by a 3rd party tool. Capable of being updated them at runtime. |SW_ES_SECRETS_MANAGEMENT_FILE | - | | - | - | dayStep| Represents the number of days in the one-minute/hour/day index. | SW_STORAGE_DAY_STEP | 1| | - | - | indexShardsNumber | Shard number of new indexes. | SW_STORAGE_ES_INDEX_SHARDS_NUMBER | 1 | diff --git a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java index 2a457f4..cd023a8 100644 --- a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java +++ b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java @@ -33,7 +33,6 @@ import java.util.function.Supplier; import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.library.elasticsearch.ElasticSearch; import org.apache.skywalking.library.elasticsearch.ElasticSearchBuilder; import org.apache.skywalking.library.elasticsearch.ElasticSearchVersion; @@ -49,6 +48,7 @@ import org.apache.skywalking.oap.server.library.client.Client; import org.apache.skywalking.oap.server.library.client.healthcheck.DelegatedHealthChecker; import org.apache.skywalking.oap.server.library.client.healthcheck.HealthCheckable; import org.apache.skywalking.oap.server.library.util.HealthChecker; +import org.apache.skywalking.oap.server.library.util.StringUtil; /** * ElasticSearchClient connects to the ES server by using ES client APIs. @@ -67,6 +67,8 @@ public class ElasticSearchClient implements Client, HealthCheckable { @Setter private volatile String trustStorePass; + private final String insecureHosts; + @Setter private volatile String user; @@ -94,7 +96,8 @@ public class ElasticSearchClient implements Client, HealthCheckable { Function<String, String> indexNameConverter, int connectTimeout, int socketTimeout, - int numHttpClientThread) { + int numHttpClientThread, + String insecureHosts) { this.clusterNodes = clusterNodes; this.protocol = protocol; this.trustStorePath = trustStorePath; @@ -105,6 +108,7 @@ public class ElasticSearchClient implements Client, HealthCheckable { this.connectTimeout = connectTimeout; this.socketTimeout = socketTimeout; this.numHttpClientThread = numHttpClientThread; + this.insecureHosts = insecureHosts; } @Override @@ -139,6 +143,9 @@ public class ElasticSearchClient implements Client, HealthCheckable { if (!Strings.isNullOrEmpty(password)) { cb.password(password); } + if (!Strings.isNullOrEmpty(insecureHosts)) { + cb.insecureHosts(insecureHosts); + } final ElasticSearch newOne = cb.build(); // Only swap the old / new after the new one established a new connection. diff --git a/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/ITElasticSearch.java b/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/ITElasticSearch.java index b7b103d..899703c 100644 --- a/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/ITElasticSearch.java +++ b/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/ITElasticSearch.java @@ -80,7 +80,7 @@ public class ITElasticSearch { server.getHttpHostAddress(), "http", "", "", "test", "test", indexNameConverter(namespace), 500, 6000, - 0 + 0, "" ); client.connect(); } diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchBuilder.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchBuilder.java index 13df927..f5a22bd 100644 --- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchBuilder.java +++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/ElasticSearchBuilder.java @@ -37,12 +37,11 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import javax.net.ssl.TrustManagerFactory; import lombok.SneakyThrows; +import org.apache.skywalking.oap.server.library.util.StringUtil; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; -import org.apache.skywalking.oap.server.library.util.StringUtil; - public final class ElasticSearchBuilder { private static final int NUM_PROC = Runtime.getRuntime().availableProcessors(); @@ -60,6 +59,8 @@ public final class ElasticSearchBuilder { private String trustStorePass; + private String insecureHosts; + private Duration connectTimeout = Duration.ofMillis(500); private Duration socketTimeout = Duration.ofSeconds(30); @@ -94,6 +95,11 @@ public final class ElasticSearchBuilder { return endpoints(Arrays.asList(endpoints)); } + public ElasticSearchBuilder insecureHosts(String insecureHosts) { + this.insecureHosts = insecureHosts; + return this; + } + public ElasticSearchBuilder healthCheckRetryInterval(Duration healthCheckRetryInterval) { requireNonNull(healthCheckRetryInterval, "healthCheckRetryInterval"); this.healthCheckRetryInterval = healthCheckRetryInterval; @@ -149,6 +155,10 @@ public final class ElasticSearchBuilder { .useHttp2Preface(false) .workerGroup(numHttpClientThread > 0 ? numHttpClientThread : NUM_PROC); + if (StringUtil.isNotBlank(insecureHosts)) { + factoryBuilder.tlsNoVerifyHosts(insecureHosts.split(",")); + } + if (StringUtil.isNotBlank(trustStorePath)) { final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 631be41..ffcf8b9 100755 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -118,7 +118,7 @@ core: # Turn it on then automatically grouping endpoint by the given OpenAPI definitions. enableEndpointNameGroupingByOpenapi: ${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:true} storage: - selector: ${SW_STORAGE:h2} + selector: ${SW_STORAGE:elasticsearch} elasticsearch: namespace: ${SW_NAMESPACE:""} clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200} @@ -130,6 +130,7 @@ storage: password: ${SW_ES_PASSWORD:""} trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""} trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""} + insecureHosts: ${SW_STORAGE_ES_SSL_INSECURE_HOSTS:""} # e.g. "172.16.1.1,172.16.1.2". You should never use this in production but only for a testing purpose. secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool. dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index. indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1} # Shard number of new indexes diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java index aecf642..aafe439 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java @@ -101,6 +101,7 @@ public class StorageModuleElasticsearchConfig extends ModuleConfig { * @since 7.0.0 This could be managed inside {@link #secretsManagementFile} */ private String trustStorePass; + private String insecureHosts; private int resultWindowMaxSize = 10000; private int metadataQueryMaxSize = 5000; private int segmentQueryMaxSize = 200; diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java index 04fe917..3dbb7af 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java @@ -22,7 +22,6 @@ import java.io.ByteArrayInputStream; import java.util.Properties; import java.util.function.Function; import lombok.extern.slf4j.Slf4j; -import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.storage.IBatchDAO; import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; @@ -52,6 +51,7 @@ import org.apache.skywalking.oap.server.library.module.ModuleProvider; import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; import org.apache.skywalking.oap.server.library.util.MultipleFilesChangeMonitor; +import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.HistoryDeleteEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.StorageEsDAO; @@ -156,7 +156,7 @@ public class StorageModuleElasticsearchProvider extends ModuleProvider { config.getClusterNodes(), config.getProtocol(), config.getTrustStorePath(), config .getTrustStorePass(), config.getUser(), config.getPassword(), indexNameConverter(config.getNamespace()), config.getConnectTimeout(), - config.getSocketTimeout(), config.getNumHttpClientThread() + config.getSocketTimeout(), config.getNumHttpClientThread(), config.getInsecureHosts() ); this.registerServiceImplementation( IBatchDAO.class,