This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 5c2418e Support nacos grouped dynamic configurations. (#7632)
5c2418e is described below
commit 5c2418ee9d9c0c33b76b29717c36168f9040cb0f
Author: wankai123 <[email protected]>
AuthorDate: Wed Sep 1 21:22:06 2021 +0800
Support nacos grouped dynamic configurations. (#7632)
---
CHANGES.md | 1 +
docs/en/setup/backend/dynamic-config-nacos.md | 63 +++++++++++++++++++++-
.../nacos/NacosConfigWatcherRegister.java | 41 ++++++++++----
.../nacos/NacosConfigurationProvider.java | 8 ++-
.../nacos/ITNacosConfigurationTest.java | 52 ++++++++++++++++--
.../nacos/NacosConfigurationTestProvider.java | 38 +++++++++++--
6 files changed, 178 insertions(+), 25 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index aaeefb1..07f248c 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -49,6 +49,7 @@ Release Notes.
* Fix dynamic configuration watch implementation current value not null when
the config is deleted.
* Fix `LoggingConfigWatcher` return `watch.value` would not consistent with
the real configuration content.
* Fix `ZookeeperConfigWatcherRegister.readConfig()` could cause `NPE` when
`data.getData()` is null.
+* Support nacos grouped dynamic configurations.
#### UI
diff --git a/docs/en/setup/backend/dynamic-config-nacos.md
b/docs/en/setup/backend/dynamic-config-nacos.md
index a7a8a52..73562b1 100755
--- a/docs/en/setup/backend/dynamic-config-nacos.md
+++ b/docs/en/setup/backend/dynamic-config-nacos.md
@@ -18,4 +18,65 @@ configuration:
period: ${SW_CONFIG_NACOS_PERIOD:60}
# the name of current cluster, set the name if you want to upstream system
known.
clusterName: ${SW_CONFIG_NACOS_CLUSTER_NAME:default}
-```
\ No newline at end of file
+```
+
+## Config Storage
+### Single Config
+
+| Data Id | Group | Config Value |
+|-----|-----|-----|
+| configKey | {group} | configValue |
+
+e.g. The config is:
+```
+{agent-analyzer.default.slowDBAccessThreshold}:{default:200,mongodb:50}
+```
+If `group = skywalking` the config in nacos is:
+
+| Data Id | Group | Config Value |
+|-----|-----|-----|
+| agent-analyzer.default.slowDBAccessThreshold | skywalking |
default:200,mongodb:50 |
+
+### Group Config
+
+| Data Id | Group | Config Value | Config Type |
+|-----|-----|-----|-----|
+| configKey | {group} | subItemkey1</br>subItemkey2</br>... | TEXT |
+| subItemkey1 | {group} | subItemValue1 |
+| subItemkey2 | {group} | subItemValue2 |
+| ... | ... | ... |
+
+Notice: If you add/remove a subItem, you need to add/remove the subItemKey
from the group which the subItem belongs:
+
+| Data Id | Group | Config Value | Config Type |
+|-----|-----|-----|-----|
+| configKey | {group} | subItemkey1</br>subItemkey2</br>... | TEXT |
+
+We separate subItemkeys by `\n` or `\r\n`, trim leading and trailing
whitespace, if you set the config by `Nacos UI` each subItemkey should in a new
line:
+```
+subItemValue1
+subItemValue2
+...
+
+```
+If you set the config by `API` each subItemkey should separated by `\n` or
`\r\n`:
+```
+configService.publishConfig("test-module.default.testKeyGroup", "skywalking",
"subItemkey1\n subItemkey2"));
+```
+
+e.g. The config is:
+```
+{core.default.endpoint-name-grouping-openapi}:|{customerAPI-v1}:{value of
customerAPI-v1}
+ |{productAPI-v1}:{value of
productAPI-v1}
+ |{productAPI-v2}:{value of
productAPI-v2}
+```
+If `group = skywalking` the config in nacos is:
+
+| Data Id | Group | Config Value | Config Type |
+|-----|-----|-----|-----|
+| core.default.endpoint-name-grouping-openapi | skywalking |
customerAPI-v1</br>productAPI-v1</br>productAPI-v2 | TEXT |
+| customerAPI-v1 | skywalking | value of customerAPI-v1 |
+| productAPI-v1 | skywalking | value of productAPI-v1 |
+| productAPI-v2 | skywalking | value of productAPI-v2 |
+
+
diff --git
a/oap-server/server-configuration/configuration-nacos/src/main/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigWatcherRegister.java
b/oap-server/server-configuration/configuration-nacos/src/main/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigWatcherRegister.java
index 47c741a..3a62040 100644
---
a/oap-server/server-configuration/configuration-nacos/src/main/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigWatcherRegister.java
+++
b/oap-server/server-configuration/configuration-nacos/src/main/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigWatcherRegister.java
@@ -23,6 +23,7 @@ import com.alibaba.nacos.api.PropertyKeyConst;
import com.alibaba.nacos.api.config.ConfigService;
import com.alibaba.nacos.api.config.listener.Listener;
import com.alibaba.nacos.api.exception.NacosException;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
@@ -31,16 +32,14 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
+import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.apm.util.StringUtil;
import org.apache.skywalking.oap.server.configuration.api.ConfigTable;
import
org.apache.skywalking.oap.server.configuration.api.ConfigWatcherRegister;
import org.apache.skywalking.oap.server.configuration.api.GroupConfigTable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+@Slf4j
public class NacosConfigWatcherRegister extends ConfigWatcherRegister {
- private static final Logger LOGGER =
LoggerFactory.getLogger(NacosConfigWatcherRegister.class);
-
private final NacosServerSettings settings;
private final ConfigService configService;
private final Map<String, Optional<String>> configItemKeyedByName;
@@ -92,8 +91,32 @@ public class NacosConfigWatcherRegister extends
ConfigWatcherRegister {
@Override
public Optional<GroupConfigTable> readGroupConfig(final Set<String> keys) {
- // TODO: implement readGroupConfig
- return Optional.empty();
+ GroupConfigTable groupConfigTable = new GroupConfigTable();
+ keys.forEach(key -> {
+ GroupConfigTable.GroupConfigItems groupConfigItems = new
GroupConfigTable.GroupConfigItems(key);
+ groupConfigTable.addGroupConfigItems(groupConfigItems);
+ String config = null;
+ try {
+ config = configService.getConfig(key, settings.getGroup(),
1000);
+ if (StringUtil.isNotEmpty(config)) {
+ String[] itemNames = config.split("\\n|\\r\\n");
+
Arrays.stream(itemNames).map(String::trim).forEach(itemName -> {
+ String itemValue = null;
+ try {
+ itemValue = configService.getConfig(itemName,
settings.getGroup(), 1000);
+ } catch (NacosException e) {
+ log.error("Failed to register Nacos listener for
dataId: {}", itemName, e);
+ }
+ groupConfigItems.add(
+ new ConfigTable.ConfigItem(itemName, itemValue));
+ });
+ }
+ } catch (NacosException e) {
+ log.error("Failed to register Nacos listener for dataId: {}",
key, e);
+ }
+ });
+
+ return Optional.of(groupConfigTable);
}
private void registerKeyListeners(final Set<String> keys) {
@@ -121,7 +144,7 @@ public class NacosConfigWatcherRegister extends
ConfigWatcherRegister {
final String config = configService.getConfig(dataId, group,
1000);
onDataIdValueChanged(dataId, config);
} catch (NacosException e) {
- LOGGER.warn("Failed to register Nacos listener for dataId:
{}", dataId);
+ log.warn("Failed to register Nacos listener for dataId: {}",
dataId);
}
}
}
@@ -141,8 +164,8 @@ public class NacosConfigWatcherRegister extends
ConfigWatcherRegister {
}
void onDataIdValueChanged(String dataId, String configInfo) {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Nacos config changed: {}: {}", dataId, configInfo);
+ if (log.isInfoEnabled()) {
+ log.info("Nacos config changed: {}: {}", dataId, configInfo);
}
configItemKeyedByName.put(dataId, Optional.ofNullable(configInfo));
diff --git
a/oap-server/server-configuration/configuration-nacos/src/main/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigurationProvider.java
b/oap-server/server-configuration/configuration-nacos/src/main/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigurationProvider.java
index ad12683..1ab594b 100644
---
a/oap-server/server-configuration/configuration-nacos/src/main/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigurationProvider.java
+++
b/oap-server/server-configuration/configuration-nacos/src/main/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigurationProvider.java
@@ -20,20 +20,18 @@ package
org.apache.skywalking.oap.server.configuration.nacos;
import com.alibaba.nacos.api.exception.NacosException;
import com.google.common.base.Strings;
+import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.apm.util.StringUtil;
import
org.apache.skywalking.oap.server.configuration.api.AbstractConfigurationProvider;
import
org.apache.skywalking.oap.server.configuration.api.ConfigWatcherRegister;
import org.apache.skywalking.oap.server.library.module.ModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Get configuration from Nacos.
*/
+@Slf4j
public class NacosConfigurationProvider extends AbstractConfigurationProvider {
- private static final Logger LOGGER =
LoggerFactory.getLogger(NacosConfigurationProvider.class);
-
private NacosServerSettings settings;
public NacosConfigurationProvider() {
@@ -52,7 +50,7 @@ public class NacosConfigurationProvider extends
AbstractConfigurationProvider {
@Override
protected ConfigWatcherRegister initConfigReader() throws
ModuleStartException {
- LOGGER.info("settings: {}", settings);
+ log.info("settings: {}", settings);
if (Strings.isNullOrEmpty(settings.getServerAddr())) {
throw new ModuleStartException("Nacos serverAddr cannot be null or
empty.");
}
diff --git
a/oap-server/server-configuration/configuration-nacos/src/test/java/org/apache/skywalking/oap/server/configuration/nacos/ITNacosConfigurationTest.java
b/oap-server/server-configuration/configuration-nacos/src/test/java/org/apache/skywalking/oap/server/configuration/nacos/ITNacosConfigurationTest.java
index d71d0ee..595cbe6 100644
---
a/oap-server/server-configuration/configuration-nacos/src/test/java/org/apache/skywalking/oap/server/configuration/nacos/ITNacosConfigurationTest.java
+++
b/oap-server/server-configuration/configuration-nacos/src/test/java/org/apache/skywalking/oap/server/configuration/nacos/ITNacosConfigurationTest.java
@@ -25,6 +25,7 @@ import java.io.FileNotFoundException;
import java.io.Reader;
import java.util.Map;
import java.util.Properties;
+import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.apm.util.PropertyPlaceholderHelper;
import
org.apache.skywalking.oap.server.library.module.ApplicationConfiguration;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
@@ -32,8 +33,6 @@ import
org.apache.skywalking.oap.server.library.util.CollectionUtils;
import org.apache.skywalking.oap.server.library.util.ResourceUtils;
import org.junit.Before;
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import static org.junit.Assert.assertEquals;
@@ -41,9 +40,8 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+@Slf4j
public class ITNacosConfigurationTest {
- private static final Logger LOGGER =
LoggerFactory.getLogger(ITNacosConfigurationTest.class);
-
private final Yaml yaml = new Yaml();
private NacosConfigurationTestProvider provider;
@@ -69,7 +67,7 @@ public class ITNacosConfigurationTest {
final Properties properties = new Properties();
final String nacosHost = System.getProperty("nacos.host");
final String nacosPort = System.getProperty("nacos.port");
- LOGGER.info("nacosHost: {}, nacosPort: {}", nacosHost, nacosPort);
+ log.info("nacosHost: {}, nacosPort: {}", nacosHost, nacosPort);
properties.put("serverAddr", nacosHost + ":" + nacosPort);
final ConfigService configService =
NacosFactory.createConfigService(properties);
@@ -88,6 +86,50 @@ public class ITNacosConfigurationTest {
assertNull(provider.watcher.value());
}
+ @Test(timeout = 20000)
+ public void shouldReadUpdatedGroup() throws NacosException {
+ assertNull(provider.watcher.value());
+
+ final Properties properties = new Properties();
+ final String nacosHost = System.getProperty("nacos.host");
+ final String nacosPort = System.getProperty("nacos.port");
+ log.info("nacosHost: {}, nacosPort: {}", nacosHost, nacosPort);
+ properties.put("serverAddr", nacosHost + ":" + nacosPort);
+
+ final ConfigService configService =
NacosFactory.createConfigService(properties);
+ //test add group key and item1 item2
+
assertTrue(configService.publishConfig("test-module.default.testKeyGroup",
"skywalking", "item1\n item2"));
+ assertTrue(configService.publishConfig("item1", "skywalking", "100"));
+ assertTrue(configService.publishConfig("item2", "skywalking", "200"));
+ for (String v = provider.groupWatcher.groupItems().get("item1"); v ==
null; v = provider.groupWatcher.groupItems().get("item1")) {
+ }
+ for (String v = provider.groupWatcher.groupItems().get("item2"); v ==
null; v = provider.groupWatcher.groupItems().get("item2")) {
+ }
+ assertEquals("100", provider.groupWatcher.groupItems().get("item1"));
+ assertEquals("200", provider.groupWatcher.groupItems().get("item2"));
+
+ //test remove item1
+ assertTrue(configService.removeConfig("item1", "skywalking"));
+ for (String v = provider.groupWatcher.groupItems().get("item1"); v !=
null; v = provider.groupWatcher.groupItems().get("item1")) {
+ }
+ assertNull(provider.groupWatcher.groupItems().get("item1"));
+
+ //test modify item1
+ assertTrue(configService.publishConfig("item1", "skywalking", "300"));
+ for (String v = provider.groupWatcher.groupItems().get("item1"); v ==
null; v = provider.groupWatcher.groupItems().get("item1")) {
+ }
+ assertEquals("300", provider.groupWatcher.groupItems().get("item1"));
+
+ //test remove group key
+
assertTrue(configService.removeConfig("test-module.default.testKeyGroup",
"skywalking"));
+ for (String v = provider.groupWatcher.groupItems().get("item2"); v !=
null; v = provider.groupWatcher.groupItems().get("item2")) {
+ }
+ assertNull(provider.groupWatcher.groupItems().get("item2"));
+ //chean
+ assertTrue(configService.removeConfig("item1", "skywalking"));
+ assertTrue(configService.removeConfig("item2", "skywalking"));
+ }
+
@SuppressWarnings("unchecked")
private void loadConfig(ApplicationConfiguration configuration) throws
FileNotFoundException {
Reader applicationReader = ResourceUtils.read("application.yml");
diff --git
a/oap-server/server-configuration/configuration-nacos/src/test/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigurationTestProvider.java
b/oap-server/server-configuration/configuration-nacos/src/test/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigurationTestProvider.java
index ff2d097..fee0826 100644
---
a/oap-server/server-configuration/configuration-nacos/src/test/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigurationTestProvider.java
+++
b/oap-server/server-configuration/configuration-nacos/src/test/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigurationTestProvider.java
@@ -18,21 +18,23 @@
package org.apache.skywalking.oap.server.configuration.nacos;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher;
import org.apache.skywalking.oap.server.configuration.api.ConfigurationModule;
import
org.apache.skywalking.oap.server.configuration.api.DynamicConfigurationService;
+import
org.apache.skywalking.oap.server.configuration.api.GroupConfigChangeWatcher;
import org.apache.skywalking.oap.server.library.module.ModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleDefine;
import org.apache.skywalking.oap.server.library.module.ModuleProvider;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import
org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+@Slf4j
public class NacosConfigurationTestProvider extends ModuleProvider {
- private static final Logger LOGGER =
LoggerFactory.getLogger(NacosConfigurationTestProvider.class);
-
ConfigChangeWatcher watcher;
+ GroupConfigChangeWatcher groupWatcher;
@Override
public String name() {
@@ -57,7 +59,7 @@ public class NacosConfigurationTestProvider extends
ModuleProvider {
@Override
public void notify(ConfigChangeEvent value) {
- LOGGER.info("ConfigChangeWatcher.ConfigChangeEvent: {}",
value);
+ log.info("ConfigChangeWatcher.ConfigChangeEvent: {}", value);
if (EventType.DELETE.equals(value.getEventType())) {
testValue = null;
} else {
@@ -70,6 +72,27 @@ public class NacosConfigurationTestProvider extends
ModuleProvider {
return testValue;
}
};
+
+ groupWatcher = new
GroupConfigChangeWatcher(NacosConfigurationTestModule.NAME, this,
"testKeyGroup") {
+ private Map<String, String> config = new ConcurrentHashMap<>();
+
+ @Override
+ public void notifyGroup(Map<String, ConfigChangeEvent> groupItems)
{
+ log.info("GroupConfigChangeWatcher.ConfigChangeEvents: {}",
groupItems);
+ groupItems.forEach((groupItemName, event) -> {
+ if (EventType.DELETE.equals(event.getEventType())) {
+ config.remove(groupItemName);
+ } else {
+ config.put(groupItemName, event.getNewValue());
+ }
+ });
+ }
+
+ @Override
+ public Map<String, String> groupItems() {
+ return config;
+ }
+ };
}
@Override
@@ -78,6 +101,11 @@ public class NacosConfigurationTestProvider extends
ModuleProvider {
.provider()
.getService(DynamicConfigurationService.class)
.registerConfigChangeWatcher(watcher);
+
+ getManager().find(ConfigurationModule.NAME)
+ .provider()
+ .getService(DynamicConfigurationService.class)
+ .registerConfigChangeWatcher(groupWatcher);
}
@Override
