This is an automated email from the ASF dual-hosted git repository.
houyu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/bigtop-manager.git
The following commit(s) were added to refs/heads/main by this push:
new 9eac134c BIGTOP-4482: Add config property action to reduce payload
size (#254)
9eac134c is described below
commit 9eac134c19b49b743e125ae0e1a2ea6c1c66853d
Author: Zhiguo Wu <[email protected]>
AuthorDate: Tue Aug 5 21:07:31 2025 +0800
BIGTOP-4482: Add config property action to reduce payload size (#254)
---
bigtop-manager-bom/pom.xml | 16 ----
.../server/command/helper/JobCacheHelper.java | 5 ++
.../validator/RequiredServicesValidator.java | 22 ++---
.../PropertyDTO.java => enums/PropertyAction.java} | 25 +++---
.../manager/server/model/dto/PropertyDTO.java | 5 ++
.../manager/server/model/req/PropertyReq.java | 8 ++
.../server/service/impl/ServiceServiceImpl.java | 2 +-
.../manager/server/utils/StackConfigUtils.java | 100 ++++++++++++++-------
.../services/zookeeper/configuration/zoo.cfg.xml | 10 +++
9 files changed, 116 insertions(+), 77 deletions(-)
diff --git a/bigtop-manager-bom/pom.xml b/bigtop-manager-bom/pom.xml
index a361d541..d701c4f0 100644
--- a/bigtop-manager-bom/pom.xml
+++ b/bigtop-manager-bom/pom.xml
@@ -58,7 +58,6 @@
<pagehelper-spring-boot-starter.version>2.1.0</pagehelper-spring-boot-starter.version>
<victools.version>4.29.0</victools.version>
<arrow.version>18.3.0</arrow.version>
- <netty-buffer.version>4.1.123.Final</netty-buffer.version>
</properties>
<dependencyManagement>
@@ -224,21 +223,6 @@
<version>${pagehelper-spring-boot-starter.version}</version>
</dependency>
- <!-- dameng -->
- <dependency>
- <groupId>com.dameng</groupId>
- <artifactId>DmJdbcDriver18</artifactId>
- <version>${jdbc.dm.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>com.dameng</groupId>
- <artifactId>DmDialect-for-hibernate6.1</artifactId>
- <version>${jdbc.dm.version}</version>
- <scope>provided</scope>
- </dependency>
-
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>flight-sql-jdbc-driver</artifactId>
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/JobCacheHelper.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/JobCacheHelper.java
index d02f4fb5..5ee1368e 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/JobCacheHelper.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/JobCacheHelper.java
@@ -219,4 +219,9 @@ public class JobCacheHelper {
return hostMap;
}
+
+ private static Boolean hostRequiresAllData(String hostname) {
+ // Some services like prometheus requires all clusters info to collect
metrics.
+ return false;
+ }
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/RequiredServicesValidator.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/RequiredServicesValidator.java
index 81e8c34f..bf3675a4 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/RequiredServicesValidator.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/validator/RequiredServicesValidator.java
@@ -20,7 +20,6 @@ package org.apache.bigtop.manager.server.command.validator;
import org.apache.bigtop.manager.common.enums.Command;
import org.apache.bigtop.manager.dao.po.ServicePO;
-import org.apache.bigtop.manager.dao.repository.ClusterDao;
import org.apache.bigtop.manager.dao.repository.ServiceDao;
import org.apache.bigtop.manager.server.command.CommandIdentifier;
import org.apache.bigtop.manager.server.enums.ApiExceptionEnum;
@@ -36,15 +35,13 @@ import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Component;
import jakarta.annotation.Resource;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@Component
public class RequiredServicesValidator implements CommandValidator {
- @Resource
- private ClusterDao clusterDao;
-
@Resource
private ServiceDao serviceDao;
@@ -67,16 +64,19 @@ public class RequiredServicesValidator implements
CommandValidator {
ServiceDTO serviceDTO = StackUtils.getServiceDTO(serviceName);
List<String> requiredServices = serviceDTO.getRequiredServices();
if (CollectionUtils.isEmpty(requiredServices)) {
- return;
+ continue;
}
- List<ServicePO> servicePOList =
serviceDao.findByClusterIdAndNames(clusterId, requiredServices);
- List<String> list =
servicePOList.stream().map(ServicePO::getName).toList();
-
- requiredServices.removeAll(list);
+ List<ServicePO> services = serviceDao.findByClusterId(clusterId);
+ List<ServicePO> infraServices = serviceDao.findByClusterId(0L);
+ services.addAll(infraServices);
- if (!new HashSet<>(serviceNames).containsAll(requiredServices)) {
- throw new
ApiException(ApiExceptionEnum.SERVICE_REQUIRED_NOT_FOUND, String.join(",",
requiredServices));
+ List<String> allServices =
+ new
ArrayList<>(services.stream().map(ServicePO::getName).toList());
+ allServices.addAll(serviceNames);
+ if (!new HashSet<>(allServices).containsAll(requiredServices)) {
+ requiredServices.removeAll(allServices);
+ throw new ApiException(ApiExceptionEnum.SERVICE_NOT_FOUND,
String.join(",", requiredServices));
}
}
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PropertyDTO.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/PropertyAction.java
similarity index 72%
copy from
bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PropertyDTO.java
copy to
bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/PropertyAction.java
index f4845c4e..8a0b88b9 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PropertyDTO.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/PropertyAction.java
@@ -16,22 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.bigtop.manager.server.model.dto;
+package org.apache.bigtop.manager.server.enums;
-import lombok.Data;
+import com.fasterxml.jackson.annotation.JsonCreator;
-import java.io.Serializable;
+public enum PropertyAction {
+ ADD,
+ UPDATE,
+ DELETE;
-@Data
-public class PropertyDTO implements Serializable {
-
- private String name;
-
- private String value;
-
- private String displayName;
-
- private String desc;
-
- private AttrsDTO attrs;
+ @JsonCreator
+ public static PropertyAction fromString(String value) {
+ return PropertyAction.valueOf(value.toUpperCase());
+ }
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PropertyDTO.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PropertyDTO.java
index f4845c4e..1eec26f2 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PropertyDTO.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PropertyDTO.java
@@ -18,6 +18,8 @@
*/
package org.apache.bigtop.manager.server.model.dto;
+import org.apache.bigtop.manager.server.enums.PropertyAction;
+
import lombok.Data;
import java.io.Serializable;
@@ -34,4 +36,7 @@ public class PropertyDTO implements Serializable {
private String desc;
private AttrsDTO attrs;
+
+ // Only exists in request body
+ private PropertyAction action;
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/PropertyReq.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/PropertyReq.java
index e3e961a5..87439be4 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/PropertyReq.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/PropertyReq.java
@@ -18,6 +18,8 @@
*/
package org.apache.bigtop.manager.server.model.req;
+import org.apache.bigtop.manager.server.enums.PropertyAction;
+
import lombok.Data;
import jakarta.validation.constraints.NotBlank;
@@ -35,4 +37,10 @@ public class PropertyReq {
private String desc;
private AttrsReq attrs;
+
+ /**
+ * Action to be performed on the property.
+ * This could be used to indicate operations like 'add', 'update', or
'delete'.
+ */
+ private PropertyAction action;
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java
index 3e4cd61f..3ee0b874 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java
@@ -169,7 +169,7 @@ public class ServiceServiceImpl implements ServiceService {
List<ServiceConfigDTO> newConfigs;
List<ServiceConfigDTO> mergedConfigs;
- // Merge stack config with existing config first, in case new property
has been added to stack config.
+ // Merge stack config with existing config first, in case new property
has been added to config xml.
oriConfigs = StackUtils.SERVICE_CONFIG_MAP.get(servicePO.getName());
newConfigs = ServiceConfigConverter.INSTANCE.fromPO2DTO(configs);
mergedConfigs = StackConfigUtils.mergeServiceConfigs(oriConfigs,
newConfigs);
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java
index 430172e4..d4a86d7c 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java
@@ -18,6 +18,7 @@
*/
package org.apache.bigtop.manager.server.utils;
+import org.apache.bigtop.manager.server.enums.PropertyAction;
import org.apache.bigtop.manager.server.model.dto.AttrsDTO;
import org.apache.bigtop.manager.server.model.dto.PropertyDTO;
import org.apache.bigtop.manager.server.model.dto.ServiceConfigDTO;
@@ -26,7 +27,6 @@ import
org.apache.bigtop.manager.server.stack.model.PropertyModel;
import org.apache.bigtop.manager.server.stack.xml.ConfigurationXml;
import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
@@ -87,43 +87,60 @@ public class StackConfigUtils {
return mergedConfigs;
}
- // Assign id for each service config
- for (ServiceConfigDTO config : mergedConfigs) {
- config.setId(overrideConfigs.stream()
- .filter(x -> x.getName().equals(config.getName()))
- .findFirst()
- .map(ServiceConfigDTO::getId)
- .orElse(null));
- }
+ Map<String, Map<String, PropertyDTO>> mergedConfigsMap =
serviceConfig2Map(mergedConfigs);
+ for (ServiceConfigDTO overrideConfig : overrideConfigs) {
+ String configName = overrideConfig.getName();
- Map<String, Map<String, String>> overrideConfigsMap =
serviceConfig2Map(overrideConfigs);
- for (ServiceConfigDTO mergedConfig : mergedConfigs) {
- String configName = mergedConfig.getName();
- if (!overrideConfigsMap.containsKey(configName)) {
- continue;
- }
-
- // Override existing properties
- Map<String, String> overridePropertiesMap =
overrideConfigsMap.get(configName);
- for (PropertyDTO property : mergedConfig.getProperties()) {
+ // Merge properties from override config to existing config
+ Map<String, PropertyDTO> mergedPropertiesMap =
mergedConfigsMap.get(configName);
+ for (PropertyDTO property : overrideConfig.getProperties()) {
String propertyName = property.getName();
- String value = overridePropertiesMap.remove(propertyName);
- if (value != null) {
- property.setValue(value);
+ PropertyAction action = property.getAction();
+ if (action == null) {
+ // null means property does not come from request body,
but from config xml or database.
+ // the logic the same as `UPDATE` action.
+ action = PropertyAction.UPDATE;
+ }
+ switch (action) {
+ case ADD, UPDATE -> {
+ PropertyDTO propertyDTO =
mergedPropertiesMap.get(propertyName);
+ if (propertyDTO == null) {
+ propertyDTO = new PropertyDTO();
+ propertyDTO.setName(propertyName);
+ }
+
+ propertyDTO.setValue(property.getValue());
+ mergedPropertiesMap.put(propertyName, propertyDTO);
+ }
+ case DELETE -> mergedPropertiesMap.remove(propertyName);
}
}
+ }
- // We may still have some properties added by user manually
- if (MapUtils.isNotEmpty(overridePropertiesMap)) {
- for (Map.Entry<String, String> entry :
overridePropertiesMap.entrySet()) {
- PropertyDTO property = new PropertyDTO();
- property.setName(entry.getKey());
- property.setValue(entry.getValue());
- mergedConfig.getProperties().add(property);
- }
+ mergedConfigs = map2ServiceConfig(mergedConfigsMap);
+
+ // Assign id for each service config
+ Map<String, Long> configIdMap = new HashMap<>();
+ for (ServiceConfigDTO config : oriConfigs) {
+ String name = config.getName();
+ Long id = config.getId();
+ if (id != null) {
+ configIdMap.put(name, id);
}
}
+ for (ServiceConfigDTO config : overrideConfigs) {
+ String name = config.getName();
+ Long id = config.getId();
+ if (id != null) {
+ configIdMap.put(name, id);
+ }
+ }
+
+ for (ServiceConfigDTO config : mergedConfigs) {
+ config.setId(configIdMap.get(config.getName()));
+ }
+
return mergedConfigs;
}
@@ -133,16 +150,16 @@ public class StackConfigUtils {
* @param configs List<ServiceConfigDTO>
* @return Map<String, Map<String, String>>
*/
- private static Map<String, Map<String, String>>
serviceConfig2Map(List<ServiceConfigDTO> configs) {
- Map<String, Map<String, String>> outerMap = new HashMap<>();
+ private static Map<String, Map<String, PropertyDTO>>
serviceConfig2Map(List<ServiceConfigDTO> configs) {
+ Map<String, Map<String, PropertyDTO>> outerMap = new HashMap<>();
if (CollectionUtils.isEmpty(configs)) {
return outerMap;
}
for (ServiceConfigDTO config : configs) {
- Map<String, String> innerMap = new HashMap<>();
+ Map<String, PropertyDTO> innerMap = new HashMap<>();
for (PropertyDTO property : config.getProperties()) {
- innerMap.put(property.getName(), property.getValue());
+ innerMap.put(property.getName(), property);
}
outerMap.put(config.getName(), innerMap);
@@ -150,4 +167,19 @@ public class StackConfigUtils {
return outerMap;
}
+
+ private static List<ServiceConfigDTO> map2ServiceConfig(Map<String,
Map<String, PropertyDTO>> configMap) {
+ List<ServiceConfigDTO> configs = new ArrayList<>();
+ for (Map.Entry<String, Map<String, PropertyDTO>> entry :
configMap.entrySet()) {
+ ServiceConfigDTO serviceConfig = new ServiceConfigDTO();
+ String name = entry.getKey();
+ List<PropertyDTO> properties = new
ArrayList<>(entry.getValue().values());
+
+ serviceConfig.setName(name);
+ serviceConfig.setProperties(properties);
+ configs.add(serviceConfig);
+ }
+
+ return configs;
+ }
}
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zoo.cfg.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zoo.cfg.xml
index bd09086c..09d3478b 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zoo.cfg.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zoo.cfg.xml
@@ -89,6 +89,16 @@
<value>9393</value>
<description>The port the embedded Jetty server listens on. Defaults
to 8080.</description>
</property>
+ <property>
+ <name>metricsProvider.className</name>
+
<value>org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider</value>
+ <description>ZooKeeper prometheus metrics provider.</description>
+ </property>
+ <property>
+ <name>metricsProvider.httpPort</name>
+ <value>7000</value>
+ <description>ZooKeeper prometheus metrics provider http
port.</description>
+ </property>
<property>
<name>content</name>
<description>The port the embedded Jetty server listens on. Defaults
to 8080.</description>
