This is an automated email from the ASF dual-hosted git repository. DaanHoogland pushed a commit to branch ghi13586-prometheusDrainage-20 in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 4261c3d54d942aba069934618630d4e7c0eec2c3 Author: Daan Hoogland <[email protected]> AuthorDate: Mon Jul 20 12:40:07 2026 +0200 optimise prometheus scraping workflow --- .../cloudstack/metrics/PrometheusExporterImpl.java | 12 +++++--- .../java/com/cloud/alert/AlertManagerImpl.java | 33 ++++++++++++++++------ 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index b49f11c7774..1d540cee5be 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -32,7 +32,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.commons.lang3.StringUtils; -import com.cloud.alert.AlertManager; import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.DomainJoinDao; import com.cloud.api.query.dao.StoragePoolJoinDao; @@ -126,8 +125,6 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp @Inject private DomainJoinDao domainDao; @Inject - private AlertManager alertManager; - @Inject DedicatedResourceDao _dedicatedDao; @Inject private AccountDao _accountDao; @@ -494,10 +491,17 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp public void updateMetrics() { final List<Item> latestMetricsItems = new ArrayList<Item>(); try { + // NOTE: capacity data is refreshed independently by AlertManagerImpl's own + // periodic CapacityChecker timer (see AlertManagerImpl#start()). Do NOT force a + // synchronous recalculateCapacity() here: it spins up a fresh thread pool per host + // and per storage pool across ALL zones on every single scrape, so with Z zones a + // single Prometheus scrape triggered Z redundant full recalculations. That extra, + // uncoordinated load compounds over time (thread churn + overlapping runs with the + // timer) and was the cause of https://github.com/apache/cloudstack/issues/13586 + // (scrape_duration_seconds climbing until a management-server restart). for (final DataCenterVO dc : dcDao.listAll()) { final String zoneName = dc.getName(); final String zoneUuid = dc.getUuid(); - alertManager.recalculateCapacity(); addHostMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); addVMMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); addVolumeMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); diff --git a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java index 7bf00037ee4..c85b3aa0117 100644 --- a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java +++ b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java @@ -161,6 +161,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi private final ExecutorService _executor; + private ExecutorService _capacityExecutorService; + protected SMTPMailSender mailSender; protected String[] recipients = null; protected String senderAddress = null; @@ -249,6 +251,9 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi @Override public boolean stop() { _timer.cancel(); + if (_capacityExecutorService != null) { + _capacityExecutorService.shutdown(); + } return true; } @@ -281,6 +286,24 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi } } + /** + * Shared, long-lived pool for capacity recalculation, reused across every + * recalculateHostCapacities()/recalculateStorageCapacities() call instead of creating and + * tearing down a new thread pool per invocation. Repeatedly creating/shutting down pools was + * unnecessary overhead under frequent callers (e.g. the Prometheus exporter used to trigger a + * full recalculation on every scrape, see https://github.com/apache/cloudstack/issues/13586). + * Lazily created so this remains safe for callers that invoke the recalculate methods directly + * without going through configure()/start() (e.g. unit tests). + */ + private synchronized ExecutorService getCapacityExecutorService() { + if (_capacityExecutorService == null || _capacityExecutorService.isShutdown()) { + _capacityExecutorService = Executors.newFixedThreadPool( + Math.max(1, CapacityManager.CapacityCalculateWorkers.value()), + new NamedThreadFactory("Capacity-Calculator")); + } + return _capacityExecutorService; + } + /** * Recalculates the capacities of hosts, including CPU and RAM. */ @@ -290,10 +313,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi return; } ConcurrentHashMap<Long, Future<Void>> futures = new ConcurrentHashMap<>(); - ExecutorService executorService = Executors.newFixedThreadPool(Math.max(1, - Math.min(CapacityManager.CapacityCalculateWorkers.value(), hostIds.size()))); for (Long hostId : hostIds) { - futures.put(hostId, executorService.submit(() -> { + futures.put(hostId, getCapacityExecutorService().submit(() -> { final HostVO host = hostDao.findById(hostId); _capacityMgr.updateCapacityForHost(host); return null; @@ -307,7 +328,6 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi entry.getKey(), e.getMessage()), e); } } - executorService.shutdown(); } protected void recalculateStorageCapacities() { @@ -316,10 +336,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi return; } ConcurrentHashMap<Long, Future<Void>> futures = new ConcurrentHashMap<>(); - ExecutorService executorService = Executors.newFixedThreadPool(Math.max(1, - Math.min(CapacityManager.CapacityCalculateWorkers.value(), storagePoolIds.size()))); for (Long poolId: storagePoolIds) { - futures.put(poolId, executorService.submit(() -> { + futures.put(poolId, getCapacityExecutorService().submit(() -> { Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { @@ -343,7 +361,6 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi entry.getKey(), e.getMessage()), e); } } - executorService.shutdown(); } @Override
