This is an automated email from the ASF dual-hosted git repository.
arnold pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/fineract.git
The following commit(s) were added to refs/heads/develop by this push:
new da5dad27f FINERACT-2166: Expose tenant connection pool metrics
da5dad27f is described below
commit da5dad27f3b93aeabb20d7237c112a9c7d33ee15
Author: Arnold Galovics <[email protected]>
AuthorDate: Thu Jan 9 21:27:45 2025 +0100
FINERACT-2166: Expose tenant connection pool metrics
---
docker-compose-web-app.yml | 2 +-
.../DataSourcePerTenantServiceFactory.java | 23 ++--
.../TomcatJdbcDataSourcePerTenantService.java | 16 +--
.../TenantConnectionPoolMetricsTracker.java | 151 +++++++++++++++++++++
.../TenantConnectionPoolMetricsTrackerFactory.java | 37 +++++
.../DataSourcePerTenantServiceFactoryTest.java | 13 +-
6 files changed, 210 insertions(+), 32 deletions(-)
diff --git a/docker-compose-web-app.yml b/docker-compose-web-app.yml
index 7d1b768ef..f7d7031d6 100644
--- a/docker-compose-web-app.yml
+++ b/docker-compose-web-app.yml
@@ -20,7 +20,7 @@ version: "3.8"
services:
# Frontend service
community-app:
- image: openmf/web-app:latest
+ image: openmf/web-app:master
container_name: mifos-web-app
restart: always
ports:
diff --git
a/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/DataSourcePerTenantServiceFactory.java
b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/DataSourcePerTenantServiceFactory.java
index ea980e516..f62807168 100644
---
a/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/DataSourcePerTenantServiceFactory.java
+++
b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/DataSourcePerTenantServiceFactory.java
@@ -23,46 +23,40 @@ import static
org.apache.fineract.infrastructure.core.domain.FineractPlatformTen
import com.zaxxer.hikari.HikariConfig;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import io.micrometer.core.instrument.MeterRegistry;
import javax.sql.DataSource;
+import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
import
org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection;
+import
org.apache.fineract.infrastructure.core.service.database.metrics.TenantConnectionPoolMetricsTrackerFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component;
/**
- *
* Factory class to get data source service based on the details stored in
{@link FineractPlatformTenantConnection}
* variable
- *
*/
@Component
@Slf4j
+@RequiredArgsConstructor
public class DataSourcePerTenantServiceFactory {
private final HikariConfig hikariConfig;
private final FineractProperties fineractProperties;
private final ApplicationContext context;
+ @Qualifier("hikariTenantDataSource")
private final DataSource tenantDataSource;
private final HikariDataSourceFactory hikariDataSourceFactory;
private final DatabasePasswordEncryptor databasePasswordEncryptor;
-
- public
DataSourcePerTenantServiceFactory(@Qualifier("hikariTenantDataSource")
DataSource tenantDataSource, HikariConfig hikariConfig,
- FineractProperties fineractProperties, ApplicationContext context,
HikariDataSourceFactory hikariDataSourceFactory,
- DatabasePasswordEncryptor databasePasswordEncryptor) {
- this.hikariConfig = hikariConfig;
- this.fineractProperties = fineractProperties;
- this.context = context;
- this.tenantDataSource = tenantDataSource;
- this.hikariDataSourceFactory = hikariDataSourceFactory;
- this.databasePasswordEncryptor = databasePasswordEncryptor;
- }
+ private final MeterRegistry meterRegistry;
@SuppressFBWarnings(value = "SLF4J_SIGN_ONLY_FORMAT")
- public DataSource createNewDataSourceFor(final
FineractPlatformTenantConnection tenantConnection) {
+ public DataSource createNewDataSourceFor(FineractPlatformTenant tenant,
FineractPlatformTenantConnection tenantConnection) {
if
(!databasePasswordEncryptor.isMasterPasswordHashValid(tenantConnection.getMasterPasswordHash()))
{
throw new IllegalArgumentException(
"Invalid master password on tenant connection
%d.".formatted(tenantConnection.getConnectionId()));
@@ -103,6 +97,7 @@ public class DataSourcePerTenantServiceFactory {
//
https://github.com/brettwooldridge/HikariCP/wiki/MBean-(JMX)-Monitoring-and-Management
config.setRegisterMbeans(true);
+ config.setMetricsTrackerFactory(new
TenantConnectionPoolMetricsTrackerFactory(tenant.getTenantIdentifier(),
meterRegistry));
// https://github.com/brettwooldridge/HikariCP/wiki/MySQL-Configuration
// These are the properties for each Tenant DB; the same configuration
diff --git
a/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/TomcatJdbcDataSourcePerTenantService.java
b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/TomcatJdbcDataSourcePerTenantService.java
index 9e11be36c..e3b792e1b 100644
---
a/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/TomcatJdbcDataSourcePerTenantService.java
+++
b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/TomcatJdbcDataSourcePerTenantService.java
@@ -24,12 +24,12 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.sql.DataSource;
+import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
import
org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
import
org.apache.fineract.infrastructure.core.service.tenant.TenantDetailsService;
-import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationListener;
import org.springframework.context.event.ContextRefreshedEvent;
@@ -43,22 +43,16 @@ import org.springframework.stereotype.Service;
*/
@Slf4j
@Service
+@RequiredArgsConstructor
public class TomcatJdbcDataSourcePerTenantService implements
RoutingDataSourceService, ApplicationListener<ContextRefreshedEvent> {
private static final Map<Long, DataSource> TENANT_TO_DATA_SOURCE_MAP = new
ConcurrentHashMap<>();
+ @Qualifier("hikariTenantDataSource")
private final DataSource tenantDataSource;
private final TenantDetailsService tenantDetailsService;
private final DataSourcePerTenantServiceFactory
dataSourcePerTenantServiceFactory;
- @Autowired
- public TomcatJdbcDataSourcePerTenantService(final
@Qualifier("hikariTenantDataSource") DataSource tenantDataSource,
- final DataSourcePerTenantServiceFactory
dataSourcePerTenantServiceFactory, final TenantDetailsService
tenantDetailsService) {
- this.tenantDataSource = tenantDataSource;
- this.dataSourcePerTenantServiceFactory =
dataSourcePerTenantServiceFactory;
- this.tenantDetailsService = tenantDetailsService;
- }
-
@Override
public DataSource retrieveDataSource() {
// default to tenant database datasource
@@ -71,7 +65,7 @@ public class TomcatJdbcDataSourcePerTenantService implements
RoutingDataSourceSe
// if tenantConnection information available switch to the
// appropriate datasource for that tenant.
actualDataSource =
TENANT_TO_DATA_SOURCE_MAP.computeIfAbsent(tenantConnectionKey,
- (key) ->
dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenantConnection));
+ (key) ->
dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenant,
tenantConnection));
}
@@ -91,7 +85,7 @@ public class TomcatJdbcDataSourcePerTenantService implements
RoutingDataSourceSe
final FineractPlatformTenantConnection tenantConnection =
tenant.getConnection();
Long tenantConnectionKey = tenantConnection.getConnectionId();
TENANT_TO_DATA_SOURCE_MAP.computeIfAbsent(tenantConnectionKey, (key)
-> {
- DataSource tenantSpecificDataSource =
dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenantConnection);
+ DataSource tenantSpecificDataSource =
dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenant,
tenantConnection);
try (Connection connection =
tenantSpecificDataSource.getConnection()) {
String url = connection.getMetaData().getURL();
log.debug("Established database connection with URL {}", url);
diff --git
a/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTracker.java
b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTracker.java
new file mode 100644
index 000000000..9b1f16822
--- /dev/null
+++
b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTracker.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fineract.infrastructure.core.service.database.metrics;
+
+import com.zaxxer.hikari.metrics.IMetricsTracker;
+import com.zaxxer.hikari.metrics.PoolStats;
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Timer;
+import java.util.concurrent.TimeUnit;
+
+public class TenantConnectionPoolMetricsTracker implements IMetricsTracker {
+
+ public static final String HIKARI_METRIC_NAME_PREFIX = ".hikaricp";
+
+ private static final String METRIC_CATEGORY = "pool";
+ private static final String METRIC_NAME_WAIT = HIKARI_METRIC_NAME_PREFIX +
".connections.acquire";
+ private static final String METRIC_NAME_USAGE = HIKARI_METRIC_NAME_PREFIX
+ ".connections.usage";
+ private static final String METRIC_NAME_CONNECT =
HIKARI_METRIC_NAME_PREFIX + ".connections.creation";
+
+ private static final String METRIC_NAME_TIMEOUT_RATE =
HIKARI_METRIC_NAME_PREFIX + ".connections.timeout";
+ private static final String METRIC_NAME_TOTAL_CONNECTIONS =
HIKARI_METRIC_NAME_PREFIX + ".connections";
+ private static final String METRIC_NAME_IDLE_CONNECTIONS =
HIKARI_METRIC_NAME_PREFIX + ".connections.idle";
+ private static final String METRIC_NAME_ACTIVE_CONNECTIONS =
HIKARI_METRIC_NAME_PREFIX + ".connections.active";
+ private static final String METRIC_NAME_PENDING_CONNECTIONS =
HIKARI_METRIC_NAME_PREFIX + ".connections.pending";
+ private static final String METRIC_NAME_MAX_CONNECTIONS =
HIKARI_METRIC_NAME_PREFIX + ".connections.max";
+ private static final String METRIC_NAME_MIN_CONNECTIONS =
HIKARI_METRIC_NAME_PREFIX + ".connections.min";
+
+ private final Timer connectionObtainTimer;
+ private final Counter connectionTimeoutCounter;
+ private final Timer connectionUsage;
+ private final Timer connectionCreation;
+ private final Gauge totalConnectionGauge;
+ private final Gauge idleConnectionGauge;
+ private final Gauge activeConnectionGauge;
+ private final Gauge pendingConnectionGauge;
+ private final Gauge maxConnectionGauge;
+ private final Gauge minConnectionGauge;
+ private final MeterRegistry meterRegistry;
+ private final PoolStats poolStats;
+
+ public TenantConnectionPoolMetricsTracker(String tenantIdentifier, String
poolName, PoolStats poolStats, MeterRegistry meterRegistry) {
+ // poolStats must be held with a 'strong reference' even though it is
never referenced within this class
+ this.poolStats = poolStats; // DO NOT REMOVE
+
+ this.meterRegistry = meterRegistry;
+
+ String metricPrefix = "fineract.tenants." + tenantIdentifier;
+
+ this.connectionObtainTimer = Timer.builder(metricPrefix +
METRIC_NAME_WAIT).description("Connection acquire time") //
+ .tags(METRIC_CATEGORY, poolName) //
+ .register(meterRegistry);
+
+ this.connectionCreation = Timer.builder(metricPrefix +
METRIC_NAME_CONNECT).description("Connection creation time") //
+ .tags(METRIC_CATEGORY, poolName) //
+ .register(meterRegistry);
+
+ this.connectionUsage = Timer.builder(metricPrefix +
METRIC_NAME_USAGE).description("Connection usage time") //
+ .tags(METRIC_CATEGORY, poolName) //
+ .register(meterRegistry);
+
+ this.connectionTimeoutCounter = Counter.builder(metricPrefix +
METRIC_NAME_TIMEOUT_RATE)
+ .description("Connection timeout total count") //
+ .tags(METRIC_CATEGORY, poolName) //
+ .register(meterRegistry);
+
+ this.totalConnectionGauge = Gauge.builder(metricPrefix +
METRIC_NAME_TOTAL_CONNECTIONS, poolStats, PoolStats::getTotalConnections)
+ .description("Total connections") //
+ .tags(METRIC_CATEGORY, poolName) //
+ .register(meterRegistry);
+
+ this.idleConnectionGauge = Gauge.builder(metricPrefix +
METRIC_NAME_IDLE_CONNECTIONS, poolStats, PoolStats::getIdleConnections)
+ .description("Idle connections") //
+ .tags(METRIC_CATEGORY, poolName) //
+ .register(meterRegistry);
+
+ this.activeConnectionGauge = Gauge
+ .builder(metricPrefix + METRIC_NAME_ACTIVE_CONNECTIONS,
poolStats, PoolStats::getActiveConnections)
+ .description("Active connections") //
+ .tags(METRIC_CATEGORY, poolName) //
+ .register(meterRegistry);
+
+ this.pendingConnectionGauge = Gauge.builder(metricPrefix +
METRIC_NAME_PENDING_CONNECTIONS, poolStats, PoolStats::getPendingThreads)
+ .description("Pending threads") //
+ .tags(METRIC_CATEGORY, poolName) //
+ .register(meterRegistry);
+
+ this.maxConnectionGauge = Gauge.builder(metricPrefix +
METRIC_NAME_MAX_CONNECTIONS, poolStats, PoolStats::getMaxConnections)
+ .description("Max connections") //
+ .tags(METRIC_CATEGORY, poolName) //
+ .register(meterRegistry);
+
+ this.minConnectionGauge = Gauge.builder(metricPrefix +
METRIC_NAME_MIN_CONNECTIONS, poolStats, PoolStats::getMinConnections)
+ .description("Min connections") //
+ .tags(METRIC_CATEGORY, poolName) //
+ .register(meterRegistry);
+
+ }
+
+ @Override
+ public void recordConnectionAcquiredNanos(final long elapsedAcquiredNanos)
{
+ connectionObtainTimer.record(elapsedAcquiredNanos,
TimeUnit.NANOSECONDS);
+ }
+
+ @Override
+ public void recordConnectionUsageMillis(final long elapsedBorrowedMillis) {
+ connectionUsage.record(elapsedBorrowedMillis, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public void recordConnectionTimeout() {
+ connectionTimeoutCounter.increment();
+ }
+
+ @Override
+ public void recordConnectionCreatedMillis(long connectionCreatedMillis) {
+ connectionCreation.record(connectionCreatedMillis,
TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public void close() {
+ meterRegistry.remove(connectionObtainTimer);
+ meterRegistry.remove(connectionTimeoutCounter);
+ meterRegistry.remove(connectionUsage);
+ meterRegistry.remove(connectionCreation);
+ meterRegistry.remove(totalConnectionGauge);
+ meterRegistry.remove(idleConnectionGauge);
+ meterRegistry.remove(activeConnectionGauge);
+ meterRegistry.remove(pendingConnectionGauge);
+ meterRegistry.remove(maxConnectionGauge);
+ meterRegistry.remove(minConnectionGauge);
+ }
+}
diff --git
a/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTrackerFactory.java
b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTrackerFactory.java
new file mode 100644
index 000000000..6a7631db4
--- /dev/null
+++
b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTrackerFactory.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.database.metrics;
+
+import com.zaxxer.hikari.metrics.IMetricsTracker;
+import com.zaxxer.hikari.metrics.MetricsTrackerFactory;
+import com.zaxxer.hikari.metrics.PoolStats;
+import io.micrometer.core.instrument.MeterRegistry;
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor
+public class TenantConnectionPoolMetricsTrackerFactory implements
MetricsTrackerFactory {
+
+ private final String tenantIdentifier;
+ private final MeterRegistry registry;
+
+ @Override
+ public IMetricsTracker create(String poolName, PoolStats poolStats) {
+ return new TenantConnectionPoolMetricsTracker(tenantIdentifier,
poolName, poolStats, registry);
+ }
+}
diff --git
a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/core/DataSourcePerTenantServiceFactoryTest.java
b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/core/DataSourcePerTenantServiceFactoryTest.java
index 5ead9f5f0..546bc541a 100644
---
a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/core/DataSourcePerTenantServiceFactoryTest.java
+++
b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/core/DataSourcePerTenantServiceFactoryTest.java
@@ -87,6 +87,7 @@ public class DataSourcePerTenantServiceFactoryTest {
public static final String MASTER_MASTER_PASSWORD = "fineract";
public static final String MASTER_ENCRYPTION = "AES/CBC/PKCS5Padding";
+ public static final FineractPlatformTenant TENANT = new
FineractPlatformTenant(1L, "", "", "", null);
@Mock
private FineractProperties fineractProperties;
@@ -183,7 +184,7 @@ public class DataSourcePerTenantServiceFactoryTest {
given(fineractProperties.getMode()).willReturn(modeProperties);
// when
- DataSource dataSource =
underTest.createNewDataSourceFor(defaultTenant.getConnection());
+ DataSource dataSource = underTest.createNewDataSourceFor(TENANT,
defaultTenant.getConnection());
// then
assertNotNull(dataSource);
@@ -215,7 +216,7 @@ public class DataSourcePerTenantServiceFactoryTest {
config.setMinPoolSize(minPoolSize);
// when
- DataSource dataSource =
underTest.createNewDataSourceFor(defaultTenant.getConnection());
+ DataSource dataSource = underTest.createNewDataSourceFor(TENANT,
defaultTenant.getConnection());
// then
assertNotNull(dataSource);
@@ -247,7 +248,7 @@ public class DataSourcePerTenantServiceFactoryTest {
config.setMaxPoolSize(maxPoolSize);
// when
- DataSource dataSource =
underTest.createNewDataSourceFor(defaultTenant.getConnection());
+ DataSource dataSource = underTest.createNewDataSourceFor(TENANT,
defaultTenant.getConnection());
// then
assertNotNull(dataSource);
@@ -281,7 +282,7 @@ public class DataSourcePerTenantServiceFactoryTest {
config.setMaxPoolSize(maxPoolSize);
// when
- DataSource dataSource =
underTest.createNewDataSourceFor(defaultTenant.getConnection());
+ DataSource dataSource = underTest.createNewDataSourceFor(TENANT,
defaultTenant.getConnection());
// then
assertNotNull(dataSource);
@@ -307,7 +308,7 @@ public class DataSourcePerTenantServiceFactoryTest {
given(fineractProperties.getMode()).willReturn(modeProperties);
// when
- DataSource dataSource =
underTest.createNewDataSourceFor(defaultTenant.getConnection());
+ DataSource dataSource = underTest.createNewDataSourceFor(TENANT,
defaultTenant.getConnection());
// then
assertNotNull(dataSource);
@@ -333,7 +334,7 @@ public class DataSourcePerTenantServiceFactoryTest {
given(fineractProperties.getMode()).willReturn(modeProperties);
// when
- DataSource dataSource =
underTest.createNewDataSourceFor(defaultTenant.getConnection());
+ DataSource dataSource = underTest.createNewDataSourceFor(TENANT,
defaultTenant.getConnection());
// then
assertNotNull(dataSource);
