This is an automated email from the ASF dual-hosted git repository.
yasith pushed a commit to branch merge-svcs
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/merge-svcs by this push:
new d10e9a895d fix monitoring server. update python sdk to use API_SERVER
configs + multiplexing for all services.
d10e9a895d is described below
commit d10e9a895d72dfbe7f35b815d3d32a38f1ebdb15
Author: yasithdev <[email protected]>
AuthorDate: Tue Aug 19 18:41:38 2025 -0400
fix monitoring server. update python sdk to use API_SERVER configs +
multiplexing for all services.
---
.../src/main/java/org/apache/airavata/Main.java | 6 +--
.../helix/core/participant/HelixParticipant.java | 1 -
.../helix/impl/workflow/ParserWorkflowManager.java | 13 +-----
.../helix/impl/workflow/WorkflowManager.java | 13 ------
.../monitor/platform/MonitoringServer.java | 1 -
.../src/main/resources/airavata-server.properties | 8 +---
.../airavata-python-sdk/airavata_sdk/__init__.py | 48 ----------------------
.../clients/credential_store_client.py | 6 +--
.../airavata_sdk/clients/group_manager_client.py | 6 +--
.../airavata_sdk/clients/iam_admin_client.py | 6 +--
.../clients/sharing_registry_client.py | 6 +--
.../airavata_sdk/clients/tenant_profile_client.py | 6 +--
.../airavata_sdk/clients/user_profile_client.py | 6 +--
.../airavata_sdk/transport/utils.py | 42 +++++++++----------
14 files changed, 45 insertions(+), 123 deletions(-)
diff --git a/airavata-api/src/main/java/org/apache/airavata/Main.java
b/airavata-api/src/main/java/org/apache/airavata/Main.java
index 493270b0ee..52cd16e1dc 100644
--- a/airavata-api/src/main/java/org/apache/airavata/Main.java
+++ b/airavata-api/src/main/java/org/apache/airavata/Main.java
@@ -72,10 +72,10 @@ public class Main {
assert jobScheduler != null;
// jobScheduler.scheduleClusterStatusMonitoring();
- if
(ServerSettings.getBooleanSetting("post.workflow.manager.monitoring.enabled")) {
+ if (ServerSettings.getBooleanSetting("api.monitoring.enabled")) {
var monitoringServer = new MonitoringServer(
-
ServerSettings.getSetting("post.workflow.manager.monitoring.host"),
-
ServerSettings.getIntSetting("post.workflow.manager.monitoring.port"));
+ ServerSettings.getSetting("api.monitoring.host"),
+ ServerSettings.getIntSetting("api.monitoring.port"));
monitoringServer.start();
Runtime.getRuntime().addShutdownHook(new
Thread(monitoringServer::stop));
}
diff --git
a/airavata-api/src/main/java/org/apache/airavata/helix/core/participant/HelixParticipant.java
b/airavata-api/src/main/java/org/apache/airavata/helix/core/participant/HelixParticipant.java
index 716853d3ca..6f16c919c6 100644
---
a/airavata-api/src/main/java/org/apache/airavata/helix/core/participant/HelixParticipant.java
+++
b/airavata-api/src/main/java/org/apache/airavata/helix/core/participant/HelixParticipant.java
@@ -26,7 +26,6 @@ import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.helix.core.AbstractTask;
import org.apache.airavata.helix.core.support.TaskHelperImpl;
import org.apache.airavata.helix.task.api.annotation.TaskDef;
-import org.apache.airavata.monitor.platform.MonitoringServer;
import org.apache.helix.InstanceType;
import org.apache.helix.constants.InstanceConstants.InstanceOperation;
import org.apache.helix.examples.OnlineOfflineStateModelFactory;
diff --git
a/airavata-api/src/main/java/org/apache/airavata/helix/impl/workflow/ParserWorkflowManager.java
b/airavata-api/src/main/java/org/apache/airavata/helix/impl/workflow/ParserWorkflowManager.java
index 3ada0b9720..1c0583396f 100644
---
a/airavata-api/src/main/java/org/apache/airavata/helix/impl/workflow/ParserWorkflowManager.java
+++
b/airavata-api/src/main/java/org/apache/airavata/helix/impl/workflow/ParserWorkflowManager.java
@@ -37,7 +37,6 @@ import
org.apache.airavata.model.application.io.OutputDataObjectType;
import org.apache.airavata.model.experiment.ExperimentModel;
import org.apache.airavata.model.process.ProcessModel;
import org.apache.airavata.monitor.platform.CountMonitor;
-import org.apache.airavata.monitor.platform.MonitoringServer;
import org.apache.airavata.registry.api.RegistryService;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;
@@ -67,16 +66,8 @@ public class ParserWorkflowManager extends WorkflowManager {
@Override
public void run() {
try {
- if
(ServerSettings.getBooleanSetting("parser.workflow.manager.monitoring.enabled"))
{
- MonitoringServer monitoringServer = new MonitoringServer(
-
ServerSettings.getSetting("parser.workflow.manager.monitoring.host"),
-
ServerSettings.getIntSetting("parser.workflow.manager.monitoring.port"));
- monitoringServer.start();
-
- Runtime.getRuntime().addShutdownHook(new
Thread(monitoringServer::stop));
- }
- this.init();
- this.runConsumer();
+ this.init();
+ this.runConsumer();
} catch (Exception e) {
logger.error("Error starting PreWorkflowManager", e);
}
diff --git
a/airavata-api/src/main/java/org/apache/airavata/helix/impl/workflow/WorkflowManager.java
b/airavata-api/src/main/java/org/apache/airavata/helix/impl/workflow/WorkflowManager.java
index 7a692024fb..6b6efa03ff 100644
---
a/airavata-api/src/main/java/org/apache/airavata/helix/impl/workflow/WorkflowManager.java
+++
b/airavata-api/src/main/java/org/apache/airavata/helix/impl/workflow/WorkflowManager.java
@@ -39,7 +39,6 @@ import
org.apache.airavata.model.messaging.event.ProcessStatusChangeEvent;
import org.apache.airavata.model.process.ProcessWorkflow;
import org.apache.airavata.model.status.ProcessState;
import org.apache.airavata.model.status.ProcessStatus;
-import org.apache.airavata.monitor.platform.MonitoringServer;
import org.apache.airavata.registry.api.RegistryService;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.zookeeper.api.client.RealmAwareZkClient.RealmMode;
@@ -66,23 +65,11 @@ public abstract class WorkflowManager implements Runnable {
protected void initComponents() throws Exception {
this.registry = AiravataServiceFactory.getRegistry();
- this.initMonitoring();
initHelixAdmin();
initWorkflowOperators();
initStatusPublisher();
}
- private void initMonitoring() throws ApplicationSettingsException,
IOException {
- if
(ServerSettings.getBooleanSetting("pre.workflow.manager.monitoring.enabled")) {
- MonitoringServer monitoringServer = new MonitoringServer(
-
ServerSettings.getSetting("pre.workflow.manager.monitoring.host"),
-
ServerSettings.getIntSetting("pre.workflow.manager.monitoring.port"));
- monitoringServer.start();
-
- Runtime.getRuntime().addShutdownHook(new
Thread(monitoringServer::stop));
- }
- }
-
private void initWorkflowOperators() throws Exception {
if (!loadBalanceClusters) {
diff --git
a/airavata-api/src/main/java/org/apache/airavata/monitor/platform/MonitoringServer.java
b/airavata-api/src/main/java/org/apache/airavata/monitor/platform/MonitoringServer.java
index a1bfca1e42..1cde8f5593 100644
---
a/airavata-api/src/main/java/org/apache/airavata/monitor/platform/MonitoringServer.java
+++
b/airavata-api/src/main/java/org/apache/airavata/monitor/platform/MonitoringServer.java
@@ -20,7 +20,6 @@
package org.apache.airavata.monitor.platform;
import io.prometheus.client.exporter.HTTPServer;
-import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/airavata-api/src/main/resources/airavata-server.properties
b/airavata-api/src/main/resources/airavata-server.properties
index febed3a742..ff62ef6ee6 100644
--- a/airavata-api/src/main/resources/airavata-server.properties
+++ b/airavata-api/src/main/resources/airavata-server.properties
@@ -118,7 +118,7 @@ data.streaming.enabled=False
scheduler.job.scanning.enabled=false
api.server.min.threads=50
-api.server.host=localhost
+api.server.host=airavata.host
api.server.port=8970
api.monitoring.enabled=true
api.monitoring.host=airavata.host
@@ -126,15 +126,9 @@ api.monitoring.port=8930
post.workflow.manager.loadbalance.clusters=false
post.workflow.manager.name=AiravataPostWM
-post.workflow.manager.monitoring.enabled=true
-post.workflow.manager.monitoring.host=airavata.host
-post.workflow.manager.monitoring.port=9094
pre.workflow.manager.loadbalance.clusters=false
pre.workflow.manager.name=AiravataPreWM
-pre.workflow.manager.monitoring.enabled=true
-pre.workflow.manager.monitoring.host=airavata.host
-pre.workflow.manager.monitoring.port=9093
# RabbitMQ Settings
rabbitmq.broker.url=amqp://guest:[email protected]:5672/develop
diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/__init__.py
b/dev-tools/airavata-python-sdk/airavata_sdk/__init__.py
index 0a559918e8..7b858662cc 100644
--- a/dev-tools/airavata-python-sdk/airavata_sdk/__init__.py
+++ b/dev-tools/airavata-python-sdk/airavata_sdk/__init__.py
@@ -94,54 +94,6 @@ class Settings:
def FILE_SVC_URL(self):
return str(os.getenv("FILE_SVC_URL",
f"http://{self.API_SERVER_HOSTNAME}:8050";))
- # ------------------------------------------------------------
- # Profile Service Connection Settings
- # ------------------------------------------------------------
-
- @property
- def PROFILE_SERVICE_HOST(self):
- return str(os.getenv("PROFILE_SERVICE_HOST", self.API_SERVER_HOSTNAME))
-
- @property
- def PROFILE_SERVICE_PORT(self):
- return int(os.getenv("PROFILE_SERVICE_PORT", 8962))
-
- @property
- def PROFILE_SERVICE_SECURE(self):
- return bool(os.getenv("PROFILE_SERVICE_SECURE", False))
-
- # ------------------------------------------------------------
- # Sharing Service Connection Settings
- # ------------------------------------------------------------
-
- @property
- def SHARING_API_HOST(self):
- return str(os.getenv("SHARING_API_HOST", self.API_SERVER_HOSTNAME))
-
- @property
- def SHARING_API_PORT(self):
- return int(os.getenv("SHARING_API_PORT", 7878))
-
- @property
- def SHARING_API_SECURE(self):
- return bool(os.getenv("SHARING_API_SECURE", False))
-
- # ------------------------------------------------------------
- # Credential Store Connection Settings
- # ------------------------------------------------------------
-
- @property
- def CREDENTIAL_STORE_API_HOST(self):
- return str(os.getenv("CREDENTIAL_STORE_API_HOST",
self.API_SERVER_HOSTNAME))
-
- @property
- def CREDENTIAL_STORE_API_PORT(self):
- return int(os.getenv("CREDENTIAL_STORE_API_PORT", 8960))
-
- @property
- def CREDENTIAL_STORE_API_SECURE(self):
- return bool(os.getenv("CREDENTIAL_STORE_API_SECURE", False))
-
# ------------------------------------------------------------
# Gateway Settings
# ------------------------------------------------------------
diff --git
a/dev-tools/airavata-python-sdk/airavata_sdk/clients/credential_store_client.py
b/dev-tools/airavata-python-sdk/airavata_sdk/clients/credential_store_client.py
index d0113b334b..f3d2c854c0 100644
---
a/dev-tools/airavata-python-sdk/airavata_sdk/clients/credential_store_client.py
+++
b/dev-tools/airavata-python-sdk/airavata_sdk/clients/credential_store_client.py
@@ -27,9 +27,9 @@ class CredentialStoreClient(object):
def __init__(self):
self.settings = Settings()
self.client = utils.initialize_credential_store_client(
- self.settings.CREDENTIAL_STORE_API_HOST,
- self.settings.CREDENTIAL_STORE_API_PORT,
- self.settings.CREDENTIAL_STORE_API_SECURE,
+ self.settings.API_SERVER_HOSTNAME,
+ self.settings.API_SERVER_PORT,
+ self.settings.API_SERVER_SECURE,
)
# expose the needed functions
self.get_SSH_credential = self.client.getSSHCredential
diff --git
a/dev-tools/airavata-python-sdk/airavata_sdk/clients/group_manager_client.py
b/dev-tools/airavata-python-sdk/airavata_sdk/clients/group_manager_client.py
index 77db3dc639..f5b581a371 100644
--- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/group_manager_client.py
+++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/group_manager_client.py
@@ -36,9 +36,9 @@ class GroupManagerClient(object):
def __init__(self):
self.settings = Settings()
self.client = utils.initialize_group_manager_client(
- self.settings.PROFILE_SERVICE_HOST,
- self.settings.PROFILE_SERVICE_PORT,
- self.settings.PROFILE_SERVICE_SECURE,
+ self.settings.API_SERVER_HOSTNAME,
+ self.settings.API_SERVER_PORT,
+ self.settings.API_SERVER_SECURE,
)
# expose the needed functions
self.get_api_version = self.client.getAPIVersion
diff --git
a/dev-tools/airavata-python-sdk/airavata_sdk/clients/iam_admin_client.py
b/dev-tools/airavata-python-sdk/airavata_sdk/clients/iam_admin_client.py
index 45bb779cde..561a0bba77 100644
--- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/iam_admin_client.py
+++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/iam_admin_client.py
@@ -38,9 +38,9 @@ class IAMAdminClient(object):
def __init__(self):
self.settings = Settings()
self.client = utils.initialize_iam_admin_client(
- self.settings.PROFILE_SERVICE_HOST,
- self.settings.PROFILE_SERVICE_PORT,
- self.settings.PROFILE_SERVICE_SECURE,
+ self.settings.API_SERVER_HOSTNAME,
+ self.settings.API_SERVER_PORT,
+ self.settings.API_SERVER_SECURE,
)
# expose the needed functions
self.set_up_gateway = self.client.setUpGateway
diff --git
a/dev-tools/airavata-python-sdk/airavata_sdk/clients/sharing_registry_client.py
b/dev-tools/airavata-python-sdk/airavata_sdk/clients/sharing_registry_client.py
index 285e9342c3..e22c5534fa 100644
---
a/dev-tools/airavata-python-sdk/airavata_sdk/clients/sharing_registry_client.py
+++
b/dev-tools/airavata-python-sdk/airavata_sdk/clients/sharing_registry_client.py
@@ -36,9 +36,9 @@ class SharingRegistryClient(object):
def __init__(self):
self.settings = Settings()
self.client = utils.initialize_sharing_registry_client(
- self.settings.SHARING_API_HOST,
- self.settings.SHARING_API_PORT,
- self.settings.SHARING_API_SECURE,
+ self.settings.API_SERVER_HOSTNAME,
+ self.settings.API_SERVER_PORT,
+ self.settings.API_SERVER_SECURE,
)
# expose the needed functions
self.create_domain = self.client.createDomain
diff --git
a/dev-tools/airavata-python-sdk/airavata_sdk/clients/tenant_profile_client.py
b/dev-tools/airavata-python-sdk/airavata_sdk/clients/tenant_profile_client.py
index 71b2916183..95bc6735bc 100644
---
a/dev-tools/airavata-python-sdk/airavata_sdk/clients/tenant_profile_client.py
+++
b/dev-tools/airavata-python-sdk/airavata_sdk/clients/tenant_profile_client.py
@@ -36,9 +36,9 @@ class TenantProfileClient(object):
def __init__(self):
self.settings = Settings()
self.client = utils.initialize_tenant_profile_client(
- self.settings.PROFILE_SERVICE_HOST,
- self.settings.PROFILE_SERVICE_PORT,
- self.settings.PROFILE_SERVICE_SECURE,
+ self.settings.API_SERVER_HOSTNAME,
+ self.settings.API_SERVER_PORT,
+ self.settings.API_SERVER_SECURE,
)
# expose the needed functions
self.add_gateway = self.client.addGateway
diff --git
a/dev-tools/airavata-python-sdk/airavata_sdk/clients/user_profile_client.py
b/dev-tools/airavata-python-sdk/airavata_sdk/clients/user_profile_client.py
index 75adae16df..542132000c 100644
--- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/user_profile_client.py
+++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/user_profile_client.py
@@ -36,9 +36,9 @@ class UserProfileClient(object):
def __init__(self):
self.settings = Settings()
self.client = utils.initialize_user_profile_client(
- self.settings.PROFILE_SERVICE_HOST,
- self.settings.PROFILE_SERVICE_PORT,
- self.settings.PROFILE_SERVICE_SECURE,
+ self.settings.API_SERVER_HOSTNAME,
+ self.settings.API_SERVER_PORT,
+ self.settings.API_SERVER_SECURE,
)
# expose the needed functions
self.get_api_version = self.client.getAPIVersion
diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/transport/utils.py
b/dev-tools/airavata-python-sdk/airavata_sdk/transport/utils.py
index 3954f60194..4df0182edf 100644
--- a/dev-tools/airavata-python-sdk/airavata_sdk/transport/utils.py
+++ b/dev-tools/airavata-python-sdk/airavata_sdk/transport/utils.py
@@ -131,52 +131,52 @@ def initialize_api_client_pool(
port=settings.API_SERVER_PORT,
secure=settings.API_SERVER_SECURE,
) -> Airavata.Client:
- return ThriftClient(Airavata.Client, host, port, secure).client
+ return ThriftClient(Airavata.Client, host, port, secure,
"AiravataAPI").client
def initialize_group_manager_client(
- host=settings.PROFILE_SERVICE_HOST,
- port=settings.PROFILE_SERVICE_PORT,
- secure=settings.PROFILE_SERVICE_SECURE,
+ host=settings.API_SERVER_HOSTNAME,
+ port=settings.API_SERVER_PORT,
+ secure=settings.API_SERVER_SECURE,
) -> GroupManagerService.Client:
return ThriftClient(GroupManagerService.Client, host, port, secure,
GROUP_MANAGER_CPI_NAME).client
def initialize_iam_admin_client(
- host=settings.PROFILE_SERVICE_HOST,
- port=settings.PROFILE_SERVICE_PORT,
- secure=settings.PROFILE_SERVICE_SECURE,
+ host=settings.API_SERVER_HOSTNAME,
+ port=settings.API_SERVER_PORT,
+ secure=settings.API_SERVER_SECURE,
) -> IamAdminServices.Client:
return ThriftClient(IamAdminServices.Client, host, port, secure,
IAM_ADMIN_SERVICES_CPI_NAME).client
def initialize_tenant_profile_client(
- host=settings.PROFILE_SERVICE_HOST,
- port=settings.PROFILE_SERVICE_PORT,
- secure=settings.PROFILE_SERVICE_SECURE,
+ host=settings.API_SERVER_HOSTNAME,
+ port=settings.API_SERVER_PORT,
+ secure=settings.API_SERVER_SECURE,
) -> TenantProfileService.Client:
return ThriftClient(TenantProfileService.Client, host, port, secure,
TENANT_PROFILE_CPI_NAME).client
def initialize_user_profile_client(
- host=settings.PROFILE_SERVICE_HOST,
- port=settings.PROFILE_SERVICE_PORT,
- secure=settings.PROFILE_SERVICE_SECURE,
+ host=settings.API_SERVER_HOSTNAME,
+ port=settings.API_SERVER_PORT,
+ secure=settings.API_SERVER_SECURE,
) -> UserProfileService.Client:
return ThriftClient(UserProfileService.Client, host, port, secure,
USER_PROFILE_CPI_NAME).client
def initialize_sharing_registry_client(
- host=settings.SHARING_API_HOST,
- port=settings.SHARING_API_PORT,
- secure=settings.SHARING_API_SECURE,
+ host=settings.API_SERVER_HOSTNAME,
+ port=settings.API_SERVER_PORT,
+ secure=settings.API_SERVER_SECURE,
) -> SharingRegistryService.Client:
- return ThriftClient(SharingRegistryService.Client, host, port, secure).client
+ return ThriftClient(SharingRegistryService.Client, host, port, secure,
"SharingRegistry").client
def initialize_credential_store_client(
- host=settings.CREDENTIAL_STORE_API_HOST,
- port=settings.CREDENTIAL_STORE_API_PORT,
- secure=settings.CREDENTIAL_STORE_API_SECURE,
+ host=settings.API_SERVER_HOSTNAME,
+ port=settings.API_SERVER_PORT,
+ secure=settings.API_SERVER_SECURE,
) -> CredentialStoreService.Client:
- return ThriftClient(CredentialStoreService.Client, host, port, secure).client
+ return ThriftClient(CredentialStoreService.Client, host, port, secure,
"CredentialStore").client
