This is an automated email from the ASF dual-hosted git repository. yasith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata.git
commit 8850037f67e61ac4760abbf77eb29b6b7b120585 Author: yasithdev <[email protected]> AuthorDate: Mon Jul 21 18:58:09 2025 -0500 update python SDK for batch submissions, pull all settings into one place --- .../airavata_auth/device_auth.py | 30 +++-- .../airavata_experiments/airavata.py | 51 +++----- .../airavata_experiments/base.py | 2 +- .../airavata_experiments/plan.py | 13 +- .../airavata_experiments/runtime.py | 15 +-- .../airavata_experiments/scripter.py | 16 +++ .../airavata_experiments/task.py | 1 + .../airavata_jupyter_magic/__init__.py | 78 +++++++---- .../airavata-python-sdk/airavata_sdk/__init__.py | 145 +++++++++++++++++++++ .../airavata_sdk/clients/api_server_client.py | 25 +--- .../clients/credential_store_client.py | 18 +-- .../airavata_sdk/clients/group_manager_client.py | 17 +-- .../airavata_sdk/clients/iam_admin_client.py | 15 +-- .../airavata_sdk/clients/keycloak_token_fetcher.py | 64 +++++---- .../clients/sharing_registry_client.py | 18 +-- .../airavata_sdk/clients/tenant_profile_client.py | 18 +-- .../airavata_sdk/clients/user_profile_client.py | 18 +-- .../clients/utils/api_server_client_util.py | 6 +- .../clients/utils/data_model_creation_util.py | 7 +- .../clients/utils/experiment_handler_util.py | 95 +++++++------- .../samples/api_server_client_samples.py | 5 - .../samples/create_launch_echo_experiment.py | 26 ++-- .../samples/create_launch_gaussian_experiment.py | 10 +- .../samples/group_manager_client_samples.py | 5 - .../samples/iam_admin_client_samples.py | 4 - .../airavata_sdk/samples/metadata_fetcher.py | 6 +- .../samples/sharing_registry_client_samples.py | 5 - .../samples/tenant_profile_client_samples.py | 2 +- .../samples/user_profile_client_samples.py | 4 - .../airavata_sdk/transport/settings.py | 111 ---------------- .../airavata_sdk/transport/utils.py | 63 ++++----- 31 files changed, 427 insertions(+), 466 deletions(-) diff --git a/dev-tools/airavata-python-sdk/airavata_auth/device_auth.py b/dev-tools/airavata-python-sdk/airavata_auth/device_auth.py index b4070842c7..3037ab1702 100644 --- a/dev-tools/airavata-python-sdk/airavata_auth/device_auth.py +++ b/dev-tools/airavata-python-sdk/airavata_auth/device_auth.py @@ -4,18 +4,16 @@ import time import requests from rich.console import Console +from airavata_sdk import Settings + # Load environment variables from .env file + class AuthContext: - - client_id: str = "cybershuttle-agent" - realm: str = "default" - auth_server_url: str = "https://auth.cybershuttle.org"; - api_host: str = "https://api.gateway.cybershuttle.org"; - file_server_url: str = "http://api.gateway.cybershuttle.org:8050"; def __init__(self): - if not AuthContext.client_id or not AuthContext.realm or not AuthContext.auth_server_url: + self.settings = Settings() + if not self.settings.AUTH_CLIENT_ID or not self.settings.AUTH_REALM or not self.settings.AUTH_SERVER_URL: raise ValueError("Missing required environment variables for client ID, realm, or auth server URL") self.device_code = None @@ -24,11 +22,13 @@ class AuthContext: def login(self): # Step 1: Request device and user code - auth_device_url = f"{AuthContext.auth_server_url}/realms/{AuthContext.realm}/protocol/openid-connect/auth/device" - response = requests.post(auth_device_url, data={"client_id": AuthContext.client_id, "scope": "openid"}) + auth_device_url = f"{self.settings.AUTH_SERVER_URL}/realms/{self.settings.AUTH_REALM}/protocol/openid-connect/auth/device" + response = requests.post(auth_device_url, data={ + "client_id": self.settings.AUTH_CLIENT_ID, "scope": "openid"}) if response.status_code != 200: - print(f"Error in authentication request: {response.status_code} - {response.text}", flush=True) + print( + f"Error in authentication request: {response.status_code} - {response.text}", flush=True) return data = response.json() @@ -40,12 +40,12 @@ class AuthContext: def poll_for_token(self, url): assert self.interval is not None - token_url = f"{AuthContext.auth_server_url}/realms/{AuthContext.realm}/protocol/openid-connect/token" + token_url = f"{self.settings.AUTH_SERVER_URL}/realms/{self.settings.AUTH_REALM}/protocol/openid-connect/token" counter = 0 with self.console.status(f"Authenticate via link: [link={url}]{url}[/link]", refresh_per_second=1) as status: while True: response = requests.post(token_url, data={ - "client_id": AuthContext.client_id, + "client_id": self.settings.AUTH_CLIENT_ID, "grant_type": "urn:ietf:params:oauth:grant-type:device_code", "device_code": self.device_code }) @@ -57,9 +57,11 @@ class AuthContext: break elif response.status_code == 400 and response.json().get("error") == "authorization_pending": counter += 1 - status.update(f"Authenticate via link: [link={url}]{url}[/link] ({counter})") + status.update( + f"Authenticate via link: [link={url}]{url}[/link] ({counter})") else: - print(f"Error during authentication: {response.status_code} - {response.text}") + print( + f"Error during authentication: {response.status_code} - {response.text}") break time.sleep(self.interval) status.stop() diff --git a/dev-tools/airavata-python-sdk/airavata_experiments/airavata.py b/dev-tools/airavata-python-sdk/airavata_experiments/airavata.py index b3308f5995..f8f5941bb7 100644 --- a/dev-tools/airavata-python-sdk/airavata_experiments/airavata.py +++ b/dev-tools/airavata-python-sdk/airavata_experiments/airavata.py @@ -18,7 +18,9 @@ import logging from pathlib import Path from typing import Literal, NamedTuple -from airavata_sdk.transport.settings import APIServerSettings +from airavata_sdk import Settings +from airavata_sdk.clients.api_server_client import APIServerClient + from .sftp import SFTPConnector import time import warnings @@ -33,7 +35,6 @@ from airavata.model.security.ttypes import AuthzToken from airavata.model.experiment.ttypes import ExperimentModel, ExperimentType, UserConfigurationDataModel from airavata.model.scheduling.ttypes import ComputationalResourceSchedulingModel from airavata.model.data.replica.ttypes import DataProductModel, DataProductType, DataReplicaLocationModel, ReplicaLocationCategory -from airavata_sdk.clients.api_server_client import APIServerClient warnings.filterwarnings("ignore", category=DeprecationWarning) logger = logging.getLogger("airavata_sdk.clients") @@ -48,29 +49,6 @@ LaunchState = NamedTuple("LaunchState", [ ("sr_host", str), ]) -class Settings: - - def __init__(self, config_path: str) -> None: - - import configparser - config = configparser.ConfigParser() - config.read(config_path) - - # api server client settings - self.API_SERVER_HOST = config.get('APIServer', 'API_HOST') - self.API_SERVER_PORT = config.getint('APIServer', 'API_PORT') - self.API_SERVER_SECURE = config.getboolean('APIServer', 'API_SECURE') - self.CONNECTION_SVC_URL = config.get('APIServer', 'CONNECTION_SVC_URL') - self.FILEMGR_SVC_URL = config.get('APIServer', 'FILEMGR_SVC_URL') - - # gateway settings - self.GATEWAY_ID = config.get('Gateway', 'GATEWAY_ID') - self.GATEWAY_URL = config.get('Gateway', 'GATEWAY_URL') - self.GATEWAY_DATA_STORE_DIR = config.get('Gateway', 'GATEWAY_DATA_STORE_DIR') - self.STORAGE_RESOURCE_HOST = config.get('Gateway', 'STORAGE_RESOURCE_HOST') - self.SFTP_PORT = config.get('Gateway', 'SFTP_PORT') - - class AiravataOperator: def register_input_file( @@ -158,13 +136,11 @@ class AiravataOperator: return experiment_model - def __init__(self, access_token: str, config_file: str = "settings.ini"): + def __init__(self, access_token: str): # store variables + self.settings = Settings() self.access_token = access_token - self.settings = Settings(config_file) - # load api server settings and create client - api_server_settings = APIServerSettings(config_file) - self.api_server_client = APIServerClient(api_server_settings=api_server_settings) + self.api_server_client = APIServerClient() # load gateway settings gateway_id = self.default_gateway_id() self.airavata_token = self.__airavata_token__(self.access_token, gateway_id) @@ -182,10 +158,10 @@ class AiravataOperator: return self.settings.STORAGE_RESOURCE_HOST def connection_svc_url(self): - return self.settings.CONNECTION_SVC_URL + return f"{self.settings.API_SERVER_URL}/api/v1" def filemgr_svc_url(self): - return self.settings.FILEMGR_SVC_URL + return self.settings.FILE_SVC_URL def __airavata_token__(self, access_token: str, gateway_id: str): """ @@ -658,12 +634,19 @@ class AiravataOperator: self.upload_files(None, None, storage.hostName, files_to_upload, exp_dir) # create experiment - ex_id = self.api_server_client.create_experiment(self.airavata_token, gateway_id, experiment) + try: + ex_id = self.api_server_client.create_experiment(self.airavata_token, gateway_id, experiment) + except Exception as e: + raise Exception(f"[AV] Failed to create experiment: {repr(e)}").with_traceback(e.__traceback__) + ex_id = str(ex_id) print(f"[AV] Experiment {experiment_name} CREATED with id: {ex_id}") # launch experiment - self.api_server_client.launch_experiment(self.airavata_token, ex_id, gateway_id) + try: + self.api_server_client.launch_experiment(self.airavata_token, ex_id, gateway_id) + except Exception as e: + raise Exception(f"[AV] Failed to launch experiment: {repr(e)}").with_traceback(e.__traceback__) print(f"[AV] Experiment {experiment_name} STARTED with id: {ex_id}") # wait until experiment begins, then get process id diff --git a/dev-tools/airavata-python-sdk/airavata_experiments/base.py b/dev-tools/airavata-python-sdk/airavata_experiments/base.py index 56104ca5c8..e9ad36b68e 100644 --- a/dev-tools/airavata-python-sdk/airavata_experiments/base.py +++ b/dev-tools/airavata-python-sdk/airavata_experiments/base.py @@ -66,7 +66,7 @@ class Experiment(Generic[T], abc.ABC): application: T inputs: dict[str, Any] input_mapping: dict[str, tuple[Any, str]] - resource: Runtime = Runtime.default() + resource: Runtime = Runtime.Local() tasks: list[Task] = [] def __init__(self, name: str, application: T): diff --git a/dev-tools/airavata-python-sdk/airavata_experiments/plan.py b/dev-tools/airavata-python-sdk/airavata_experiments/plan.py index 800089a8f9..f231e2583d 100644 --- a/dev-tools/airavata-python-sdk/airavata_experiments/plan.py +++ b/dev-tools/airavata-python-sdk/airavata_experiments/plan.py @@ -28,7 +28,7 @@ import uuid from .airavata import AiravataOperator -from airavata_auth.device_auth import AuthContext +from airavata_sdk import Settings class Plan(pydantic.BaseModel): @@ -128,6 +128,7 @@ class Plan(pydantic.BaseModel): json.dump(self.model_dump(), f, indent=2) def save(self) -> None: + settings = Settings() av = AiravataOperator(os.environ['CS_ACCESS_TOKEN']) az = av.__airavata_token__(av.access_token, av.default_gateway_id()) assert az.accessToken is not None @@ -140,10 +141,10 @@ class Plan(pydantic.BaseModel): import requests if self.id is None: self.id = str(uuid.uuid4()) - response = requests.post(f"{AuthContext.api_host}/api/v1/plan", headers=headers, json=self.model_dump()) + response = requests.post(f"{settings.API_SERVER_URL}/api/v1/plan", headers=headers, json=self.model_dump()) print(f"Plan saved: {self.id}") else: - response = requests.put(f"{AuthContext.api_host}/api/v1/plan/{self.id}", headers=headers, json=self.model_dump()) + response = requests.put(f"{settings.API_SERVER_URL}/api/v1/plan/{self.id}", headers=headers, json=self.model_dump()) print(f"Plan updated: {self.id}") if response.status_code == 200: @@ -159,6 +160,7 @@ def load_json(filename: str) -> Plan: return Plan(**model) def load(id: str | None) -> Plan: + settings = Settings() assert id is not None av = AiravataOperator(os.environ['CS_ACCESS_TOKEN']) az = av.__airavata_token__(av.access_token, av.default_gateway_id()) @@ -170,7 +172,7 @@ def load(id: str | None) -> Plan: 'X-Claims': json.dumps(az.claimsMap) } import requests - response = requests.get(f"{AuthContext.api_host}/api/v1/plan/{id}", headers=headers) + response = requests.get(f"{settings.API_SERVER_URL}/api/v1/plan/{id}", headers=headers) if response.status_code == 200: body = response.json() @@ -180,6 +182,7 @@ def load(id: str | None) -> Plan: raise Exception(response) def query() -> list[Plan]: + settings = Settings() av = AiravataOperator(os.environ['CS_ACCESS_TOKEN']) az = av.__airavata_token__(av.access_token, av.default_gateway_id()) assert az.accessToken is not None @@ -190,7 +193,7 @@ def query() -> list[Plan]: 'X-Claims': json.dumps(az.claimsMap) } import requests - response = requests.get(f"{AuthContext.api_host}/api/v1/plan/user", headers=headers) + response = requests.get(f"{settings.API_SERVER_URL}/api/v1/plan/user", headers=headers) if response.status_code == 200: items: list = response.json() diff --git a/dev-tools/airavata-python-sdk/airavata_experiments/runtime.py b/dev-tools/airavata-python-sdk/airavata_experiments/runtime.py index ed92a1f9a2..e843135843 100644 --- a/dev-tools/airavata-python-sdk/airavata_experiments/runtime.py +++ b/dev-tools/airavata-python-sdk/airavata_experiments/runtime.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # + from __future__ import annotations import abc from typing import Any @@ -56,10 +57,6 @@ class Runtime(abc.ABC, pydantic.BaseModel): def __str__(self) -> str: return f"{self.__class__.__name__}(args={self.args})" - @staticmethod - def default(): - return Remote.default() - @staticmethod def create(id: str, args: dict[str, Any]) -> Runtime: if id == "mock": @@ -116,10 +113,6 @@ class Mock(Runtime): def cat(self, file: str, task: Task) -> bytes: return b"" - @staticmethod - def default(): - return Mock() - class Remote(Runtime): @@ -163,7 +156,7 @@ class Remote(Runtime): task.sr_host = launch_state.sr_host print(f"[Remote] Experiment Launched: id={task.ref}") except Exception as e: - print(f"[Remote] Failed to launch experiment: {e}") + print(f"[Remote] Failed to launch experiment: {repr(e)}") raise e def execute_py(self, libraries: list[str], code: str, task: Task) -> None: @@ -245,10 +238,6 @@ class Remote(Runtime): content = av.cat_file(task.pid, task.agent_ref, task.sr_host, file, task.workdir) return content - @staticmethod - def default(): - return list_runtimes(cluster="login.expanse.sdsc.edu", category="gpu").pop() - def list_runtimes( cluster: str | None = None, diff --git a/dev-tools/airavata-python-sdk/airavata_experiments/scripter.py b/dev-tools/airavata-python-sdk/airavata_experiments/scripter.py index 76aa874ddb..38bf9b3f8e 100644 --- a/dev-tools/airavata-python-sdk/airavata_experiments/scripter.py +++ b/dev-tools/airavata-python-sdk/airavata_experiments/scripter.py @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + import inspect import ast import textwrap diff --git a/dev-tools/airavata-python-sdk/airavata_experiments/task.py b/dev-tools/airavata-python-sdk/airavata_experiments/task.py index 285c959c66..bcda796518 100644 --- a/dev-tools/airavata-python-sdk/airavata_experiments/task.py +++ b/dev-tools/airavata-python-sdk/airavata_experiments/task.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # + from __future__ import annotations from typing import Any import pydantic diff --git a/dev-tools/airavata-python-sdk/airavata_jupyter_magic/__init__.py b/dev-tools/airavata-python-sdk/airavata_jupyter_magic/__init__.py index 7944043df4..1528304462 100644 --- a/dev-tools/airavata-python-sdk/airavata_jupyter_magic/__init__.py +++ b/dev-tools/airavata-python-sdk/airavata_jupyter_magic/__init__.py @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + import base64 import binascii import importlib.metadata @@ -27,6 +43,7 @@ from rich.live import Live from jupyter_client.blocking.client import BlockingKernelClient from airavata_auth.device_auth import AuthContext +from airavata_sdk import Settings # ======================================================================== # DATA STRUCTURES @@ -149,6 +166,7 @@ def is_runtime_ready(access_token: str, rt: RuntimeInfo, rt_name: str): @returns: True if ready, False otherwise """ + settings = Settings() # first, check the experiment state headers = generate_headers(access_token, rt.gateway_id) @@ -168,7 +186,7 @@ def is_runtime_ready(access_token: str, rt: RuntimeInfo, rt_name: str): raise InvalidStateError(msg) # third, check the state of agent - url = f"{AuthContext.api_host}/api/v1/agent/{rt.agentId}" + url = f"{settings.API_SERVER_URL}/api/v1/agent/{rt.agentId}" res = requests.get(url) code = res.status_code astate = "CREATING_WORKSPACE" @@ -191,8 +209,9 @@ def execute_shell_async(access_token: str, rt_name: str, arguments: list[str]) - rt = state.all_runtimes.get(rt_name, None) if rt is None: raise Exception(f"Runtime {rt_name} not found.") - - url = f"{AuthContext.api_host}/api/v1/agent/execute/asyncshell" + + settings = Settings() + url = f"{settings.API_SERVER_URL}/api/v1/agent/execute/asyncshell" headers = generate_headers(access_token, rt_name) res = requests.post(url, headers=headers, data=json.dumps({ "agentId": rt.agentId, @@ -210,7 +229,7 @@ def execute_shell_async(access_token: str, rt_name: str, arguments: list[str]) - # Check if the request was successful while True: - url = f"{AuthContext.api_host}/api/v1/agent/execute/asyncshell/{executionId}" + url = f"{settings.API_SERVER_URL}/api/v1/agent/execute/asyncshell/{executionId}" res = requests.get(url, headers={'Accept': 'application/json'}) data = res.json() @@ -235,7 +254,8 @@ def get_hostname(access_token: str, rt_name: str) -> str | None: if rt is None: raise Exception(f"Runtime {rt_name} not found.") - url = f"{AuthContext.api_host}/api/v1/agent/execute/shell" + settings = Settings() + url = f"{settings.API_SERVER_URL}/api/v1/agent/execute/shell" headers = generate_headers(access_token, rt_name) res = requests.post(url, headers=headers, data=json.dumps({ "agentId": rt.agentId, @@ -251,7 +271,7 @@ def get_hostname(access_token: str, rt_name: str) -> str | None: return print(f"Failed to get hostname for runtime={rt_name}") while True: - url = f"{AuthContext.api_host}/api/v1/agent/execute/shell/{executionId}" + url = f"{settings.API_SERVER_URL}/api/v1/agent/execute/shell/{executionId}" res = requests.get(url, headers={'Accept': 'application/json'}) data = res.json() if data.get('executed'): @@ -273,7 +293,8 @@ def open_tunnel(access_token: str, rt_name: str, rt_hostname: str, rt_port: int) if rt is None: raise Exception(f"Runtime {rt_name} not found.") - url = f"{AuthContext.api_host}/api/v1/agent/setup/tunnel" + settings = Settings() + url = f"{settings.API_SERVER_URL}/api/v1/agent/setup/tunnel" headers = generate_headers(access_token, rt_name) res = requests.post(url, headers=headers, data=json.dumps({ "agentId": rt.agentId, @@ -289,7 +310,7 @@ def open_tunnel(access_token: str, rt_name: str, rt_hostname: str, rt_port: int) return print(f"Failed to setup tunnel for runtime={rt_name}") while True: - url = f"{AuthContext.api_host}/api/v1/agent/setup/tunnel/{executionId}" + url = f"{settings.API_SERVER_URL}/api/v1/agent/setup/tunnel/{executionId}" res = requests.get(url, headers={'Accept': 'application/json'}) data = res.json() if data.get('status') == "OK": @@ -328,7 +349,8 @@ def terminate_shell_async(access_token: str, rt_name: str, process_id: str, proc if rt is None: raise Exception(f"Runtime {rt_name} not found.") - url = f"{AuthContext.api_host}/api/v1/agent/terminate/asyncshell" + settings = Settings() + url = f"{settings.API_SERVER_URL}/api/v1/agent/terminate/asyncshell" headers = generate_headers(access_token, rt_name) res = requests.post(url, headers=headers, data=json.dumps({ "agentId": rt.agentId, @@ -356,7 +378,8 @@ def get_experiment_state(experiment_id: str, headers: dict) -> tuple[ProcessStat @returns: the experiment state """ - url = f"{AuthContext.api_host}/api/v1/exp/{experiment_id}" + settings = Settings() + url = f"{settings.API_SERVER_URL}/api/v1/exp/{experiment_id}" res = requests.get(url, headers=headers) code = res.status_code if code != 200: @@ -380,7 +403,8 @@ def get_process_state(experiment_id: str, headers: dict) -> tuple[str, ProcessSt @returns: process id and state """ - url = f"{AuthContext.api_host}/api/v1/exp/{experiment_id}/process" + settings = Settings() + url = f"{settings.API_SERVER_URL}/api/v1/exp/{experiment_id}/process" pid, pstate = "", ProcessState.CREATED while not pid: res = requests.get(url, headers=headers) @@ -468,7 +492,8 @@ def submit_agent_job( """ # URL to which the POST request will be sent - url = f"{AuthContext.api_host}/api/v1/exp/launch" + settings = Settings() + url = f"{settings.API_SERVER_URL}/api/v1/exp/launch" # data from file min_cpu: int = 1 @@ -574,8 +599,9 @@ def fetch_logs(rt_name: str) -> tuple[str, str]: """ pid = state.all_runtimes[rt_name].processId - stdout_res = requests.get(f"{AuthContext.file_server_url}/download/live/{pid}/AiravataAgent.stdout") - stderr_res = requests.get(f"{AuthContext.file_server_url}/download/live/{pid}/AiravataAgent.stderr") + settings = Settings() + stdout_res = requests.get(f"{settings.FILE_SVC_URL}/download/live/{pid}/AiravataAgent.stdout") + stderr_res = requests.get(f"{settings.FILE_SVC_URL}/download/live/{pid}/AiravataAgent.stderr") stdout = "No STDOUT" if stdout_res.status_code != 200 else stdout_res.content.decode('utf-8').strip() stderr = "No STDERR" if stderr_res.status_code != 200 else stderr_res.content.decode('utf-8').strip() return stdout, stderr @@ -646,8 +672,8 @@ def restart_runtime_kernel(access_token: str, rt_name: str, env_name: str, runti @returns: None """ - - url = f"{AuthContext.api_host}/api/v1/agent/setup/restart" + settings = Settings() + url = f"{settings.API_SERVER_URL}/api/v1/agent/setup/restart" decode = jwt.decode(access_token, options={"verify_signature": False}) user_id = decode['preferred_username'] @@ -677,7 +703,7 @@ def restart_runtime_kernel(access_token: str, rt_name: str, env_name: str, runti # Check if the request was successful while True: - url = f"{AuthContext.api_host}/api/v1/agent/setup/restart/{executionId}" + url = f"{settings.API_SERVER_URL}/api/v1/agent/setup/restart/{executionId}" res = requests.get(url, headers={'Accept': 'application/json'}) data = res.json() if data.get('restarted'): @@ -696,8 +722,8 @@ def stop_agent_job(access_token: str, runtime_name: str, runtime: RuntimeInfo): @returns: None """ - - url = f"{AuthContext.api_host}/api/v1/exp/terminate/{runtime.experimentId}" + settings = Settings() + url = f"{settings.API_SERVER_URL}/api/v1/exp/terminate/{runtime.experimentId}" decode = jwt.decode(access_token, options={"verify_signature": False}) user_id = decode['preferred_username'] @@ -733,7 +759,8 @@ def run_on_runtime(rt_name: str, code_obj: str, result: ExecutionResult) -> bool result.error_in_exec = Exception(f"Runtime {rt_name} not found.") return False - url = f"{AuthContext.api_host}/api/v1/agent/execute/jupyter" + settings = Settings() + url = f"{settings.API_SERVER_URL}/api/v1/agent/execute/jupyter" data = { "agentId": rt.agentId, "envName": rt.envName, @@ -756,7 +783,7 @@ def run_on_runtime(rt_name: str, code_obj: str, result: ExecutionResult) -> bool return False while True: - url = f"{AuthContext.api_host}/api/v1/agent/execute/jupyter/{execution_id}" + url = f"{settings.API_SERVER_URL}/api/v1/agent/execute/jupyter/{execution_id}" response = requests.get(url, headers={'Accept': 'application/json'}) json_response = response.json() if json_response.get('executed'): @@ -914,7 +941,8 @@ def push_remote(local_path: str, remot_rt: str, remot_path: str) -> None: # upload file print(f"local:{local_path} --> {remot_rt}:{remot_path}...", end=" ", flush=True) pid = state.all_runtimes[remot_rt].processId - url = f"{AuthContext.file_server_url}/upload/live/{pid}/{remot_path}" + settings = Settings() + url = f"{settings.FILE_SVC_URL}/upload/live/{pid}/{remot_path}" with open(local_path, "rb") as file: files = {"file": file} response = requests.post(url, files=files) @@ -922,7 +950,8 @@ def push_remote(local_path: str, remot_rt: str, remot_path: str) -> None: def pull_remote_file(remot_rt: str, remot_fp: str, local_fp: str) -> None: pid = state.all_runtimes[remot_rt].processId - url = f"{AuthContext.file_server_url}/download/live/{pid}/{remot_fp}" + settings = Settings() + url = f"{settings.FILE_SVC_URL}/download/live/{pid}/{remot_fp}" print(f"GET {url}") response = requests.get(url) with open(local_fp, "wb") as file: @@ -942,7 +971,8 @@ def pull_remote(remot_rt: str, remot_path: str, local_path: Path, local_is_dir: if not state.all_runtimes.get(remot_rt, None): return print(MSG_NOT_INITIALIZED) pid = state.all_runtimes[remot_rt].processId - url = f"{AuthContext.file_server_url}/list/live/{pid}/{remot_path}" + settings = Settings() + url = f"{settings.FILE_SVC_URL}/list/live/{pid}/{remot_path}" print(f"GET {url}") response = requests.get(url) res = response.json() diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/__init__.py b/dev-tools/airavata-python-sdk/airavata_sdk/__init__.py index fe95886d5c..eb2b503233 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/__init__.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/__init__.py @@ -16,3 +16,148 @@ # specific language governing permissions and limitations # under the License. # + +import os + + +class Settings: + """ + Settings for the Airavata Python SDK + + """ + + # ------------------------------------------------------------ + # Keycloak Connection Settings + # ------------------------------------------------------------ + + @property + def AUTH_CLIENT_ID(self): + return str(os.getenv("AUTH_CLIENT_ID", "cybershuttle-agent")) + + @property + def AUTH_REALM(self): + return str(os.getenv("AUTH_REALM", "default")) + + @property + def AUTH_SERVER_URL(self): + return str(os.getenv("AUTH_SERVER_URL", "https://auth.cybershuttle.org";)) + + # ------------------------------------------------------------ + # Thrift Connection Settings + # ------------------------------------------------------------ + + @property + def THRIFT_CLIENT_POOL_KEEPALIVE(self): + return int(os.getenv("THRIFT_CLIENT_POOL_KEEPALIVE", 5)) + + @property + def VERIFY_SSL(self): + return bool(os.getenv("VERIFY_SSL", True)) + + # ------------------------------------------------------------ + # API Server Connection Settings + # ------------------------------------------------------------ + + @property + def API_SERVER_HOSTNAME(self): + return str(os.getenv("API_SERVER_HOSTNAME", "api.gateway.cybershuttle.org")) + + @property + def API_SERVER_PORT(self): + return int(os.getenv("API_SERVER_PORT", 9930)) + + @property + def API_SERVER_URL(self): + return str(os.getenv("API_SERVER_URL", f"https://{self.API_SERVER_HOSTNAME}";)) + + @property + def API_SERVER_SECURE(self): + return bool(os.getenv("API_SERVER_SECURE", True)) + + @property + def MONITOR_STATUS(self): + return bool(os.getenv("MONITOR_STATUS", False)) + + # ------------------------------------------------------------ + # File Service Connection Settings + # ------------------------------------------------------------ + + @property + def FILE_SVC_URL(self): + return str(os.getenv("FILE_SVC_URL", f"http://{self.API_SERVER_HOSTNAME}:8050";)) + + # ------------------------------------------------------------ + # Profile Service Connection Settings + # ------------------------------------------------------------ + + @property + def PROFILE_SERVICE_HOST(self): + return str(os.getenv("PROFILE_SERVICE_HOST", self.API_SERVER_HOSTNAME)) + + @property + def PROFILE_SERVICE_PORT(self): + return int(os.getenv("PROFILE_SERVICE_PORT", 8962)) + + @property + def PROFILE_SERVICE_SECURE(self): + return bool(os.getenv("PROFILE_SERVICE_SECURE", False)) + + # ------------------------------------------------------------ + # Sharing Service Connection Settings + # ------------------------------------------------------------ + + @property + def SHARING_API_HOST(self): + return str(os.getenv("SHARING_API_HOST", self.API_SERVER_HOSTNAME)) + + @property + def SHARING_API_PORT(self): + return int(os.getenv("SHARING_API_PORT", 7878)) + + @property + def SHARING_API_SECURE(self): + return bool(os.getenv("SHARING_API_SECURE", False)) + + # ------------------------------------------------------------ + # Credential Store Connection Settings + # ------------------------------------------------------------ + + @property + def CREDENTIAL_STORE_API_HOST(self): + return str(os.getenv("CREDENTIAL_STORE_API_HOST", self.API_SERVER_HOSTNAME)) + + @property + def CREDENTIAL_STORE_API_PORT(self): + return int(os.getenv("CREDENTIAL_STORE_API_PORT", 8960)) + + @property + def CREDENTIAL_STORE_API_SECURE(self): + return bool(os.getenv("CREDENTIAL_STORE_API_SECURE", False)) + + # ------------------------------------------------------------ + # Gateway Settings + # ------------------------------------------------------------ + + @property + def GATEWAY_ID(self): + return str(os.getenv("GATEWAY_ID", "default")) + + @property + def GATEWAY_URL(self): + return str(os.getenv("GATEWAY_URL", "https://gateway.cybershuttle.org";)) + + @property + def GATEWAY_DATA_STORE_DIR(self): + return str(os.getenv("GATEWAY_DATA_STORE_DIR", "/var/www/portals/gateway-user-data/cybershuttle")) + + # ------------------------------------------------------------ + # Storage Settings + # ------------------------------------------------------------ + + @property + def STORAGE_RESOURCE_HOST(self): + return str(os.getenv("STORAGE_RESOURCE_HOST", "cybershuttle.org")) + + @property + def SFTP_PORT(self): + return int(os.getenv("SFTP_PORT", 9000)) diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/clients/api_server_client.py b/dev-tools/airavata-python-sdk/airavata_sdk/clients/api_server_client.py index a54caff4a4..cf6fd2edaa 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/api_server_client.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/api_server_client.py @@ -14,27 +14,20 @@ # limitations under the License. # -import configparser import logging -from typing import Optional +from airavata_sdk import Settings from airavata_sdk.transport import utils -from airavata_sdk.transport.settings import APIServerSettings logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) +class APIServerClient: -class APIServerClient(object): - - def __init__(self, configuration_file_location: Optional[str] = None, api_server_settings: Optional[APIServerSettings] = None): - if api_server_settings is not None: - self.settings = api_server_settings - elif configuration_file_location is not None: - self.settings = APIServerSettings(configuration_file_location) - self._load_settings(configuration_file_location) + def __init__(self): + self.settings = Settings() self.client = utils.initialize_api_client_pool( - self.settings.API_SERVER_HOST, + self.settings.API_SERVER_HOSTNAME, self.settings.API_SERVER_PORT, self.settings.API_SERVER_SECURE, ) @@ -230,11 +223,3 @@ class APIServerClient(object): self.save_parsing_template = self.client.saveParsingTemplate self.remove_parsing_template = self.client.removeParsingTemplate self.list_all_parsing_templates = self.client.listAllParsingTemplates - - def _load_settings(self, configuration_file_location: Optional[str]): - if configuration_file_location is not None: - config = configparser.ConfigParser() - config.read(configuration_file_location) - self.settings.API_SERVER_HOST = config.get('APIServer', 'API_HOST') - self.settings.API_SERVER_PORT = config.getint('APIServer', 'API_PORT') - self.settings.API_SERVER_SECURE = config.getboolean('APIServer', 'API_SECURE') diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/clients/credential_store_client.py b/dev-tools/airavata-python-sdk/airavata_sdk/clients/credential_store_client.py index 3b9ddaeb7c..d0113b334b 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/credential_store_client.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/credential_store_client.py @@ -14,22 +14,18 @@ # limitations under the License. # -import configparser import logging -from typing import Optional from airavata_sdk.transport import utils -from airavata_sdk.transport.settings import CredentialStoreServerSettings +from airavata_sdk import Settings logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) - class CredentialStoreClient(object): - def __init__(self, configuration_file_location: Optional[str] = None): - self.settings = CredentialStoreServerSettings(configuration_file_location) - self._load_settings(configuration_file_location) + def __init__(self): + self.settings = Settings() self.client = utils.initialize_credential_store_client( self.settings.CREDENTIAL_STORE_API_HOST, self.settings.CREDENTIAL_STORE_API_PORT, @@ -37,11 +33,3 @@ class CredentialStoreClient(object): ) # expose the needed functions self.get_SSH_credential = self.client.getSSHCredential - - def _load_settings(self, configuration_file_location: Optional[str]): - if configuration_file_location is not None: - config = configparser.ConfigParser() - config.read(configuration_file_location) - self.settings.CREDENTIAL_STORE_API_HOST = config.get('CredentialStoreServer', 'CREDENTIAL_STORE_API_HOST') - self.settings.CREDENTIAL_STORE_API_PORT = config.getint('CredentialStoreServer', 'CREDENTIAL_STORE_API_PORT') - self.settings.CREDENTIAL_STORE_API_SECURE = config.getboolean('CredentialStoreServer', 'CREDENTIAL_STORE_API_SECURE') diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/clients/group_manager_client.py b/dev-tools/airavata-python-sdk/airavata_sdk/clients/group_manager_client.py index dba238bf4a..77db3dc639 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/group_manager_client.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/group_manager_client.py @@ -14,12 +14,10 @@ # limitations under the License. # -import configparser import logging -from typing import Optional from airavata_sdk.transport import utils -from airavata_sdk.transport.settings import ProfileServerSettings +from airavata_sdk import Settings logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -35,9 +33,8 @@ logger.addHandler(handler) class GroupManagerClient(object): - def __init__(self, configuration_file_location: Optional[str] = None): - self.settings = ProfileServerSettings(configuration_file_location) - self._load_settings(configuration_file_location) + def __init__(self): + self.settings = Settings() self.client = utils.initialize_group_manager_client( self.settings.PROFILE_SERVICE_HOST, self.settings.PROFILE_SERVICE_PORT, @@ -58,11 +55,3 @@ class GroupManagerClient(object): self.remove_group_admins = self.client.removeGroupAdmins self.has_admin_access = self.client.hasAdminAccess self.has_owner_access = self.client.hasOwnerAccess - - def _load_settings(self, configuration_file_location: Optional[str]): - if configuration_file_location is not None: - config = configparser.ConfigParser() - config.read(configuration_file_location) - self.settings.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') - self.settings.PROFILE_SERVICE_PORT = config.getint('ProfileServer', 'PROFILE_SERVICE_PORT') - self.settings.PROFILE_SERVICE_SECURE = config.getboolean('ProfileServer', 'PROFILE_SERVICE_SECURE') diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/clients/iam_admin_client.py b/dev-tools/airavata-python-sdk/airavata_sdk/clients/iam_admin_client.py index aeab7c01a1..45bb779cde 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/iam_admin_client.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/iam_admin_client.py @@ -19,7 +19,7 @@ import logging from typing import Optional from airavata_sdk.transport import utils -from airavata_sdk.transport.settings import ProfileServerSettings +from airavata_sdk import Settings logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -35,9 +35,8 @@ logger.addHandler(handler) class IAMAdminClient(object): - def __init__(self, configuration_file_location: Optional[str] = None): - self.settings = ProfileServerSettings(configuration_file_location) - self._load_settings(configuration_file_location) + def __init__(self): + self.settings = Settings() self.client = utils.initialize_iam_admin_client( self.settings.PROFILE_SERVICE_HOST, self.settings.PROFILE_SERVICE_PORT, @@ -59,11 +58,3 @@ class IAMAdminClient(object): self.add_role_to_user = self.client.addRoleToUser self.remove_role_from_user = self.client.removeRoleFromUser self.get_users_with_role = self.client.getUsersWithRole - - def _load_settings(self, configuration_file_location: Optional[str]): - if configuration_file_location is not None: - config = configparser.ConfigParser() - config.read(configuration_file_location) - self.settings.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') - self.settings.PROFILE_SERVICE_PORT = config.getint('ProfileServer', 'PROFILE_SERVICE_PORT') - self.settings.PROFILE_SERVICE_SECURE = config.getboolean('ProfileServer', 'PROFILE_SERVICE_SECURE') diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/clients/keycloak_token_fetcher.py b/dev-tools/airavata-python-sdk/airavata_sdk/clients/keycloak_token_fetcher.py index c403f75a09..8465d78dec 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/keycloak_token_fetcher.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/keycloak_token_fetcher.py @@ -14,31 +14,50 @@ # limitations under the License. # -import configparser import os -from typing import Optional from oauthlib.oauth2 import LegacyApplicationClient from requests_oauthlib import OAuth2Session from airavata.model.security.ttypes import AuthzToken -from airavata_sdk.transport.settings import KeycloakServerSettings +from airavata_sdk import Settings BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + class Authenticator(object): - def __init__(self, configuration_file_location: Optional[str] = None): - self.settings = KeycloakServerSettings(configuration_file_location) - self._load_settings(configuration_file_location) + def __init__(self): + self.settings = Settings() + + @property + def TOKEN_URL(self): + return f"{Settings.AUTH_SERVER_URL}/realms/{Settings.AUTH_REALM}/protocol/openid-connect/token" + + @property + def USER_INFO_URL(self): + return f"{Settings.AUTH_SERVER_URL}/realms/{Settings.AUTH_REALM}/protocol/openid-connect/userinfo" + + @property + def LOGIN_DESKTOP_URI(self): + return f"{Settings.AUTH_SERVER_URL}/realms/{Settings.AUTH_REALM}/protocol/openid-connect/auth" + + @property + def CLIENT_ID(self): + return "airavata" + + @property + def CLIENT_SECRET(self): + return "airavata" def get_token_and_user_info_password_flow(self, username: str, password: str, gateway_id: str): - client_id = self.settings.CLIENT_ID - client_secret = self.settings.CLIENT_SECRET - token_url = self.settings.TOKEN_URL + client_id = self.CLIENT_ID + client_secret = self.CLIENT_SECRET + token_url = self.TOKEN_URL # userinfo_url = self.keycloak_settings.USER_INFO_URL verify_ssl = self.settings.VERIFY_SSL - oauth2_session = OAuth2Session(client=LegacyApplicationClient(client_id=client_id)) + oauth2_session = OAuth2Session( + client=LegacyApplicationClient(client_id=client_id)) token = oauth2_session.fetch_token( token_url=token_url, username=username, @@ -61,12 +80,12 @@ class Authenticator(object): return AuthzToken(accessToken=token, claimsMap=claimsMap) def get_authorize_url(self, username: str, password: str, gateway_id: str): - client_id = self.settings.CLIENT_ID - client_secret = self.settings.CLIENT_SECRET - token_url = self.settings.TOKEN_URL - # userinfo_url = self.keycloak_settings.USER_INFO_URL + client_id = self.CLIENT_ID + client_secret = self.CLIENT_SECRET + token_url = self.TOKEN_URL verify_ssl = self.settings.VERIFY_SSL - oauth2_session = OAuth2Session(client=LegacyApplicationClient(client_id=client_id)) + oauth2_session = OAuth2Session( + client=LegacyApplicationClient(client_id=client_id)) token = oauth2_session.fetch_token( token_url=token_url, username=username, @@ -82,16 +101,5 @@ class Authenticator(object): return AuthzToken(accessToken=token['access_token'], claimsMap=claimsMap) def authenticate_with_auth_code(self): - print("Click on Login URI ", self.settings.LOGIN_DESKTOP_URI) - return self.settings.LOGIN_DESKTOP_URI - - def _load_settings(self, configuration_file_location: Optional[str]): - if configuration_file_location is not None: - config = configparser.ConfigParser() - config.read(configuration_file_location) - # self.keycloak_settings.KEYCLOAK_CA_CERTIFICATE = config.get("KeycloakServer", 'CERTIFICATE_FILE_PATH') - self.settings.CLIENT_ID = config.get('KeycloakServer', 'CLIENT_ID') - self.settings.CLIENT_SECRET = config.get('KeycloakServer', 'CLIENT_SECRET') - self.settings.TOKEN_URL = config.get('KeycloakServer', 'TOKEN_URL') - self.settings.USER_INFO_URL = config.get('KeycloakServer', 'USER_INFO_URL') - self.settings.VERIFY_SSL = config.getboolean('KeycloakServer', 'VERIFY_SSL') + print("Click on Login URI ", self.LOGIN_DESKTOP_URI) + return self.LOGIN_DESKTOP_URI diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/clients/sharing_registry_client.py b/dev-tools/airavata-python-sdk/airavata_sdk/clients/sharing_registry_client.py index e098fefcbc..285e9342c3 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/sharing_registry_client.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/sharing_registry_client.py @@ -14,12 +14,10 @@ # limitations under the License. # -import configparser import logging -from typing import Optional from airavata_sdk.transport import utils -from airavata_sdk.transport.settings import SharingServerSettings +from airavata_sdk import Settings logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -35,9 +33,8 @@ logger.addHandler(handler) class SharingRegistryClient(object): - def __init__(self, configuration_file_location: Optional[str] = None): - self.settings = SharingServerSettings(configuration_file_location) - self._load_settings(configuration_file_location) + def __init__(self): + self.settings = Settings() self.client = utils.initialize_sharing_registry_client( self.settings.SHARING_API_HOST, self.settings.SHARING_API_PORT, @@ -101,12 +98,3 @@ class SharingRegistryClient(object): self.share_entity_with_groups = self.client.shareEntityWithGroups self.revoke_entity_sharing_from_groups = self.client.revokeEntitySharingFromGroups self.user_has_access = self.client.userHasAccess - - - def _load_settings(self, configuration_file_location: Optional[str]): - if configuration_file_location is not None: - config = configparser.ConfigParser() - config.read(configuration_file_location) - self.settings.SHARING_API_HOST = config.get('SharingServer', 'SHARING_API_HOST') - self.settings.SHARING_API_PORT = config.getint('SharingServer', 'SHARING_API_PORT') - self.settings.SHARING_API_SECURE = config.getboolean('SharingServer', 'SHARING_API_SECURE') diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/clients/tenant_profile_client.py b/dev-tools/airavata-python-sdk/airavata_sdk/clients/tenant_profile_client.py index dd0f920403..71b2916183 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/tenant_profile_client.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/tenant_profile_client.py @@ -14,12 +14,10 @@ # limitations under the License. # -import configparser import logging -from typing import Optional from airavata_sdk.transport import utils -from airavata_sdk.transport.settings import ProfileServerSettings +from airavata_sdk import Settings logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -35,9 +33,8 @@ logger.addHandler(handler) class TenantProfileClient(object): - def __init__(self, configuration_file_location: Optional[str] = None): - self.settings = ProfileServerSettings(configuration_file_location) - self._load_settings(configuration_file_location) + def __init__(self): + self.settings = Settings() self.client = utils.initialize_tenant_profile_client( self.settings.PROFILE_SERVICE_HOST, self.settings.PROFILE_SERVICE_PORT, @@ -51,12 +48,3 @@ class TenantProfileClient(object): self.get_all_gateways = self.client.getAllGateways self.is_gateway_exist = self.client.isGatewayExist self.get_all_gateways_for_user = self.client.getAllGatewaysForUser - - - def _load_settings(self, configuration_file_location: Optional[str]): - if configuration_file_location is not None: - config = configparser.ConfigParser() - config.read(configuration_file_location) - self.settings.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') - self.settings.PROFILE_SERVICE_PORT = config.getint('ProfileServer', 'PROFILE_SERVICE_PORT') - self.settings.PROFILE_SERVICE_SECURE = config.getboolean('ProfileServer', 'PROFILE_SERVICE_SECURE') diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/clients/user_profile_client.py b/dev-tools/airavata-python-sdk/airavata_sdk/clients/user_profile_client.py index f42e91cac1..75adae16df 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/user_profile_client.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/user_profile_client.py @@ -14,12 +14,10 @@ # limitations under the License. # -import configparser import logging -from typing import Optional from airavata_sdk.transport import utils -from airavata_sdk.transport.settings import ProfileServerSettings +from airavata_sdk import Settings logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -35,9 +33,8 @@ logger.addHandler(handler) class UserProfileClient(object): - def __init__(self, configuration_file_location: Optional[str] = None): - self.settings = ProfileServerSettings(configuration_file_location) - self._load_settings(configuration_file_location) + def __init__(self): + self.settings = Settings() self.client = utils.initialize_user_profile_client( self.settings.PROFILE_SERVICE_HOST, self.settings.PROFILE_SERVICE_PORT, @@ -52,12 +49,3 @@ class UserProfileClient(object): self.delete_user_profile = self.client.deleteUserProfile self.get_all_user_profiles_in_gateway = self.client.getAllUserProfilesInGateway self.does_user_exist = self.client.doesUserExist - - - def _load_settings(self, configuration_file_location: Optional[str]): - if configuration_file_location is not None: - config = configparser.ConfigParser() - config.read(configuration_file_location) - self.settings.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') - self.settings.PROFILE_SERVICE_PORT = config.getint('ProfileServer', 'PROFILE_SERVICE_PORT') - self.settings.PROFILE_SERVICE_SECURE = config.getboolean('ProfileServer', 'PROFILE_SERVICE_SECURE') diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/clients/utils/api_server_client_util.py b/dev-tools/airavata-python-sdk/airavata_sdk/clients/utils/api_server_client_util.py index 957e3f3a84..ee992c5847 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/utils/api_server_client_util.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/utils/api_server_client_util.py @@ -26,8 +26,8 @@ logger.setLevel(logging.INFO) class APIServerClientUtil(object): - def __init__(self, configuration_file_location: Optional[str], gateway_id: str, username: str, password: Optional[str], access_token: Optional[str] = None): - self.authenticator = Authenticator(configuration_file_location) + def __init__(self, gateway_id: str, username: str, password: Optional[str], access_token: Optional[str] = None): + self.authenticator = Authenticator() if access_token: self.token = self.authenticator.get_airavata_authz_token( gateway_id=gateway_id, @@ -43,7 +43,7 @@ class APIServerClientUtil(object): ) self.gateway_id = gateway_id self.username = username - self.api_server_client = APIServerClient(configuration_file_location) + self.api_server_client = APIServerClient() def get_project_id(self, project_name: str) -> Optional[str]: response = self.api_server_client.get_user_projects(self.token, self.gateway_id, self.username, 10, 0) diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/clients/utils/data_model_creation_util.py b/dev-tools/airavata-python-sdk/airavata_sdk/clients/utils/data_model_creation_util.py index 14bfbad073..65dfa7df07 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/utils/data_model_creation_util.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/utils/data_model_creation_util.py @@ -31,8 +31,8 @@ logger.setLevel(logging.DEBUG) class DataModelCreationUtil(object): - def __init__(self, configuration_file_location: Optional[str], gateway_id: str, username: str, password: Optional[str], access_token: Optional[str] = None): - self.authenticator = Authenticator(configuration_file_location) + def __init__(self, gateway_id: str, username: str, password: Optional[str], access_token: Optional[str] = None): + self.authenticator = Authenticator() if access_token: self.token = self.authenticator.get_airavata_authz_token( gateway_id=gateway_id, @@ -49,9 +49,8 @@ class DataModelCreationUtil(object): self.gateway_id = gateway_id self.username = username self.password = password - self.api_server_client = APIServerClient(configuration_file_location) + self.api_server_client = APIServerClient() self.airavata_util = APIServerClientUtil( - configuration_file_location, self.gateway_id, self.username, self.password, diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/clients/utils/experiment_handler_util.py b/dev-tools/airavata-python-sdk/airavata_sdk/clients/utils/experiment_handler_util.py index 389f3dd9d8..442d3e90ad 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/utils/experiment_handler_util.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/utils/experiment_handler_util.py @@ -26,18 +26,16 @@ from airavata_sdk.clients.keycloak_token_fetcher import Authenticator from airavata_sdk.clients.sftp_file_handling_client import SFTPConnector from airavata_sdk.clients.utils.api_server_client_util import APIServerClientUtil from airavata_sdk.clients.utils.data_model_creation_util import DataModelCreationUtil -from airavata_sdk.transport.settings import ExperimentSettings, GatewaySettings +from airavata_sdk import Settings logger = logging.getLogger('airavata_sdk.clients') logger.setLevel(logging.INFO) class ExperimentHandlerUtil(object): - def __init__(self, configuration_file_location: Optional[str] = None, access_token: Optional[str] = None): - self.configuration_file = configuration_file_location - self.gateway_conf = GatewaySettings(configuration_file_location) - self.experiment_conf = ExperimentSettings(configuration_file_location) - self.authenticator = Authenticator(self.configuration_file) + def __init__(self, access_token: Optional[str] = None): + self.settings = Settings() + self.authenticator = Authenticator() if access_token is None: self.authenticator.authenticate_with_auth_code() access_token = getpass.getpass('Copy paste the access token') @@ -45,25 +43,24 @@ class ExperimentHandlerUtil(object): decode = jwt.decode(access_token, options={"verify_signature": False}) self.user_id = decode['preferred_username'] self.airavata_token = self.authenticator.get_airavata_authz_token( - gateway_id=self.gateway_conf.GATEWAY_ID, + gateway_id=self.settings.GATEWAY_ID, username=self.user_id, token=access_token, ) self.airavata_util = APIServerClientUtil( - self.configuration_file, - gateway_id=self.gateway_conf.GATEWAY_ID, + gateway_id=self.settings.GATEWAY_ID, username=self.user_id, password=None, access_token=access_token, ) - self.data_model_client = DataModelCreationUtil(self.configuration_file, + self.data_model_client = DataModelCreationUtil( username=self.user_id, password=None, - gateway_id=self.gateway_conf.GATEWAY_ID, + gateway_id=self.settings.GATEWAY_ID, access_token=access_token) - self.api_server_client = APIServerClient(self.configuration_file) + self.api_server_client = APIServerClient() def queue_names(self, computation_resource_name: str): resource_id = self.airavata_util.get_resource_host_id(computation_resource_name) @@ -83,52 +80,62 @@ class ExperimentHandlerUtil(object): walltime: int = 30, auto_schedule: bool = False, output_path: str = '.', + group_name: str = "Default", + application_name: str = "Default Application", + project_name: str = "Default Project", ): - execution_id = self.airavata_util.get_execution_id(self.experiment_conf.APPLICATION_NAME) + execution_id = self.airavata_util.get_execution_id(application_name) assert execution_id is not None - project_id = self.airavata_util.get_project_id(self.experiment_conf.PROJECT_NAME) - hosts = self.experiment_conf.COMPUTE_HOST_DOMAIN.split(",") - - computation_resource_name = computation_resource_name if computation_resource_name is not None else hosts[0] + project_id = self.airavata_util.get_project_id(project_name) + assert computation_resource_name is not None resource_host_id = self.airavata_util.get_resource_host_id(computation_resource_name) - group_resource_profile_id = self.airavata_util.get_group_resource_profile_id(self.experiment_conf.GROUP_RESOURCE_PROFILE_NAME) - storage_id = self.airavata_util.get_storage_resource_id(self.experiment_conf.STORAGE_RESOURCE_HOST) + group_resource_profile_id = self.airavata_util.get_group_resource_profile_id(group_name) + + storage_host = self.settings.STORAGE_RESOURCE_HOST + assert storage_host is not None + + sftp_port = self.settings.SFTP_PORT + assert sftp_port is not None + + storage_id = self.airavata_util.get_storage_resource_id(storage_host) assert storage_id is not None + assert project_name is not None + assert application_name is not None + assert experiment_name is not None + assert description is not None logger.info("creating experiment %s", experiment_name) experiment = self.data_model_client.get_experiment_data_model_for_single_application( - project_name=self.experiment_conf.PROJECT_NAME, - application_name=self.experiment_conf.APPLICATION_NAME, + project_name=project_name, + application_name=application_name, experiment_name=experiment_name, description=description, ) - logger.info("connnecting to file upload endpoint %s : %s", self.experiment_conf.STORAGE_RESOURCE_HOST, self.experiment_conf.SFTP_PORT) - sftp_connector = SFTPConnector(host=self.experiment_conf.STORAGE_RESOURCE_HOST, - port=self.experiment_conf.SFTP_PORT, + logger.info("connnecting to file upload endpoint %s : %s", storage_host, sftp_port) + sftp_connector = SFTPConnector(host=storage_host, + port=sftp_port, username=self.user_id, password=self.access_token) + assert local_input_path is not None path_suffix = sftp_connector.upload_files(local_input_path, - self.experiment_conf.PROJECT_NAME, - experiment.experimentName) + project_name, + experiment_name) logger.info("Input files uploaded to %s", path_suffix) - path = self.gateway_conf.GATEWAY_DATA_STORE_DIR + path_suffix - - queue_name = queue_name if queue_name is not None else self.experiment_conf.QUEUE_NAME - - node_count = node_count if node_count is not None else self.experiment_conf.NODE_COUNT - - cpu_count = cpu_count if cpu_count is not None else self.experiment_conf.TOTAL_CPU_COUNT + path = self.settings.GATEWAY_DATA_STORE_DIR + path_suffix - walltime = walltime if walltime is not None else self.experiment_conf.WALL_TIME_LIMIT + assert queue_name is not None + assert node_count is not None + assert cpu_count is not None + assert walltime is not None logger.info("configuring inputs ......") experiment = self.data_model_client.configure_computation_resource_scheduling(experiment_model=experiment, computation_resource_name=computation_resource_name, - group_resource_profile_name=self.experiment_conf.GROUP_RESOURCE_PROFILE_NAME, + group_resource_profile_name=group_name, storageId=storage_id, node_count=int(node_count), total_cpu_count=int(cpu_count), @@ -144,7 +151,7 @@ class ExperimentHandlerUtil(object): data_uris = [] for x in input_file_mapping[key]: data_uri = self.data_model_client.register_input_file(file_identifier=x, - storage_name=self.experiment_conf.STORAGE_RESOURCE_HOST, + storage_name=storage_host, storageId=storage_id, input_file_name=x, uploaded_storage_path=path) @@ -153,13 +160,13 @@ class ExperimentHandlerUtil(object): else: x = input_file_mapping[key] data_uri = self.data_model_client.register_input_file(file_identifier=x, - storage_name=self.experiment_conf.STORAGE_RESOURCE_HOST, + storage_name=storage_host, storageId=storage_id, input_file_name=x, uploaded_storage_path=path) new_file_mapping[key] = data_uri experiment = self.data_model_client.configure_input_and_outputs(experiment, input_files=[], - application_name=self.experiment_conf.APPLICATION_NAME, + application_name=application_name, file_mapping=new_file_mapping) else: for x in os.listdir(local_input_path): @@ -170,13 +177,13 @@ class ExperimentHandlerUtil(object): data_uris = [] for x in input_files: data_uri = self.data_model_client.register_input_file(file_identifier=x, - storage_name=self.experiment_conf.STORAGE_RESOURCE_HOST, + storage_name=storage_host, storageId=storage_id, input_file_name=x, uploaded_storage_path=path) data_uris.append(data_uri) experiment = self.data_model_client.configure_input_and_outputs(experiment, input_files=data_uris, - application_name=self.experiment_conf.APPLICATION_NAME) + application_name=application_name) else: inputs = self.api_server_client.get_application_inputs(self.airavata_token, execution_id) experiment.experimentInputs = inputs @@ -186,17 +193,17 @@ class ExperimentHandlerUtil(object): experiment.experimentOutputs = outputs # create experiment - ex_id = self.api_server_client.create_experiment(self.airavata_token, self.gateway_conf.GATEWAY_ID, experiment) + ex_id = self.api_server_client.create_experiment(self.airavata_token, self.settings.GATEWAY_ID, experiment) # launch experiment - self.api_server_client.launch_experiment(self.airavata_token, ex_id, self.gateway_conf.GATEWAY_ID) + self.api_server_client.launch_experiment(self.airavata_token, ex_id, self.settings.GATEWAY_ID) logger.info("experiment launched id: %s", ex_id) - experiment_url = 'https://' + self.gateway_conf.GATEWAY_URL + '.org/workspace/experiments/' + ex_id + experiment_url = f"{self.settings.GATEWAY_URL}/workspace/experiments/{ex_id}" logger.info("For more information visit %s", experiment_url) - if self.experiment_conf.MONITOR_STATUS: + if self.settings.MONITOR_STATUS: status = self.api_server_client.get_experiment_status(self.airavata_token, ex_id) status_dict = {'0': 'EXECUTING', '4': 'JOB_ACTIVE', '7': 'COMPLETED'} diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/samples/api_server_client_samples.py b/dev-tools/airavata-python-sdk/airavata_sdk/samples/api_server_client_samples.py index 33191ce2f9..7ca737aae1 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/samples/api_server_client_samples.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/samples/api_server_client_samples.py @@ -36,11 +36,6 @@ token = authenticator.get_token_and_user_info_password_flow("default-admin", "12 client = APIServerClient() -# load client with given configuration file (e.g customized_settings.ini) - -# client = APIServerClient('../transport/settings.ini') - - # check for given gateway exists def is_gateway_exists(): try: diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/samples/create_launch_echo_experiment.py b/dev-tools/airavata-python-sdk/airavata_sdk/samples/create_launch_echo_experiment.py index 65e063adfd..53801a74c9 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/samples/create_launch_echo_experiment.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/samples/create_launch_echo_experiment.py @@ -7,32 +7,29 @@ from airavata_sdk.clients.keycloak_token_fetcher import Authenticator from airavata_sdk.clients.sftp_file_handling_client import SFTPConnector from airavata_sdk.clients.utils.api_server_client_util import APIServerClientUtil from airavata_sdk.clients.utils.data_model_creation_util import DataModelCreationUtil -from airavata_sdk.transport.settings import GatewaySettings +from airavata_sdk import Settings logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) - -configFile: str = "/Users/isururanawaka/Documents/Airavata_Repository/airavata/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/transport/settings.ini" - -authenticator = Authenticator(configFile) +authenticator = Authenticator() username: str = "username" password: str = "password" gateway_id: str = "cyberwater" token = authenticator.get_token_and_user_info_password_flow(username=username, password=password, gateway_id=gateway_id) -api_server_client = APIServerClient(configFile) +api_server_client = APIServerClient() -data_model_client = DataModelCreationUtil(configFile, - gateway_id=gateway_id, - username=username, - password=password, - access_token=token.accessToken) +data_model_client = DataModelCreationUtil( + gateway_id=gateway_id, + username=username, + password=password, + access_token=token.accessToken, +) -credential_store_client = CredentialStoreClient(configFile) +credential_store_client = CredentialStoreClient() airavata_util = APIServerClientUtil( - configFile, gateway_id=gateway_id, username=username, password=password, @@ -68,8 +65,7 @@ path_suffix = sftp_connector.upload_files("/Users/isururanawaka/Documents/Cyberw "Default_Project", experiment.experimentName) -gateway_settings = GatewaySettings(configFile) -path = gateway_settings.GATEWAY_DATA_STORE_DIR + path_suffix +path = Settings().GATEWAY_DATA_STORE_DIR + path_suffix # configure computational resources experiment = data_model_client.configure_computation_resource_scheduling(experiment_model=experiment, diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/samples/create_launch_gaussian_experiment.py b/dev-tools/airavata-python-sdk/airavata_sdk/samples/create_launch_gaussian_experiment.py index e0b8e4920f..92b7c3b2e8 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/samples/create_launch_gaussian_experiment.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/samples/create_launch_gaussian_experiment.py @@ -28,9 +28,7 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -configFile = "/Users/isururanawaka/Documents/Cyberwater/poc/resources/settings.ini" - -authenticator = Authenticator(configFile) +authenticator = Authenticator() user_name = "username" password = "password" @@ -42,22 +40,20 @@ token = authenticator.get_token_and_user_info_password_flow( gateway_id=gateway_id, ) -api_server_client = APIServerClient(configFile) +api_server_client = APIServerClient() airavata_util = APIServerClientUtil( - configFile, gateway_id=gateway_id, username=user_name, password=password, ) data_model_client = DataModelCreationUtil( - configFile, gateway_id=gateway_id, username=user_name, password=password, ) -credential_store_client = CredentialStoreClient(configFile) +credential_store_client = CredentialStoreClient() executionId = airavata_util.get_execution_id("Gaussian") projectId = airavata_util.get_project_id("Default Project") diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/samples/group_manager_client_samples.py b/dev-tools/airavata-python-sdk/airavata_sdk/samples/group_manager_client_samples.py index a568ab3189..7185341732 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/samples/group_manager_client_samples.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/samples/group_manager_client_samples.py @@ -32,11 +32,6 @@ token = authenticator.get_token_and_user_info_password_flow("default-admin", "12 client = GroupManagerClient() -# load client with given configuration file (e.g customized_settings.ini) - -#client = GroupManagerClient('../transport/settings.ini') - - # create group in airavata def create_group(): try: diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/samples/iam_admin_client_samples.py b/dev-tools/airavata-python-sdk/airavata_sdk/samples/iam_admin_client_samples.py index 6b4465ea7b..1c0c84de5d 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/samples/iam_admin_client_samples.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/samples/iam_admin_client_samples.py @@ -31,10 +31,6 @@ token = authenticator.get_token_and_user_info_password_flow("default-admin", "12 client = IAMAdminClient() -# load client with given configuration file (e.g customized_settings.ini) -# client = IAMAdminClient('../transport/settings.ini') - - def is_user_exisits(): try: user = client.is_user_exist(token, "default-admin") diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/samples/metadata_fetcher.py b/dev-tools/airavata-python-sdk/airavata_sdk/samples/metadata_fetcher.py index 82da39e1b9..350fe899cb 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/samples/metadata_fetcher.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/samples/metadata_fetcher.py @@ -23,12 +23,10 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -configFile = "/Users/isururanawaka/Documents/Cyberwater/poc/resources/settings.ini" - -authenticator = Authenticator(configFile) +authenticator = Authenticator() token = authenticator.get_token_and_user_info_password_flow("username", "password", "cyberwater") -api_server_client = APIServerClient(configFile) +api_server_client = APIServerClient() # fetch all application deployments deployments = api_server_client.get_all_application_deployments(token, "cyberwater"); diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/samples/sharing_registry_client_samples.py b/dev-tools/airavata-python-sdk/airavata_sdk/samples/sharing_registry_client_samples.py index 2e543d9e4a..8b77352c8a 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/samples/sharing_registry_client_samples.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/samples/sharing_registry_client_samples.py @@ -32,11 +32,6 @@ token = authenticator.get_token_and_user_info_password_flow("default-admin", "12 # load GroupManagerClient with default configuration client = SharingRegistryClient() - -# load client with given configuration file (e.g customized_settings.ini) - -# client = SharingRegistryClient('../transport/settings.ini') - # create domian def create_domain(): try: diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/samples/tenant_profile_client_samples.py b/dev-tools/airavata-python-sdk/airavata_sdk/samples/tenant_profile_client_samples.py index 5bca7bb10b..4eee41674e 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/samples/tenant_profile_client_samples.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/samples/tenant_profile_client_samples.py @@ -32,7 +32,7 @@ token = authenticator.get_token_and_user_info_password_flow("default-admin", "12 # load client with given configuration file (e.g customized_settings.ini) -client = TenantProfileClient('../transport/settings.ini') +client = TenantProfileClient() def get_all_gateways(): diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/samples/user_profile_client_samples.py b/dev-tools/airavata-python-sdk/airavata_sdk/samples/user_profile_client_samples.py index ef9054ed1a..e42542889c 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/samples/user_profile_client_samples.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/samples/user_profile_client_samples.py @@ -32,10 +32,6 @@ token = authenticator.get_token_and_user_info_password_flow("default-admin", "12 client = UserProfileClient() -# load client with given configuration file (e.g customized_settings.ini) -# client = UserProfileClient('../transport/settings.ini') - - def add_user_profile(): try: profile = UserProfile() diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/transport/settings.py b/dev-tools/airavata-python-sdk/airavata_sdk/transport/settings.py deleted file mode 100644 index a127af61cb..0000000000 --- a/dev-tools/airavata-python-sdk/airavata_sdk/transport/settings.py +++ /dev/null @@ -1,111 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import configparser -import os -from typing import Optional - -config = configparser.ConfigParser() - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -defaultSettings = os.path.join(BASE_DIR, "transport", "settings.ini") -config.read(defaultSettings) - - -class APIServerSettings(object): - def __init__(self, configFileLocation: Optional[str] = None): - if configFileLocation is not None: - config.read(configFileLocation) - self.API_SERVER_HOST = config.get('APIServer', 'API_HOST') - self.API_SERVER_PORT = config.getint('APIServer', 'API_PORT') - self.API_SERVER_SECURE = config.getboolean('APIServer', 'API_SECURE') - - -class ProfileServerSettings(object): - def __init__(self, configFileLocation: Optional[str] = None): - if configFileLocation is not None: - config.read(configFileLocation) - self.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') - self.PROFILE_SERVICE_PORT = config.getint('ProfileServer', 'PROFILE_SERVICE_PORT') - self.PROFILE_SERVICE_SECURE = config.getboolean('ProfileServer', 'PROFILE_SERVICE_SECURE') - - -class SharingServerSettings(object): - def __init__(self, configFileLocation: Optional[str] = None): - if configFileLocation is not None: - config.read(configFileLocation) - self.SHARING_API_HOST = config.get('SharingServer', 'SHARING_API_HOST') - self.SHARING_API_PORT = config.getint('SharingServer', 'SHARING_API_PORT') - self.SHARING_API_SECURE = config.getboolean('SharingServer', 'SHARING_API_SECURE') - - -class CredentialStoreServerSettings(object): - def __init__(self, configFileLocation: Optional[str] = None): - if configFileLocation is not None: - config.read(configFileLocation) - self.CREDENTIAL_STORE_API_HOST = config.get('CredentialStoreServer', 'CREDENTIAL_STORE_API_HOST') - self.CREDENTIAL_STORE_API_PORT = config.getint('CredentialStoreServer', 'CREDENTIAL_STORE_API_PORT') - self.CREDENTIAL_STORE_API_SECURE = config.getboolean('CredentialStoreServer', 'CREDENTIAL_STORE_API_SECURE') - - -class ThriftSettings(object): - def __init__(self, configFileLocation: Optional[str] = None): - if configFileLocation is not None: - config.read(configFileLocation) - self.THRIFT_CLIENT_POOL_KEEPALIVE = config.getfloat('Thrift', 'THRIFT_CLIENT_POOL_KEEPALIVE') - - -class KeycloakServerSettings(object): - def __init__(self, configFileLocation: Optional[str] = None): - if configFileLocation is not None: - config.read(configFileLocation) - # self.KEYCLOAK_CA_CERTIFICATE = os.path.join(BASE_DIR, "samples", "resources", "incommon_rsa_server_ca.pem") - self.CLIENT_ID = config.get('KeycloakServer', 'CLIENT_ID') - self.CLIENT_SECRET = config.get('KeycloakServer', 'CLIENT_SECRET') - self.TOKEN_URL = config.get('KeycloakServer', 'TOKEN_URL') - self.USER_INFO_URL = config.get('KeycloakServer', 'USER_INFO_URL') - self.VERIFY_SSL = config.getboolean('KeycloakServer', 'VERIFY_SSL') - self.LOGIN_DESKTOP_URI = config.get('KeycloakServer', 'LOGIN_DESKTOP_URI') - - -class GatewaySettings(object): - def __init__(self, configFileLocation: Optional[str] = None): - if configFileLocation is not None: - config.read(configFileLocation) - self.GATEWAY_ID = config.get('Gateway', 'GATEWAY_ID') - self.GATEWAY_URL = config.get('Gateway', 'GATEWAY_URL') - self.GATEWAY_DATA_STORE_RESOURCE_ID = config.get('Gateway', 'GATEWAY_DATA_STORE_RESOURCE_ID') - self.GATEWAY_DATA_STORE_DIR = config.get('Gateway', 'GATEWAY_DATA_STORE_DIR') - self.GATEWAY_DATA_STORE_HOSTNAME = config.get('Gateway', 'GATEWAY_DATA_STORE_HOSTNAME') - self.FILE_UPLOAD_TEMP_DIR = config.get('Gateway', 'FILE_UPLOAD_TEMP_DIR') - - -class ExperimentSettings(object): - def __init__(self, configFileLocation: Optional[str] = None): - if configFileLocation is not None: - config.read(configFileLocation) - self.APPLICATION_NAME = config.get('ExperimentConf', 'APPLICATION_NAME') - self.PROJECT_NAME = config.get('ExperimentConf', 'PROJECT_NAME') - self.COMPUTE_HOST_DOMAIN = config.get('ExperimentConf', 'COMPUTE_HOST_DOMAIN') - self.GROUP_RESOURCE_PROFILE_NAME = config.get('ExperimentConf', 'GROUP_RESOURCE_PROFILE_NAME') - self.STORAGE_RESOURCE_HOST = config.get('ExperimentConf', 'STORAGE_RESOURCE_HOST') - self.SFTP_PORT = config.get('ExperimentConf', 'SFTP_PORT') - self.NODE_COUNT = config.get('ExperimentConf', 'NODE_COUNT') - self.TOTAL_CPU_COUNT = config.get('ExperimentConf', 'TOTAL_CPU_COUNT') - self.WALL_TIME_LIMIT = config.get('ExperimentConf', 'WALL_TIME_LIMIT') - self.QUEUE_NAME = config.get('ExperimentConf', 'QUEUE_NAME') - self.MONITOR_STATUS = config.getboolean('ExperimentConf', 'MONITOR_STATUS') - diff --git a/dev-tools/airavata-python-sdk/airavata_sdk/transport/utils.py b/dev-tools/airavata-python-sdk/airavata_sdk/transport/utils.py index 7a1dfa1855..b6610d1c3d 100644 --- a/dev-tools/airavata-python-sdk/airavata_sdk/transport/utils.py +++ b/dev-tools/airavata-python-sdk/airavata_sdk/transport/utils.py @@ -15,7 +15,8 @@ # import logging -from typing import Generic, Optional, TypeVar +import ssl +from typing import Optional, TypeVar from thrift.protocol import TBinaryProtocol from thrift.protocol.TMultiplexedProtocol import TMultiplexedProtocol @@ -32,15 +33,11 @@ from airavata.service.profile.tenant.cpi import TenantProfileService from airavata.service.profile.tenant.cpi.constants import TENANT_PROFILE_CPI_NAME from airavata.service.profile.user.cpi import UserProfileService from airavata.service.profile.user.cpi.constants import USER_PROFILE_CPI_NAME -from airavata_sdk.transport import settings +from airavata_sdk import Settings log = logging.getLogger(__name__) -default_api_server_settings = settings.APIServerSettings() -default_profile_server_settings = settings.ProfileServerSettings() -default_sharing_server_settings = settings.SharingServerSettings() -default_credential_store_server_settings = settings.CredentialStoreServerSettings() -default_thrift_settings = settings.ThriftSettings() +settings = Settings() T = TypeVar( 'T', @@ -67,7 +64,15 @@ class ThriftClient: self.service_name = service_name if self.secure: - self.transport = TSSLSocket.TSSLSocket(self.host, self.port, validate=False, socket_keepalive=True) + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_REQUIRED + self.transport = TSSLSocket.TSSLSocket( + self.host, + self.port, + ssl_context=ssl_context, + socket_keepalive=True, + ) else: self.transport = TSocket.TSocket(self.host, self.port, socket_keepalive=True) self.transport = TTransport.TBufferedTransport(self.transport) @@ -94,56 +99,56 @@ class ThriftClient: def initialize_api_client_pool( - host=default_api_server_settings.API_SERVER_HOST, - port=default_api_server_settings.API_SERVER_PORT, - secure=default_api_server_settings.API_SERVER_SECURE, + host=settings.API_SERVER_HOSTNAME, + port=settings.API_SERVER_PORT, + secure=settings.API_SERVER_SECURE, ) -> Airavata.Client: return ThriftClient(Airavata.Client, host, port, secure).client def initialize_group_manager_client( - host=default_profile_server_settings.PROFILE_SERVICE_HOST, - port=default_profile_server_settings.PROFILE_SERVICE_PORT, - secure=default_profile_server_settings.PROFILE_SERVICE_SECURE, + host=settings.PROFILE_SERVICE_HOST, + port=settings.PROFILE_SERVICE_PORT, + secure=settings.PROFILE_SERVICE_SECURE, ) -> GroupManagerService.Client: return ThriftClient(GroupManagerService.Client, host, port, secure, GROUP_MANAGER_CPI_NAME).client def initialize_iam_admin_client( - host=default_profile_server_settings.PROFILE_SERVICE_HOST, - port=default_profile_server_settings.PROFILE_SERVICE_PORT, - secure=default_profile_server_settings.PROFILE_SERVICE_SECURE, + host=settings.PROFILE_SERVICE_HOST, + port=settings.PROFILE_SERVICE_PORT, + secure=settings.PROFILE_SERVICE_SECURE, ) -> IamAdminServices.Client: return ThriftClient(IamAdminServices.Client, host, port, secure, IAM_ADMIN_SERVICES_CPI_NAME).client def initialize_tenant_profile_client( - host=default_profile_server_settings.PROFILE_SERVICE_HOST, - port=default_profile_server_settings.PROFILE_SERVICE_PORT, - secure=default_profile_server_settings.PROFILE_SERVICE_SECURE, + host=settings.PROFILE_SERVICE_HOST, + port=settings.PROFILE_SERVICE_PORT, + secure=settings.PROFILE_SERVICE_SECURE, ) -> TenantProfileService.Client: return ThriftClient(TenantProfileService.Client, host, port, secure, TENANT_PROFILE_CPI_NAME).client def initialize_user_profile_client( - host=default_profile_server_settings.PROFILE_SERVICE_HOST, - port=default_profile_server_settings.PROFILE_SERVICE_PORT, - secure=default_profile_server_settings.PROFILE_SERVICE_SECURE, + host=settings.PROFILE_SERVICE_HOST, + port=settings.PROFILE_SERVICE_PORT, + secure=settings.PROFILE_SERVICE_SECURE, ) -> UserProfileService.Client: return ThriftClient(UserProfileService.Client, host, port, secure, USER_PROFILE_CPI_NAME).client def initialize_sharing_registry_client( - host=default_sharing_server_settings.SHARING_API_HOST, - port=default_sharing_server_settings.SHARING_API_PORT, - secure=default_sharing_server_settings.SHARING_API_SECURE, + host=settings.SHARING_API_HOST, + port=settings.SHARING_API_PORT, + secure=settings.SHARING_API_SECURE, ) -> SharingRegistryService.Client: return ThriftClient(SharingRegistryService.Client, host, port, secure).client def initialize_credential_store_client( - host=default_credential_store_server_settings.CREDENTIAL_STORE_API_HOST, - port=default_credential_store_server_settings.CREDENTIAL_STORE_API_PORT, - secure=default_credential_store_server_settings.CREDENTIAL_STORE_API_SECURE, + host=settings.CREDENTIAL_STORE_API_HOST, + port=settings.CREDENTIAL_STORE_API_PORT, + secure=settings.CREDENTIAL_STORE_API_SECURE, ) -> CredentialStoreService.Client: return ThriftClient(CredentialStoreService.Client, host, port, secure).client
