This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch odahu-integration
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/odahu-integration by this push:
new ae3aaad [odahu-integration] Added all parameters and fixed some issues
ae3aaad is described below
commit ae3aaadb33d11f2a0fac37145fe7d636ace9ece4
Author: Demyan Mysakovets <demian.mysakove...@gmail.com>
AuthorDate: Fri Jan 24 12:04:09 2020 +0200
[odahu-integration] Added all parameters and fixed some issues
---
infrastructure-provisioning/scripts/deploy_dlab.py | 2 +
.../src/general/conf/dlab.ini | 53 ++++++-
.../src/general/scripts/gcp/odahu_deploy.py | 92 ++++++++++---
.../src/general/scripts/gcp/odahu_prepare.py | 4 +-
.../src/general/scripts/gcp/odahu_resume.py | 152 +++++++++++++++++++++
.../src/general/scripts/gcp/odahu_suspend.py | 150 ++++++++++++++++++++
.../src/general/scripts/gcp/odahu_terminate.py | 102 +++++++++++++-
.../src/general/scripts/gcp/ssn_configure.py | 1 +
infrastructure-provisioning/src/odahu/fabfile.py | 32 +++++
.../src/odahu/templates/profile.json | 74 ++++++++++
10 files changed, 642 insertions(+), 20 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_dlab.py
b/infrastructure-provisioning/scripts/deploy_dlab.py
index 9438873..03afac8 100644
--- a/infrastructure-provisioning/scripts/deploy_dlab.py
+++ b/infrastructure-provisioning/scripts/deploy_dlab.py
@@ -119,6 +119,8 @@ parser.add_argument('--usage', type=str,
default='line_item_usage_amount', help=
parser.add_argument('--cost', type=str, default='line_item_blended_cost',
help='Column name in report file that contains cost tag')
parser.add_argument('--resource_id', type=str,
default='line_item_resource_id', help='Column name in report file that contains
'
'dlab resource id tag')
+parser.add_argument('--odahu_tls_crt', type=str, default='', help='TLS
Certificate for Odahu cluster')
+parser.add_argument('--odahu_tls_key', type=str, default='', help='TLS Key for
Odahu cluster')
parser.add_argument('--ldap_hostname', type=str, default='localhost',
help='Ldap instance hostname')
parser.add_argument('--ldap_dn', type=str, default='dc=example,dc=com',
help='Ldap distinguished name')
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini
b/infrastructure-provisioning/src/general/conf/dlab.ini
index 7873b43..331a4ca 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -53,7 +53,7 @@ vpc_cidr = '172.31.0.0/16'
### CIDR of second VPC
vpc2_cidr = '172.32.0.0/16'
### Enable or disable duo VPC mode(true|false)
-duo_vpc_enable = false
+duo_vpc_enable = falseF
### Range of subnets which will be using for user's environments
# user_subnets_range =
### Comma-separated CIDR of IPs which will have access to SSN and Edge nodes
@@ -316,6 +316,57 @@ os_expl_memory = 3500
### Depending on RAM size on instance, this parameter determines size of RAM
when explicit allocation RAM is used
expl_instance_memory = 8000
+#--- [odahu] odahuflow parameters ---#
+[odahu]
+###
+bastion_tag = odahubastion
+###
+dns_zone_name = ailifecycle-org
+###
+docker_repo = https://hub.docker.com/u/odahu
+###
+cidr = 172.31.0.0/24
+###
+grafana_admin = grafana_admin
+###
+# grafana_pass =
+###
+initial_node_count = 6
+###
+istio_helm_repo =
https://storage.googleapis.com/istio-release/releases/1.2.2/charts
+###
+k8s_version = 1.13.12
+###
+helm_repo = https://github.com/odahu/odahu-helm
+###
+infra_version = 1.0.0
+###
+odahuflow_version = 1.0.0
+###
+mlflow_toolchain_version = 1.0.0
+###
+jupyterlab_version = 1.0.0
+###
+packager_version = 1.0.0
+###
+# node_locations =
+###
+node_version = 1.13.12-gke.2
+###
+pods_cidr = 10.42.0.0/17
+###
+root_domain = ailifecycle.org
+###
+service_cidr = 10.42.128.0/20
+###
+dns_project_id = or2-msq-epmd-legn-t1iylu
+###
+infra_vpc_peering = 0
+###
+# tls_crt =
+###
+# tls_key =
+
#--- [ldap] ldap parameters ---#
[ldap]
### Ldap hostname
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py
b/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py
index 67acdc4..56e37ac 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py
@@ -28,6 +28,7 @@ from dlab.fab import *
from dlab.meta_lib import *
from dlab.actions_lib import *
import os
+import base64
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['project_name'],
@@ -37,10 +38,19 @@ if __name__ == "__main__":
level=logging.DEBUG,
filename=local_log_filepath)
+ secret = ''.join(random.choices(string.ascii_uppercase +
+string.ascii_lowercase +
+ string.digits, k=16))
+
print('Generating infrastructure names and tags')
odahu_conf = dict()
+ odahu_conf['allowed_cidr'] =
json.dumps(os.environ['odahu_allowed_cidr'].split(','))
+ odahu_conf['bastion_tag'] = os.environ['odahu_bastion_tag']
odahu_conf['project_id'] = (os.environ['gcp_project_id'])
odahu_conf['region'] = (os.environ['gcp_region'])
+ odahu_conf['zone'] = (os.environ['gcp_zone'])
+ odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
+ odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
+ odahu_conf['odahu_cidr'] = os.environ['odahu_cidr']
odahu_conf['service_base_name'] =
(os.environ['conf_service_base_name']).lower().replace('_', '-')
odahu_conf['project_name'] =
(os.environ['project_name']).lower().replace('_', '-')
odahu_conf['odahu_cluster_name'] =
(os.environ['odahu_cluster_name']).lower().replace('_', '-')
@@ -55,28 +65,78 @@ if __name__ == "__main__":
odahu_conf['vpc_name'] = odahu_conf['service_base_name'] + '-ssn-vpc'
odahu_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
odahu_conf['private_subnet_name'] =
'{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
-
odahu_conf['project_name'])
- odahu_conf['keycloak_realm'] = os.environ['keycloak_realm']
- odahu_conf['keycloak_url'] = os.environ['keycloak_url']\
+
odahu_conf['project_name'])
+ odahu_conf['grafana_admin'] = os.environ['odahu_grafana_admin']
+ odahu_conf['grafana_pass'] = id_generator()
+ odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
+ odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
+ odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
+ odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
+ odahu_conf['oauth_oidc_issuer_url'] = os.environ['oauth_oidc_issuer_url']
odahu_conf['oauth_client_id'] = os.environ['oauth_client_id']
odahu_conf['oauth_client_secret'] = os.environ['oauth_client_secret']
- odahu_conf['oauth_coockie_secret'] = os.environ['oauth_coockie_secret']
- odahu_conf['tls_crt'] = os.environ['tls_crt']
- odahu_conf['tls_key'] = os.environ['tls_key']
+ odahu_conf['oauth_cookie_secret'] = secret
+ odahu_conf['odahu_infra_version'] = os.environ['odahu_infra_version']
+ odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
+ odahu_conf['mlflow_toolchain_version'] =
os.environ['odahu_mlflow_toolchain_version']
+ odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
+ odahu_conf['packager_version'] = os.environ['odahu_packager_version']
+ odahu_conf['model_docker_url'] = os.environ['odahu_model_docker_url']
+ odahu_conf['node_locations'] =
json.dumps(os.environ['odahu_node_locations'].split(','))
+ odahu_conf['node_version'] = os.environ['odahu_node_version']
+ odahu_conf['pods_cidr'] = os.environ['odahu_pods_cidr']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+ odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['tls_crt'])
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['tls_key'])
+ odahu_conf['ssh_key'] = os.environ['ssh_key']
+ odahu_conf['dns_project_id'] = os.environ['dns_project_id']
+ odahu_conf['decrypt_token'] = secret
+ odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
+ print('Preparing parameters file')
try:
local("cp /root/templates/profile.json /tmp/")
- local("sudo sed
\'s|<PROJECT_ID>|{}|g\'".format(odahu_conf['project_id']))
- local("sudo sed
\'s|<CLUSTER_NAME>|{}|g\'".format(odahu_conf['odahu_cluster_name']))
- local("sudo sed \'s|<REGION>|{}|g\'".format(odahu_conf['region']))
- local("sudo sed
\'s|<KEYCLOAK_REALM>|{}|g\'".format(odahu_conf['keycloak_realm']))
- local("sudo sed
\'s|<KEYCLOAK_URL>|{}|g\'".format(odahu_conf['keycloak_url']))
- local("sudo sed \'s|<VPC_NAME>|{}|g\'".format(odahu_conf['vpc_name']))
- local("sudo sed
\'s|<SUBNET_NAME>|{}|g\'".format(odahu_conf['private_subnet_name']))
- local("sudo sed
\'s|<OAUTH_CLIENT_ID>|{}|g\'".format(odahu_conf['oauth_client_id']))
- local("sudo sed
\'s|<OAUTH_CLIENT_ID>|{}|g\'".format(odahu_conf['oauth_client_id']))
- local("sudo sed
\'s|<TLS_CRT>|{}|g\'".format(odahu_conf['oauth_client_id']))
+ local("sed -i \'s|<ALLOWED_IP_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['allowed_cidr']))
+ local("sed -i \'s|<BASTION_TAG>|{}|g\'
/tmp/profile.json".format(odahu_conf['bastion_tag']))
+ local("sed -i \'s|<PROJECT_ID>|{}|g\'
/tmp/profile.json".format(odahu_conf['project_id']))
+ local("sed -i \'s|<CLUSTER_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahu_cluster_name']))
+ local("sed -i \'s|<REGION>|{}|g\'
/tmp/profile.json".format(odahu_conf['region']))
+ local("sed -i \'s|<ZONE>|{}|g\'
/tmp/profile.json".format(odahu_conf['zone']))
+ local("sed -i \'s|<DNS_ZONE_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['dns_zone_name']))
+ local("sed -i \'s|<DOCKER_REPO>|{}|g\'
/tmp/profile.json".format(odahu_conf['docker_repo']))
+ local("sed -i \'s|<ODAHU_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahu_cidr']))
+ local("sed -i \'s|<GRAFANA_ADMIN>|{}|g\'
/tmp/profile.json".format(odahu_conf['grafana_admin']))
+ local("sed -i \'s|<GRAFANA_PASS>|{}|g\'
/tmp/profile.json".format(odahu_conf['grafana_pass']))
+ local("sed -i \'s|<INITIAL_NODE_COUNT>|{}|g\'
/tmp/profile.json".format(odahu_conf['initial_node_count']))
+ local("sed -i \'s|<ISTIO_HELM_REPO>|{}|g\'
/tmp/profile.json".format(odahu_conf['istio_helm_repo']))
+ local("sed -i \'s|<HELM_REPO>|{}|g\'
/tmp/profile.json".format(odahu_conf['helm_repo']))
+ local("sed -i \'s|<K8S_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['k8s_version']))
+ local("sed -i \'s|<ODAHU_INFRA_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahu_infra_version']))
+ local("sed -i \'s|<ODAHUFLOW_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahuflow_version']))
+ local("sed -i \'s|<MLFLOW_TOOLCHAIN_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['mlflow_toolchain_version']))
+ local("sed -i \'s|<JUPYTERLAB_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['jupyterlab_version']))
+ local("sed -i \'s|<PACKAGER_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['packager_version']))
+ local("sed -i \'s|<MODEL_DOCKER_URL>|{}|g\'
/tmp/profile.json".format(odahu_conf['model_docker_url']))
+ local("sed -i \'s|<NODE_LOCATIONS>|{}|g\'
/tmp/profile.json".format(odahu_conf['node_locations']))
+ local("sed -i \'s|<NODE_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['node_version']))
+ local("sed -i \'s|<OAUTH_OIDC_ISSUER_URL>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_oidc_issuer_url']))
+ local("sed -i \'s|<VPC_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['vpc_name']))
+ local("sed -i \'s|<SUBNET_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['private_subnet_name']))
+ local("sed -i \'s|<OAUTH_CLIENT_ID>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_client_id']))
+ local("sed -i \'s|<OAUTH_CLIENT_SECRET>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_client_secret']))
+ local("sed -i \'s|<OAUTH_COOCKIE_SECRET>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_cookie_secret']))
+ local("sed -i \'s|<PODS_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['pods_cidr']))
+ local("sed -i \'s|<ROOT_DOMAIN>|{}|g\'
/tmp/profile.json".format(odahu_conf['root_domain']))
+ local("sed -i \'s|<SERVICE_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['service_cidr']))
+ local("sed -i \'s|<TLS_CRT>|{}|g\'
/tmp/profile.json".format(odahu_conf['tls_crt'].replace('\n', '')))
+ local("sed -i \'s|<TLS_KEY>|{}|g\'
/tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
+ local("sed -i \'s|<SSH_KEY>|{}|g\'
/tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
+ local("sed -i \'s|<DNS_PROJECT_ID>|{}|g\'
/tmp/profile.json".format(odahu_conf['dns_project_id']))
+ local("sed -i \'s|<DECRYPT_TOKEN>|{}|g\'
/tmp/profile.json".format(odahu_conf['decrypt_token']))
+ local("sed -i \'s|<INFRA_VPC_PEERING>|{}|g\'
/tmp/profile.json".format(odahu_conf['infra_vpc_peering']))
local('cp /tmp/profile.json /')
+ local('cat /profile.json')
except Exception as err:
traceback.print_exc()
append_result("Failed to configure parameter file.", str(err))
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/odahu_prepare.py
b/infrastructure-provisioning/src/general/scripts/gcp/odahu_prepare.py
index 7428486..ed80b5c 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/odahu_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_prepare.py
@@ -41,6 +41,7 @@ if __name__ == "__main__":
odahu_conf = dict()
odahu_conf['service_base_name'] =
(os.environ['conf_service_base_name']).lower().replace('_', '-')
odahu_conf['odahu_cluster_name'] =
(os.environ['odahu_cluster_name']).lower().replace('_', '-')
+ odahu_conf['region'] = os.environ['gcp_region']
odahu_conf['bucket_name'] =
"{}-tfstate".format((os.environ['odahu_cluster_name']).lower().replace('_',
'-'))
odahu_conf['static_address_name'] =
"{}-nat-gw".format((os.environ['odahu_cluster_name']).lower().replace('_', '-'))
@@ -63,8 +64,7 @@ if __name__ == "__main__":
try:
logging.info('[CREATE NAT GATEWAY]')
print('[CREATE NAT GATEWAY]')
- project_conf['static_ip'] = \
- GCPMeta().get_static_address(odahu_conf['region'],
odahu_conf['static_address_name'])['address']
+ GCPActions().create_static_address(odahu_conf['static_address_name'],
odahu_conf['region'])
except Exception as err:
print('Error: {0}'.format(err))
append_result("Unable to reserve static ip.", str(err))
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/odahu_resume.py
b/infrastructure-provisioning/src/general/scripts/gcp/odahu_resume.py
new file mode 100644
index 0000000..459edb8
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_resume.py
@@ -0,0 +1,152 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+import string
+import random
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['project_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/project/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ secret = ''.join(random.choices(string.ascii_uppercase +
+string.ascii_lowercase +
+ string.digits, k = 16))
+
+ print('Generating infrastructure names and tags')
+ odahu_conf = dict()
+ odahu_conf['allowed_cidr'] =
json.dumps(os.environ['odahu_allowed_cidr'].split(','))
+ odahu_conf['bastion_tag'] = os.environ['odahu_bastion_tag']
+ odahu_conf['project_id'] = (os.environ['gcp_project_id'])
+ odahu_conf['region'] = (os.environ['gcp_region'])
+ odahu_conf['zone'] = (os.environ['gcp_zone'])
+ odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
+ odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
+ odahu_conf['odahu_cidr'] = os.environ['odahu_cidr']
+ odahu_conf['service_base_name'] =
(os.environ['conf_service_base_name']).lower().replace('_', '-')
+ odahu_conf['project_name'] =
(os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['odahu_cluster_name'] =
(os.environ['odahu_cluster_name']).lower().replace('_', '-')
+ odahu_conf['bucket_name'] =
"{}-tfstate".format((os.environ['odahu_cluster_name']).lower().replace('_',
'-'))
+ odahu_conf['static_address_name'] =
"{}-nat-gw".format((os.environ['odahu_cluster_name']).lower().replace('_', '-'))
+ try:
+ if os.environ['gcp_vpc_name'] == '':
+ raise KeyError
+ else:
+ odahu_conf['vpc_name'] = os.environ['gcp_vpc_name']
+ except KeyError:
+ odahu_conf['vpc_name'] = odahu_conf['service_base_name'] + '-ssn-vpc'
+ odahu_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
+ odahu_conf['private_subnet_name'] =
'{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
+
odahu_conf['project_name'])
+ odahu_conf['grafana_admin'] = os.environ['odahu_grafana_admin']
+ odahu_conf['grafana_pass'] = os.environ['odahu_grafana_pass']
+ odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
+ odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
+ odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
+ odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
+ odahu_conf['oauth_oidc_issuer_url'] = os.environ['oauth_oidc_issuer_url']
+ odahu_conf['oauth_client_id'] = os.environ['oauth_client_id']
+ odahu_conf['oauth_client_secret'] = os.environ['oauth_client_secret']
+ odahu_conf['oauth_cookie_secret'] = secret
+ odahu_conf['odahu_infra_version'] = os.environ['odahu_infra_version']
+ odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
+ odahu_conf['mlflow_toolchain_version'] =
os.environ['odahu_mlflow_toolchain_version']
+ odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
+ odahu_conf['packager_version'] = os.environ['odahu_packager_version']
+ odahu_conf['model_docker_url'] = os.environ['odahu_model_docker_url']
+ odahu_conf['node_locations'] =
json.dumps(os.environ['odahu_node_locations'].split(','))
+ odahu_conf['node_version'] = os.environ['odahu_node_version']
+ odahu_conf['pods_cidr'] = os.environ['odahu_pods_cidr']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+ odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['tls_crt'])
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['tls_key'])
+ odahu_conf['ssh_key'] = os.environ['ssh_key']
+ odahu_conf['dns_project_id'] = os.environ['dns_project_id']
+ odahu_conf['decrypt_token'] = secret
+ odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
+
+ print('Preparing parameters file')
+ try:
+ local("cp /root/templates/profile.json /tmp/")
+ local("sed -i \'s|<ALLOWED_IP_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['allowed_cidr']))
+ local("sed -i \'s|<BASTION_TAG>|{}|g\'
/tmp/profile.json".format(odahu_conf['bastion_tag']))
+ local("sed -i \'s|<PROJECT_ID>|{}|g\'
/tmp/profile.json".format(odahu_conf['project_id']))
+ local("sed -i \'s|<CLUSTER_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahu_cluster_name']))
+ local("sed -i \'s|<REGION>|{}|g\'
/tmp/profile.json".format(odahu_conf['region']))
+ local("sed -i \'s|<ZONE>|{}|g\'
/tmp/profile.json".format(odahu_conf['zone']))
+ local("sed -i \'s|<DNS_ZONE_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['dns_zone_name']))
+ local("sed -i \'s|<DOCKER_REPO>|{}|g\'
/tmp/profile.json".format(odahu_conf['docker_repo']))
+ local("sed -i \'s|<ODAHU_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahu_cidr']))
+ local("sed -i \'s|<GRAFANA_ADMIN>|{}|g\'
/tmp/profile.json".format(odahu_conf['grafana_admin']))
+ local("sed -i \'s|<GRAFANA_PASS>|{}|g\'
/tmp/profile.json".format(odahu_conf['grafana_pass']))
+ local("sed -i \'s|<INITIAL_NODE_COUNT>|{}|g\'
/tmp/profile.json".format(odahu_conf['initial_node_count']))
+ local("sed -i \'s|<ISTIO_HELM_REPO>|{}|g\'
/tmp/profile.json".format(odahu_conf['istio_helm_repo']))
+ local("sed -i \'s|<HELM_REPO>|{}|g\'
/tmp/profile.json".format(odahu_conf['helm_repo']))
+ local("sed -i \'s|<K8S_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['k8s_version']))
+ local("sed -i \'s|<ODAHU_INFRA_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahu_infra_version']))
+ local("sed -i \'s|<ODAHUFLOW_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahuflow_version']))
+ local("sed -i \'s|<MLFLOW_TOOLCHAIN_VERSION>|{}|g\'
/tmp/profile.json".format(
+ odahu_conf['mlflow_toolchain_version']))
+ local("sed -i \'s|<JUPYTERLAB_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['jupyterlab_version']))
+ local("sed -i \'s|<PACKAGER_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['packager_version']))
+ local("sed -i \'s|<MODEL_DOCKER_URL>|{}|g\'
/tmp/profile.json".format(odahu_conf['model_docker_url']))
+ local("sed -i \'s|<NODE_LOCATIONS>|{}|g\'
/tmp/profile.json".format(odahu_conf['node_locations']))
+ local("sed -i \'s|<NODE_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['node_version']))
+ local("sed -i \'s|<OAUTH_OIDC_ISSUER_URL>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_oidc_issuer_url']))
+ local("sed -i \'s|<VPC_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['vpc_name']))
+ local("sed -i \'s|<SUBNET_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['private_subnet_name']))
+ local("sed -i \'s|<OAUTH_CLIENT_ID>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_client_id']))
+ local("sed -i \'s|<OAUTH_CLIENT_SECRET>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_client_secret']))
+ local("sed -i \'s|<OAUTH_COOCKIE_SECRET>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_cookie_secret']))
+ local("sed -i \'s|<PODS_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['pods_cidr']))
+ local("sed -i \'s|<ROOT_DOMAIN>|{}|g\'
/tmp/profile.json".format(odahu_conf['root_domain']))
+ local("sed -i \'s|<SERVICE_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['service_cidr']))
+ local("sed -i \'s|<TLS_CRT>|{}|g\'
/tmp/profile.json".format(odahu_conf['tls_crt'].replace('\n', '')))
+ local("sed -i \'s|<TLS_KEY>|{}|g\'
/tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
+ local("sed -i \'s|<SSH_KEY>|{}|g\'
/tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
+ local("sed -i \'s|<DNS_PROJECT_ID>|{}|g\'
/tmp/profile.json".format(odahu_conf['dns_project_id']))
+ local("sed -i \'s|<DECRYPT_TOKEN>|{}|g\'
/tmp/profile.json".format(odahu_conf['decrypt_token']))
+ local("sed -i \'s|<INFRA_VPC_PEERING>|{}|g\'
/tmp/profile.json".format(odahu_conf['infra_vpc_peering']))
+ local('cp /tmp/profile.json /')
+ local('cat /profile.json')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to configure parameter file.", str(err))
+ sys.exit(1)
+
+ try:
+ local('tf_runner resume -v')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to deploy Odahu cluster.", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/odahu_suspend.py
b/infrastructure-provisioning/src/general/scripts/gcp/odahu_suspend.py
new file mode 100644
index 0000000..e967315
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_suspend.py
@@ -0,0 +1,150 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['project_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/project/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ secret = ''.join(random.choices(string.ascii_uppercase +
+string.ascii_lowercase +
+ string.digits, k=16))
+
+ print('Generating infrastructure names and tags')
+ odahu_conf = dict()
+ odahu_conf['allowed_cidr'] =
json.dumps(os.environ['odahu_allowed_cidr'].split(','))
+ odahu_conf['bastion_tag'] = os.environ['odahu_bastion_tag']
+ odahu_conf['project_id'] = (os.environ['gcp_project_id'])
+ odahu_conf['region'] = (os.environ['gcp_region'])
+ odahu_conf['zone'] = (os.environ['gcp_zone'])
+ odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
+ odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
+ odahu_conf['odahu_cidr'] = os.environ['odahu_cidr']
+ odahu_conf['service_base_name'] =
(os.environ['conf_service_base_name']).lower().replace('_', '-')
+ odahu_conf['project_name'] =
(os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['odahu_cluster_name'] =
(os.environ['odahu_cluster_name']).lower().replace('_', '-')
+ odahu_conf['bucket_name'] =
"{}-tfstate".format((os.environ['odahu_cluster_name']).lower().replace('_',
'-'))
+ odahu_conf['static_address_name'] =
"{}-nat-gw".format((os.environ['odahu_cluster_name']).lower().replace('_', '-'))
+ try:
+ if os.environ['gcp_vpc_name'] == '':
+ raise KeyError
+ else:
+ odahu_conf['vpc_name'] = os.environ['gcp_vpc_name']
+ except KeyError:
+ odahu_conf['vpc_name'] = odahu_conf['service_base_name'] + '-ssn-vpc'
+ odahu_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
+ odahu_conf['private_subnet_name'] =
'{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
+
odahu_conf['project_name'])
+ odahu_conf['grafana_admin'] = os.environ['odahu_grafana_admin']
+ odahu_conf['grafana_pass'] = os.environ['odahu_grafana_pass']
+ odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
+ odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
+ odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
+ odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
+ odahu_conf['oauth_oidc_issuer_url'] = os.environ['oauth_oidc_issuer_url']
+ odahu_conf['oauth_client_id'] = os.environ['oauth_client_id']
+ odahu_conf['oauth_client_secret'] = os.environ['oauth_client_secret']
+ odahu_conf['oauth_cookie_secret'] = secret
+ odahu_conf['odahu_infra_version'] = os.environ['odahu_infra_version']
+ odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
+ odahu_conf['mlflow_toolchain_version'] =
os.environ['odahu_mlflow_toolchain_version']
+ odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
+ odahu_conf['packager_version'] = os.environ['odahu_packager_version']
+ odahu_conf['model_docker_url'] = os.environ['odahu_model_docker_url']
+ odahu_conf['node_locations'] =
json.dumps(os.environ['odahu_node_locations'].split(','))
+ odahu_conf['node_version'] = os.environ['odahu_node_version']
+ odahu_conf['pods_cidr'] = os.environ['odahu_pods_cidr']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+ odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['tls_crt'])
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['tls_key'])
+ odahu_conf['ssh_key'] = os.environ['ssh_key']
+ odahu_conf['dns_project_id'] = os.environ['dns_project_id']
+ odahu_conf['decrypt_token'] = secret
+ odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
+
+ print('Preparing parameters file')
+ try:
+ local("cp /root/templates/profile.json /tmp/")
+ local("sed -i \'s|<ALLOWED_IP_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['allowed_cidr']))
+ local("sed -i \'s|<BASTION_TAG>|{}|g\'
/tmp/profile.json".format(odahu_conf['bastion_tag']))
+ local("sed -i \'s|<PROJECT_ID>|{}|g\'
/tmp/profile.json".format(odahu_conf['project_id']))
+ local("sed -i \'s|<CLUSTER_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahu_cluster_name']))
+ local("sed -i \'s|<REGION>|{}|g\'
/tmp/profile.json".format(odahu_conf['region']))
+ local("sed -i \'s|<ZONE>|{}|g\'
/tmp/profile.json".format(odahu_conf['zone']))
+ local("sed -i \'s|<DNS_ZONE_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['dns_zone_name']))
+ local("sed -i \'s|<DOCKER_REPO>|{}|g\'
/tmp/profile.json".format(odahu_conf['docker_repo']))
+ local("sed -i \'s|<ODAHU_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahu_cidr']))
+ local("sed -i \'s|<GRAFANA_ADMIN>|{}|g\'
/tmp/profile.json".format(odahu_conf['grafana_admin']))
+ local("sed -i \'s|<GRAFANA_PASS>|{}|g\'
/tmp/profile.json".format(odahu_conf['grafana_pass']))
+ local("sed -i \'s|<INITIAL_NODE_COUNT>|{}|g\'
/tmp/profile.json".format(odahu_conf['initial_node_count']))
+ local("sed -i \'s|<ISTIO_HELM_REPO>|{}|g\'
/tmp/profile.json".format(odahu_conf['istio_helm_repo']))
+ local("sed -i \'s|<HELM_REPO>|{}|g\'
/tmp/profile.json".format(odahu_conf['helm_repo']))
+ local("sed -i \'s|<K8S_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['k8s_version']))
+ local("sed -i \'s|<ODAHU_INFRA_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahu_infra_version']))
+ local("sed -i \'s|<ODAHUFLOW_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahuflow_version']))
+ local("sed -i \'s|<MLFLOW_TOOLCHAIN_VERSION>|{}|g\'
/tmp/profile.json".format(
+ odahu_conf['mlflow_toolchain_version']))
+ local("sed -i \'s|<JUPYTERLAB_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['jupyterlab_version']))
+ local("sed -i \'s|<PACKAGER_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['packager_version']))
+ local("sed -i \'s|<MODEL_DOCKER_URL>|{}|g\'
/tmp/profile.json".format(odahu_conf['model_docker_url']))
+ local("sed -i \'s|<NODE_LOCATIONS>|{}|g\'
/tmp/profile.json".format(odahu_conf['node_locations']))
+ local("sed -i \'s|<NODE_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['node_version']))
+ local("sed -i \'s|<OAUTH_OIDC_ISSUER_URL>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_oidc_issuer_url']))
+ local("sed -i \'s|<VPC_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['vpc_name']))
+ local("sed -i \'s|<SUBNET_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['private_subnet_name']))
+ local("sed -i \'s|<OAUTH_CLIENT_ID>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_client_id']))
+ local("sed -i \'s|<OAUTH_CLIENT_SECRET>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_client_secret']))
+ local("sed -i \'s|<OAUTH_COOCKIE_SECRET>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_cookie_secret']))
+ local("sed -i \'s|<PODS_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['pods_cidr']))
+ local("sed -i \'s|<ROOT_DOMAIN>|{}|g\'
/tmp/profile.json".format(odahu_conf['root_domain']))
+ local("sed -i \'s|<SERVICE_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['service_cidr']))
+ local("sed -i \'s|<TLS_CRT>|{}|g\'
/tmp/profile.json".format(odahu_conf['tls_crt'].replace('\n', '')))
+ local("sed -i \'s|<TLS_KEY>|{}|g\'
/tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
+ local("sed -i \'s|<SSH_KEY>|{}|g\'
/tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
+ local("sed -i \'s|<DNS_PROJECT_ID>|{}|g\'
/tmp/profile.json".format(odahu_conf['dns_project_id']))
+ local("sed -i \'s|<DECRYPT_TOKEN>|{}|g\'
/tmp/profile.json".format(odahu_conf['decrypt_token']))
+ local("sed -i \'s|<INFRA_VPC_PEERING>|{}|g\'
/tmp/profile.json".format(odahu_conf['infra_vpc_peering']))
+ local('cp /tmp/profile.json /')
+ local('cat /profile.json')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to configure parameter file.", str(err))
+ sys.exit(1)
+
+ try:
+ local('tf_runner suspend -v')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to deploy Odahu cluster.", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/odahu_terminate.py
b/infrastructure-provisioning/src/general/scripts/gcp/odahu_terminate.py
index 8bc4669..23dc9a5 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/odahu_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_terminate.py
@@ -38,11 +38,111 @@ if __name__ == "__main__":
level=logging.DEBUG,
filename=local_log_filepath)
+ secret = ''.join(random.choices(string.ascii_uppercase +
+string.ascii_lowercase +
+ string.digits, k=16))
+
print('Generating infrastructure names and tags')
odahu_conf = dict()
- odahu_conf['odahu_cluster_name'] =
(os.environ['odahu_cluster_name']).lower().replace('_', '-')
+ odahu_conf['allowed_cidr'] =
json.dumps(os.environ['odahu_allowed_cidr'].split(','))
+ odahu_conf['bastion_tag'] = os.environ['odahu_bastion_tag']
+ odahu_conf['project_id'] = (os.environ['gcp_project_id'])
odahu_conf['region'] = (os.environ['gcp_region'])
+ odahu_conf['zone'] = (os.environ['gcp_zone'])
+ odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
+ odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
+ odahu_conf['odahu_cidr'] = os.environ['odahu_cidr']
+ odahu_conf['service_base_name'] =
(os.environ['conf_service_base_name']).lower().replace('_', '-')
+ odahu_conf['project_name'] =
(os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['odahu_cluster_name'] =
(os.environ['odahu_cluster_name']).lower().replace('_', '-')
+ odahu_conf['bucket_name'] =
"{}-tfstate".format((os.environ['odahu_cluster_name']).lower().replace('_',
'-'))
+ odahu_conf['static_address_name'] =
"{}-nat-gw".format((os.environ['odahu_cluster_name']).lower().replace('_', '-'))
+ try:
+ if os.environ['gcp_vpc_name'] == '':
+ raise KeyError
+ else:
+ odahu_conf['vpc_name'] = os.environ['gcp_vpc_name']
+ except KeyError:
+ odahu_conf['vpc_name'] = odahu_conf['service_base_name'] + '-ssn-vpc'
+ odahu_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
+ odahu_conf['private_subnet_name'] =
'{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
+
odahu_conf['project_name'])
+ odahu_conf['grafana_admin'] = os.environ['odahu_grafana_admin']
+ odahu_conf['grafana_pass'] = os.environ['odahu_grafana_pass']
+ odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
+ odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
+ odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
+ odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
+ odahu_conf['oauth_oidc_issuer_url'] = os.environ['oauth_oidc_issuer_url']
+ odahu_conf['oauth_client_id'] = os.environ['oauth_client_id']
+ odahu_conf['oauth_client_secret'] = os.environ['oauth_client_secret']
+ odahu_conf['oauth_cookie_secret'] = secret
+ odahu_conf['odahu_infra_version'] = os.environ['odahu_infra_version']
+ odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
+ odahu_conf['mlflow_toolchain_version'] =
os.environ['odahu_mlflow_toolchain_version']
+ odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
+ odahu_conf['packager_version'] = os.environ['odahu_packager_version']
+ odahu_conf['model_docker_url'] = os.environ['odahu_model_docker_url']
+ odahu_conf['node_locations'] =
json.dumps(os.environ['odahu_node_locations'].split(','))
+ odahu_conf['node_version'] = os.environ['odahu_node_version']
+ odahu_conf['pods_cidr'] = os.environ['odahu_pods_cidr']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+ odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['tls_crt'])
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['tls_key'])
+ odahu_conf['ssh_key'] = os.environ['ssh_key']
+ odahu_conf['dns_project_id'] = os.environ['dns_project_id']
+ odahu_conf['decrypt_token'] = secret
+ odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
+ print('Preparing parameters file')
+ try:
+ print(odahu_conf['allowed_cidr'])
+ local("cp /root/templates/profile.json /tmp/")
+ local("sed -i \'s|<ALLOWED_IP_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['allowed_cidr']))
+ local("sed -i \'s|<BASTION_TAG>|{}|g\'
/tmp/profile.json".format(odahu_conf['bastion_tag']))
+ local("sed -i \'s|<PROJECT_ID>|{}|g\'
/tmp/profile.json".format(odahu_conf['project_id']))
+ local("sed -i \'s|<CLUSTER_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahu_cluster_name']))
+ local("sed -i \'s|<REGION>|{}|g\'
/tmp/profile.json".format(odahu_conf['region']))
+ local("sed -i \'s|<ZONE>|{}|g\'
/tmp/profile.json".format(odahu_conf['zone']))
+ local("sed -i \'s|<DNS_ZONE_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['dns_zone_name']))
+ local("sed -i \'s|<DOCKER_REPO>|{}|g\'
/tmp/profile.json".format(odahu_conf['docker_repo']))
+ local("sed -i \'s|<ODAHU_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahu_cidr']))
+ local("sed -i \'s|<GRAFANA_ADMIN>|{}|g\'
/tmp/profile.json".format(odahu_conf['grafana_admin']))
+ local("sed -i \'s|<GRAFANA_PASS>|{}|g\'
/tmАp/profile.json".format(odahu_conf['grafana_pass']))
+ local("sed -i \'s|<INITIAL_NODE_COUNT>|{}|g\'
/tmp/profile.json".format(odahu_conf['initial_node_count']))
+ local("sed -i \'s|<ISTIO_HELM_REPO>|{}|g\'
/tmp/profile.json".format(odahu_conf['istio_helm_repo']))
+ local("sed -i \'s|<HELM_REPO>|{}|g\'
/tmp/profile.json".format(odahu_conf['helm_repo']))
+ local("sed -i \'s|<K8S_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['k8s_version']))
+ local("sed -i \'s|<ODAHU_INFRA_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahu_infra_version']))
+ local("sed -i \'s|<ODAHUFLOW_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['odahuflow_version']))
+ local("sed -i \'s|<MLFLOW_TOOLCHAIN_VERSION>|{}|g\'
/tmp/profile.json".format(
+ odahu_conf['mlflow_toolchain_version']))
+ local("sed -i \'s|<JUPYTERLAB_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['jupyterlab_version']))
+ local("sed -i \'s|<PACKAGER_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['packager_version']))
+ local("sed -i \'s|<MODEL_DOCKER_URL>|{}|g\'
/tmp/profile.json".format(odahu_conf['model_docker_url']))
+ local("sed -i \'s|<NODE_LOCATIONS>|{}|g\'
/tmp/profile.json".format(odahu_conf['node_locations']))
+ local("sed -i \'s|<NODE_VERSION>|{}|g\'
/tmp/profile.json".format(odahu_conf['node_version']))
+ local("sed -i \'s|<OAUTH_OIDC_ISSUER_URL>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_oidc_issuer_url']))
+ local("sed -i \'s|<VPC_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['vpc_name']))
+ local("sed -i \'s|<SUBNET_NAME>|{}|g\'
/tmp/profile.json".format(odahu_conf['private_subnet_name']))
+ local("sed -i \'s|<OAUTH_CLIENT_ID>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_client_id']))
+ local("sed -i \'s|<OAUTH_CLIENT_SECRET>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_client_secret']))
+ local("sed -i \'s|<OAUTH_COOCKIE_SECRET>|{}|g\'
/tmp/profile.json".format(odahu_conf['oauth_cookie_secret']))
+ local("sed -i \'s|<PODS_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['pods_cidr']))
+ local("sed -i \'s|<ROOT_DOMAIN>|{}|g\'
/tmp/profile.json".format(odahu_conf['root_domain']))
+ local("sed -i \'s|<SERVICE_CIDR>|{}|g\'
/tmp/profile.json".format(odahu_conf['service_cidr']))
+ local("sed -i \'s|<TLS_CRT>|{}|g\'
/tmp/profile.json".format(odahu_conf['tls_crt'].replace('\n', '')))
+ local("sed -i \'s|<TLS_KEY>|{}|g\'
/tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
+ local("sed -i \'s|<SSH_KEY>|{}|g\'
/tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
+ local("sed -i \'s|<DNS_PROJECT_ID>|{}|g\'
/tmp/profile.json".format(odahu_conf['dns_project_id']))
+ local("sed -i \'s|<DECRYPT_TOKEN>|{}|g\'
/tmp/profile.json".format(odahu_conf['decrypt_token']))
+ local("sed -i \'s|<INFRA_VPC_PEERING>|{}|g\'
/tmp/profile.json".format(odahu_conf['infra_vpc_peering']))
+ local('cp /tmp/profile.json /')
+ local('cat /profile.json')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to configure parameter file.", str(err))
+ sys.exit(1)
print('Removing Odahu cluster')
try:
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
index a68751f..cc4627a 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
@@ -223,6 +223,7 @@ if __name__ == "__main__":
additional_config = [{"name": "base", "tag": "latest"},
{"name": "project", "tag": "latest"},
{"name": "edge", "tag": "latest"},
+ {"name": "odahu", "tag": "latest"},
{"name": "jupyter", "tag": "latest"},
{"name": "jupyterlab", "tag": "latest"},
{"name": "rstudio", "tag": "latest"},
diff --git a/infrastructure-provisioning/src/odahu/fabfile.py
b/infrastructure-provisioning/src/odahu/fabfile.py
index 68626bc..19beb83 100644
--- a/infrastructure-provisioning/src/odahu/fabfile.py
+++ b/infrastructure-provisioning/src/odahu/fabfile.py
@@ -70,4 +70,36 @@ def terminate():
except Exception as err:
traceback.print_exc()
append_result("Failed to terminate Odahuflow cluster.", str(err))
+ sys.exit(1)
+
+
+# Main function for suspending Odahuflow cluster
+def stop():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" +
local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+ try:
+ params = "--uuid {}".format(notebook_config['uuid'])
+ local("~/scripts/{}.py {}".format('odahu_suspend', params))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to suspend Odahuflow cluster.", str(err))
+ sys.exit(1)
+
+
+# Main function for resuming Odahuflow cluster
+def start():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" +
local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ local("tf_runner resume")
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to resume Odahuflow cluster.", str(err))
sys.exit(1)
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/odahu/templates/profile.json
b/infrastructure-provisioning/src/odahu/templates/profile.json
new file mode 100644
index 0000000..98f93a2
--- /dev/null
+++ b/infrastructure-provisioning/src/odahu/templates/profile.json
@@ -0,0 +1,74 @@
+{
+ "alert_slack_url": "",
+ "allowed_ips": <ALLOWED_IP_CIDR>,
+ "aws_cidr": "",
+ "aws_profile": "",
+ "aws_route_table_id": "",
+ "aws_sg": "",
+ "aws_vpc_id": "",
+ "bastion_tag": "<BASTION_TAG>",
+ "cloud_type": "gcp",
+ "cluster_context": "gke_<PROJECT_ID>_<REGION>_<CLUSTER_NAME>",
+ "cluster_name": "<CLUSTER_NAME>",
+ "cluster_type": "gcp/gke",
+ "config_context_auth_info": "gke_<PROJECT_ID>_<REGION>_<CLUSTER_NAME>",
+ "config_context_cluster": "gke_<PROJECT_ID>_<REGION>_<CLUSTER_NAME>",
+ "dns_zone_name": "<DNS_ZONE_NAME>",
+ "docker_password": "",
+ "docker_repo": "<DOCKER_REPO>",
+ "docker_user": "",
+ "enclave_jwt_secret": "",
+ "gcp_cidr": "<ODAHU_CIDR>",
+ "git_examples_description": "",
+ "git_examples_key": "",
+ "git_examples_reference": "",
+ "git_examples_uri": "",
+ "git_examples_web_ui_link": "",
+ "github_org_name": "",
+ "gke_node_tag": "<CLUSTER_NAME>-gke-node",
+ "grafana_admin": "<GRAFANA_ADMIN>",
+ "grafana_pass": "<GRAFANA_PASS>",
+ "infra_cidr": "",
+ "initial_node_count": "<INITIAL_NODE_COUNT>",
+ "istio_helm_repo": "<ISTIO_HELM_REPO>",
+ "k8s_version": "<K8S_VERSION>",
+ "location": "<REGION>",
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_issuer_url": "<OAUTH_OIDC_ISSUER_URL>",
+ "data_bucket": "<CLUSTER_NAME>-data-bucket",
+ "helm_repo": "<HELM_REPO>",
+ "odahu_infra_version": "<ODAHU_INFRA_VERSION>",
+ "odahuflow_version": "<ODAHUFLOW_VERSION>",
+ "mlflow_toolchain_version": "<MLFLOW_TOOLCHAIN_VERSION>",
+ "jupyterlab_version": "<JUPYTERLAB_VERSION>",
+ "packager_version": "<PACKAGER_VERSION>",
+ "model_docker_url": "<MODEL_DOCKER_URL>",
+ "vpc_name": "<VPC_NAME>",
+ "network_name": "<VPC_NAME>",
+ "subnet_name": "<SUBNET_NAME>",
+ "node_locations": <NODE_LOCATIONS>,
+ "node_version": "<NODE_VERSION>",
+ "oauth_client_id": "<OAUTH_CLIENT_ID>",
+ "oauth_client_secret": "<OAUTH_CLIENT_SECRET>",
+ "oauth_cookie_secret": "<OAUTH_COOCKIE_SECRET>",
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "pods_cidr": "<PODS_CIDR>",
+ "project_id": "<PROJECT_ID>",
+ "region": "<REGION>",
+ "root_domain": "<ROOT_DOMAIN>",
+ "service_cidr": "<SERVICE_CIDR>",
+ "ssh_key": "<SSH_KEY>",
+ "test_oauth_auth_url": "",
+ "test_oauth_client_id": "",
+ "test_oauth_client_secret": "",
+ "test_oauth_scope": "",
+ "test_user_email": "",
+ "test_user_password": "",
+ "tfstate_bucket": "<CLUSTER_NAME>-tfstate",
+ "tls_crt": "<TLS_CRT>",
+ "tls_key": "<TLS_KEY>",
+ "zone": "<ZONE>",
+ "dns_project_id": "<DNS_PROJECT_ID>",
+ "odahuflow_connection_decrypt_token": "<DECRYPT_TOKEN>",
+ "infra_vpc_peering": "<INFRA_VPC_PEERING>"
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org
For additional commands, e-mail: commits-h...@dlab.apache.org
