This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2674 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 7182e0aa711c67a033ea5dff174ef9394c2e267b Author: leonidfrolov <[email protected]> AuthorDate: Fri Feb 4 17:09:43 2022 +0200 [DATALAB-2607]: fixed dataproc creation for rstudio and zeppelin --- .../scripts/gcp/rstudio_install_dataengine-service_kernels.py | 5 ++--- .../scripts/gcp/zeppelin_install_dataengine-service_kernels.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py index f098a95..127b492 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py @@ -43,7 +43,6 @@ parser.add_argument('--project_name', type=str, default='') parser.add_argument('--os_user', type=str, default='') parser.add_argument('--edge_hostname', type=str, default='') parser.add_argument('--proxy_port', type=str, default='') -parser.add_argument('--pip_mirror', type=str, default='') parser.add_argument('--application', type=str, default='') args = parser.parse_args() @@ -72,6 +71,6 @@ if __name__ == "__main__": hadoop_version = datalab.actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name, args.cluster_name, 'hadoop') conn.sudo('''bash -l -c 'echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf' ''') conn.sudo('''bash -l -c 'echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo "https_proxy=$proxy" >> ~/.wgetrc' ''') - conn.sudo('''bash -l -c 'unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} --application {11}' ''' + conn.sudo('''bash -l -c 'unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --application {10}' ''' .format(os.environ['gcp_project_id'], os.environ['conf_resource'], args.bucket, args.cluster_name, args.dataproc_version, spark_version, hadoop_version, - args.region, args.project_name, args.os_user, args.pip_mirror, args.application)) + args.region, args.project_name, args.os_user, args.application)) diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py index e17520a..22936d4 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py @@ -41,7 +41,6 @@ parser.add_argument('--project_name', type=str, default='') parser.add_argument('--os_user', type=str, default='') parser.add_argument('--edge_hostname', type=str, default='') parser.add_argument('--proxy_port', type=str, default='') -parser.add_argument('--pip_mirror', type=str, default='') parser.add_argument('--application', type=str, default='') args = parser.parse_args() @@ -76,7 +75,7 @@ if __name__ == "__main__": hadoop_version = datalab.actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name, args.cluster_name, 'hadoop') conn.sudo('''bash -l -c 'echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf' ''') conn.sudo('''bash -l -c 'echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo "https_proxy=$proxy" >> ~/.wgetrc' ''') - conn.sudo('''bash -l -c 'unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} --application {11} --livy_version {12} --multiple_clusters {13} --r_enabled {14}' ''' + conn.sudo('''bash -l -c 'unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --application {10} --livy_version {11} --multiple_clusters {12} --r_enabled {13}' ''' .format(os.environ['gcp_project_id'], os.environ['conf_resource'], args.bucket, args.cluster_name, args.dataproc_version, - spark_version, hadoop_version, args.region, args.project_name, args.os_user, args.pip_mirror, args.application, + spark_version, hadoop_version, args.region, args.project_name, args.os_user, args.application, os.environ['notebook_livy_version'], os.environ['notebook_multiple_clusters'], r_enabled)) \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
