This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-1864 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit f75d74204c5daf9264b1fe0caab26120a2d62407 Author: leonidfrolov <[email protected]> AuthorDate: Fri Aug 20 15:10:06 2021 +0300 [DATALAB-1864]: added part mount place check for dataengine nodes --- .../src/base/scripts/install_user_key.py | 2 +- .../src/general/lib/azure/actions_lib.py | 2 +- .../general/scripts/azure/dataengine_configure.py | 29 ++++++++++++++++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/infrastructure-provisioning/src/base/scripts/install_user_key.py b/infrastructure-provisioning/src/base/scripts/install_user_key.py index d7a5faf..2e68abd 100644 --- a/infrastructure-provisioning/src/base/scripts/install_user_key.py +++ b/infrastructure-provisioning/src/base/scripts/install_user_key.py @@ -75,7 +75,7 @@ if __name__ == "__main__": print("Installing users key...") try: copy_key(deeper_config) - conn.close() + #conn.close() except: print("Users keyfile {0} could not be found at {1}/{0}".format(args.keyfile, deeper_config['user_keydir'])) sys.exit(1) diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py index 3a5accf..b83aba7 100644 --- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py @@ -1139,7 +1139,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'): datalab.fab.conn.put(templates_dir + 'notebook_spark-defaults_local.conf', '/tmp/notebook_spark-defaults_local.conf') datalab.fab.conn.sudo("jar_list=`find {} -name '*.jar' | tr '\\n' ','` ; echo \"spark.jars $jar_list\" >> \ /tmp/notebook_spark-defaults_local.conf".format(jars_dir)) - datalab.fab.conn.sudo('\cp -f /tmp/notebook_spark-defaults_local.conf /opt/spark/conf/spark-defaults.conf') + datalab.fab.conn.sudo('cp -f /tmp/notebook_spark-defaults_local.conf /opt/spark/conf/spark-defaults.conf') if memory_type == 'driver': spark_memory = datalab.fab.get_spark_memory() datalab.fab.conn.sudo('sed -i "/spark.*.memory/d" /opt/spark/conf/spark-defaults.conf') diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py index f678c9f..61d3cf5 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py @@ -74,8 +74,21 @@ def configure_slave(slave_number, data_engine): sys.exit(1) try: - logging.info('[CLEANING INSTANCE FOR MASTER NODE]') - print('[CLEANING INSTANCE FOR MASTER NODE]') + logging.info('[ENSURING RIGHT MOUNT PATHS FOR SLAVE NODE]') + print('[ENSURING RIGHT MOUNT PATHS FOR SLAVE NODE]') + try: + AzureActions.ensure_right_mount_paths() + except: + traceback.print_exc() + raise Exception + except Exception as err: + clear_resources() + datalab.fab.append_result("Failed to ensure_right_mount_paths on slave instance..", str(err)) + sys.exit(1) + + try: + logging.info('[CLEANING INSTANCE FOR SLAVE NODE]') + print('[CLEANING INSTANCE FOR SLAVE NODE]') params = '--hostname {} --keyfile {} --os_user {} --application {}' \ .format(slave_hostname, keyfile_name, data_engine['datalab_ssh_user'], os.environ['application']) try: @@ -264,6 +277,18 @@ if __name__ == "__main__": datalab.fab.append_result("Failed to install ssh user key on master.", str(err)) sys.exit(1) + try: + logging.info('[ENSURING RIGHT MOUNT PATHS FOR MASTER NODE]') + print('[ENSURING RIGHT MOUNT PATHS FOR MASTER NODE]') + try: + AzureActions.ensure_right_mount_paths() + except: + traceback.print_exc() + raise Exception + except Exception as err: + clear_resources() + datalab.fab.append_result("Failed to ensure right mount paths on master instance..", str(err)) + sys.exit(1) try: logging.info('[CLEANING INSTANCE FOR MASTER NODE]') --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
