This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-1408 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 6e54d7f12b490b480040055ce73303669b7224c8 Author: leonidfrolov <[email protected]> AuthorDate: Tue Sep 13 16:31:40 2022 +0300 [DATALAB-1408]: specified spark version 3.1 for hdinsight --- .../src/general/files/azure/dataengine-service_description.json | 2 +- .../src/general/scripts/azure/dataengine-service_create.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json b/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json index faa7d1767..7128eeff8 100644 --- a/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json +++ b/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json @@ -15,6 +15,6 @@ }, "templates": [ - {"version":"4.0", "applications": [{"Name":"Spark", "Version": "2.4"}]} + {"version":"4.0", "applications": [{"Name":"Spark", "Version": "3.1"}]} ] } \ No newline at end of file diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py index d03518bd5..b4075318b 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py +++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py @@ -73,6 +73,10 @@ def create_cluster_parameters(location, tags, cluster_version, cluster_login_use tier=Tier.standard, cluster_definition=ClusterDefinition( kind="Spark", + component_version= + { + "Spark": "3.1" + }, configurations={ "gateway": { "restAuthCredential.isEnabled": "true", --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
