This is an automated email from the ASF dual-hosted git repository. ykinash pushed a commit to branch DATALAB-2430 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 97b493c2600b495bc758258f0d3e302259367aa6 Author: KinashYurii <[email protected]> AuthorDate: Wed Jun 16 12:36:23 2021 +0300 [DATALAB-2430] -- added missed param --- .../SparkStandaloneClusterResource.java | 35 ++++++++--- .../computational/SparkComputationalCreateGcp.java | 69 +++++++++++++++------- .../dto/SparkStandaloneClusterCreateForm.java | 31 ++++++++-- .../service/impl/ComputationalServiceImpl.java | 11 +++- .../datalab/backendapi/util/RequestBuilder.java | 23 +++++--- 5 files changed, 125 insertions(+), 44 deletions(-) diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/SparkStandaloneClusterResource.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/SparkStandaloneClusterResource.java index f4bbd27..d06e4c9 100644 --- a/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/SparkStandaloneClusterResource.java +++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/SparkStandaloneClusterResource.java @@ -39,24 +39,45 @@ import java.util.Map; @ToString(callSuper = true) public class SparkStandaloneClusterResource extends UserComputationalResource { @NotBlank - @JsonProperty("dataengine_instance_count") - private String dataEngineInstanceCount; + @JsonProperty("master_instance_count") + private String masterDataEngineInstanceCount; @NotBlank - @JsonProperty("dataengine_instance_shape") - private String dataEngineInstanceShape; + @JsonProperty("master_instance_shape") + private String masterDataEngineInstanceShape; + + @NotBlank + @JsonProperty("slave_instance_count") + private String slaveDataEngineInstanceCount; + + @NotBlank + @JsonProperty("slave_instance_shape") + private String slaveDtaEngineInstanceShape; @Builder public SparkStandaloneClusterResource(String computationalName, String computationalId, String imageName, String templateName, String status, Date uptime, SchedulerJobDTO schedulerJobData, boolean reuploadKeyRequired, - String dataEngineInstanceCount, String dataEngineInstanceShape, + String masterDataEngineInstanceCount, String masterDataEngineInstanceShape, + String slaveDataEngineInstanceCount, String slaveDataEngineInstanceShape, + String masterGpuCount, String masterGpuType, + String slaveGpuCount, String slaveGpuType, + Boolean enabledGPU, List<ResourceURL> resourceURL, LocalDateTime lastActivity, List<ClusterConfig> config, Map<String, String> tags, int totalInstanceCount) { super(computationalName, computationalId, imageName, templateName, status, uptime, schedulerJobData, reuploadKeyRequired, resourceURL, lastActivity, tags, totalInstanceCount); - this.dataEngineInstanceCount = dataEngineInstanceCount; - this.dataEngineInstanceShape = dataEngineInstanceShape; + super.setMasterGpuCount(masterGpuCount); + super.setMasterGpuType(masterGpuType); + super.setSlaveGpuCount(slaveGpuCount); + super.setSlaveGpuType(slaveGpuType); + super.setEnabledGPU(enabledGPU); + + this.masterDataEngineInstanceCount = masterDataEngineInstanceCount; + this.masterDataEngineInstanceShape = masterDataEngineInstanceShape; + this.slaveDataEngineInstanceCount = slaveDataEngineInstanceCount; + this.slaveDtaEngineInstanceShape = slaveDataEngineInstanceShape; + this.config = config; } } diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/SparkComputationalCreateGcp.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/SparkComputationalCreateGcp.java index 90bd51e..57e1d8d 100644 --- a/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/SparkComputationalCreateGcp.java +++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/SparkComputationalCreateGcp.java @@ -22,60 +22,85 @@ package com.epam.datalab.dto.gcp.computational; import com.epam.datalab.dto.aws.computational.ClusterConfig; import com.epam.datalab.dto.base.computational.ComputationalBase; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.MoreObjects; +import lombok.ToString; import java.util.List; +@ToString public class SparkComputationalCreateGcp extends ComputationalBase<SparkComputationalCreateGcp> { - @JsonProperty("dataengine_instance_count") - private String dataEngineInstanceCount; - @JsonProperty("gcp_dataengine_slave_size") - private String dataEngineSlaveSize; + @JsonProperty("dataengine_master_instance_count") + private String masterDataEngineInstanceCount; @JsonProperty("gcp_dataengine_master_size") - private String dataEngineMasterSize; + private String masterDataEngineInstanceShape; + @JsonProperty("dataengine_slave_instance_count") + private String slaveDataEngineInstanceCount; + @JsonProperty("gcp_dataengine_slave_size") + private String slaveDtaEngineInstanceShape; + + @JsonProperty("master_gpu_type") + private String masterGPUType; + @JsonProperty("slave_gpu_type") + private String slaveGPUType; + @JsonProperty("master_gpu_count") + private String masterGPUCount; + @JsonProperty("slave_gpu_count") + private String slaveGPUCount; + + @JsonProperty("spark_configurations") private List<ClusterConfig> config; @JsonProperty("conf_shared_image_enabled") private String sharedImageEnabled; public SparkComputationalCreateGcp withDataEngineInstanceCount(String dataEngineInstanceCount) { - this.dataEngineInstanceCount = dataEngineInstanceCount; + this.masterDataEngineInstanceCount = dataEngineInstanceCount; return this; } public SparkComputationalCreateGcp withDataEngineSlaveSize(String dataEngineSlaveSize) { - this.dataEngineSlaveSize = dataEngineSlaveSize; + this.slaveDtaEngineInstanceShape = dataEngineSlaveSize; return this; } public SparkComputationalCreateGcp withDataEngineMasterSize(String dataEngineMasterSize) { - this.dataEngineMasterSize = dataEngineMasterSize; + this.masterDataEngineInstanceShape = dataEngineMasterSize; return this; } - public SparkComputationalCreateGcp withConfig(List<ClusterConfig> config) { - this.config = config; + public SparkComputationalCreateGcp withSlaveDataEngineInstanceCount(String slaveDataEngineInstanceCount) { + this.slaveDataEngineInstanceCount = slaveDataEngineInstanceCount; return this; } - public SparkComputationalCreateGcp withSharedImageEnabled(String sharedImageEnabled) { - this.sharedImageEnabled = sharedImageEnabled; + public SparkComputationalCreateGcp withMasterGPUType(String masterGPUType) { + this.masterGPUType = masterGPUType; return this; } + public SparkComputationalCreateGcp withSlaveGPUType(String slaveGPUType) { + this.slaveGPUType = slaveGPUType; + return this; + } - @Override - public MoreObjects.ToStringHelper toStringHelper(Object self) { - return super.toStringHelper(self) - .add("dataEngineInstanceCount", dataEngineInstanceCount) - .add("dataEngineSlaveSize", dataEngineSlaveSize) - .add("dataEngineMasterSize", dataEngineMasterSize); + public SparkComputationalCreateGcp withMasterGPUCount(String masterGPUCount) { + this.masterGPUCount = masterGPUCount; + return this; } - @Override - public String toString() { - return toStringHelper(this).toString(); + public SparkComputationalCreateGcp withSlaveGPUCount(String slaveGPUCount) { + this.slaveGPUCount = slaveGPUCount; + return this; + } + + public SparkComputationalCreateGcp withConfig(List<ClusterConfig> config) { + this.config = config; + return this; + } + + public SparkComputationalCreateGcp withSharedImageEnabled(String sharedImageEnabled) { + this.sharedImageEnabled = sharedImageEnabled; + return this; } } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/SparkStandaloneClusterCreateForm.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/SparkStandaloneClusterCreateForm.java index 0178dff..4510c1f 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/SparkStandaloneClusterCreateForm.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/SparkStandaloneClusterCreateForm.java @@ -33,10 +33,33 @@ import org.hibernate.validator.constraints.NotBlank; public class SparkStandaloneClusterCreateForm extends ComputationalCreateFormDTO { @NotBlank - @JsonProperty("dataengine_instance_count") - private String dataEngineInstanceCount; + @JsonProperty("master_instance_count") + private String masterDataEngineInstanceCount; @NotBlank - @JsonProperty("dataengine_instance_shape") - private String dataEngineInstanceShape; + @JsonProperty("master_instance_shape") + private String masterDataEngineInstanceShape; + + @NotBlank + @JsonProperty("slave_instance_count") + private String slaveDataEngineInstanceCount; + + @NotBlank + @JsonProperty("slave_instance_shape") + private String slaveDtaEngineInstanceShape; + + @JsonProperty("gpu_enabled") + private Boolean enabledGPU; + + @JsonProperty("master_gpu_type") + private String masterGpuType; + + @JsonProperty("master_gpu_count") + private String masterGpuCount; + + @JsonProperty("slave_gpu_type") + private String slaveGpuType; + + @JsonProperty("slave_gpu_count") + private String slaveGpuCount; } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ComputationalServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ComputationalServiceImpl.java index b75ff62..9d023a9 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ComputationalServiceImpl.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ComputationalServiceImpl.java @@ -359,8 +359,15 @@ public class ComputationalServiceImpl implements ComputationalService { .imageName(form.getImage()) .templateName(form.getTemplateName()) .status(CREATING.toString()) - .dataEngineInstanceCount(form.getDataEngineInstanceCount()) - .dataEngineInstanceShape(form.getDataEngineInstanceShape()) + .masterDataEngineInstanceCount(form.getMasterDataEngineInstanceCount()) + .masterDataEngineInstanceShape(form.getMasterDataEngineInstanceShape()) + .slaveDataEngineInstanceCount(form.getSlaveDataEngineInstanceCount()) + .slaveDataEngineInstanceShape(form.getSlaveDtaEngineInstanceShape()) + .enabledGPU(form.getEnabledGPU()) + .masterGpuCount(form.getMasterGpuCount()) + .masterGpuType(form.getMasterGpuType()) + .slaveGpuCount(form.getSlaveGpuCount()) + .slaveGpuType(form.getSlaveGpuType()) .config(form.getConfig()) .build(); } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java index 5821d76..acb8add 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java @@ -391,17 +391,17 @@ public class RequestBuilder { switch (cloudProvider) { case AWS: computationalCreate = (T) newResourceSysBaseDTO(userInfo.getName(), cloudProvider, SparkComputationalCreateAws.class) - .withDataEngineInstanceCount(form.getDataEngineInstanceCount()) - .withDataEngineMasterShape(form.getDataEngineInstanceShape()) - .withDataEngineSlaveShape(form.getDataEngineInstanceShape()) + .withDataEngineInstanceCount(form.getMasterDataEngineInstanceCount()) + .withDataEngineMasterShape(form.getMasterDataEngineInstanceShape()) + .withDataEngineSlaveShape(form.getMasterDataEngineInstanceShape()) .withConfig(form.getConfig()) .withSharedImageEnabled(String.valueOf(projectDTO.isSharedImageEnabled())); break; case AZURE: computationalCreate = (T) newResourceSysBaseDTO(userInfo.getName(), cloudProvider, SparkComputationalCreateAzure.class) - .withDataEngineInstanceCount(form.getDataEngineInstanceCount()) - .withDataEngineMasterSize(form.getDataEngineInstanceShape()) - .withDataEngineSlaveSize(form.getDataEngineInstanceShape()) + .withDataEngineInstanceCount(form.getMasterDataEngineInstanceCount()) + .withDataEngineMasterSize(form.getMasterDataEngineInstanceShape()) + .withDataEngineSlaveSize(form.getMasterDataEngineInstanceShape()) .withConfig(form.getConfig()) .withSharedImageEnabled(String.valueOf(projectDTO.isSharedImageEnabled())); if (settingsDAO.isAzureDataLakeEnabled()) { @@ -415,9 +415,14 @@ public class RequestBuilder { break; case GCP: computationalCreate = (T) newResourceSysBaseDTO(userInfo.getName(), cloudProvider, SparkComputationalCreateGcp.class) - .withDataEngineInstanceCount(form.getDataEngineInstanceCount()) - .withDataEngineMasterSize(form.getDataEngineInstanceShape()) - .withDataEngineSlaveSize(form.getDataEngineInstanceShape()) + .withDataEngineInstanceCount(form.getMasterDataEngineInstanceCount()) + .withSlaveDataEngineInstanceCount(form.getSlaveDataEngineInstanceCount()) + .withDataEngineMasterSize(form.getMasterDataEngineInstanceShape()) + .withDataEngineSlaveSize(form.getMasterDataEngineInstanceShape()) + .withMasterGPUCount(form.getMasterGpuCount()) + .withMasterGPUType(form.getMasterGpuType()) + .withSlaveGPUCount(form.getSlaveGpuCount()) + .withSlaveGPUType(form.getSlaveGpuType()) .withConfig(form.getConfig()) .withSharedImageEnabled(String.valueOf(projectDTO.isSharedImageEnabled())); break; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
