This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch develop in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 7fb8160c5f8d4245f74e7c31319e6d392b379a32 Merge: 6f52d76 d7e100c Author: leonidfrolov <[email protected]> AuthorDate: Thu Mar 10 18:10:33 2022 +0200 [DATALAB]: merged tpcp deployment branch .../src/general/conf/datalab.ini | 6 +- .../general/files/aws/tensor-jupyterlab_Dockerfile | 45 ++ .../files/aws/tensor-jupyterlab_description.json | 22 + .../src/general/lib/os/fab.py | 41 ++ .../src/general/scripts/aws/edge_configure.py | 6 +- .../src/general/scripts/aws/ssn_configure.py | 1 + .../scripts/aws/tensor-jupyterlab_configure.py | 338 +++++++++ .../general/scripts/os/common_clean_instance.py | 26 + .../aws/jenkins_jobs/create_data_engine/config.xml | 1 + .../jenkins_jobs/create_notebook_server/config.xml | 1 + .../jenkins_jobs/start_notebook_server/config.xml | 1 + .../jenkins_jobs/stop_notebook_server/config.xml | 1 + .../terminate_notebook_server/config.xml | 1 + .../os/debian/jupyterlab-notebook.service | 25 +- .../src/ssn/files/aws/mongo_roles.json | 10 + .../src/ssn/scripts/docker_build.py | 15 +- .../src/tensor-jupyterlab/fabfile.py | 254 +++++++ .../scripts/configure_tensor-jupyterlab_node.py | 170 +++++ .../terraform/bin/deploy/endpoint_fab.py | 2 +- .../terraform/gcp/endpoint/provisioning.py | 7 + integration-tests-cucumber/pom.xml | 133 ---- .../java/org/apache/datalab/dto/EndpointDTO.java | 38 - .../org/apache/datalab/mongo/MongoDBHelper.java | 33 - .../org/apache/datalab/util/JacksonMapper.java | 35 - .../org/apache/datalab/util/PropertyHelper.java | 42 -- .../src/test/java/datalab/Constants.java | 24 - .../src/test/java/datalab/RunCucumberTest.java | 29 - .../test/java/datalab/endpoint/EndpointSteps.java | 116 --- .../src/test/java/datalab/login/LoginSteps.java | 63 -- .../src/test/resources/config.properties | 20 - .../src/test/resources/datalab/endpoint.feature | 75 -- integration-tests/README.MD | 80 --- .../deeplearning/deeplearning-notebook.json | 7 - .../deeplearning/spark_cluster.json | 8 - .../azure_templates/jupyter/jupyter-notebook.json | 7 - .../azure_templates/jupyter/spark_cluster.json | 8 - .../azure_templates/rstudio/rstudio-notebook.json | 7 - .../azure_templates/rstudio/spark_cluster.json | 8 - .../azure_templates/tensor/spark_cluster.json | 8 - .../azure_templates/tensor/tensor-notebook.json | 7 - .../azure_templates/zeppelin/spark_cluster.json | 8 - .../zeppelin/zeppelin-notebook.json | 7 - integration-tests/examples/config.properties | 86 --- integration-tests/examples/copy_files.py | 114 --- .../examples/ec2_templates/deeplearning/EMR.json | 10 - .../ec2_templates/deeplearning/EMR_spot.json | 12 - .../deeplearning/deeplearning-notebook.json | 7 - .../ec2_templates/deeplearning/spark_cluster.json | 8 - .../examples/ec2_templates/jupyter/EMR.json | 10 - .../examples/ec2_templates/jupyter/EMR_spot.json | 12 - .../ec2_templates/jupyter/jupyter-notebook.json | 7 - .../ec2_templates/jupyter/spark_cluster.json | 8 - .../examples/ec2_templates/rstudio/EMR.json | 10 - .../examples/ec2_templates/rstudio/EMR_spot.json | 12 - .../ec2_templates/rstudio/rstudio-notebook.json | 7 - .../ec2_templates/rstudio/spark_cluster.json | 8 - .../examples/ec2_templates/tensor/EMR.json | 10 - .../examples/ec2_templates/tensor/EMR_spot.json | 12 - .../ec2_templates/tensor/spark_cluster.json | 8 - .../ec2_templates/tensor/tensor-notebook.json | 7 - .../examples/ec2_templates/zeppelin/EMR.json | 10 - .../examples/ec2_templates/zeppelin/EMR_spot.json | 12 - .../ec2_templates/zeppelin/spark_cluster.json | 8 - .../ec2_templates/zeppelin/zeppelin-notebook.json | 7 - .../gcp_templates/deeplearning/dataproc.json | 12 - .../deeplearning/deeplearning-notebook.json | 7 - .../gcp_templates/deeplearning/spark_cluster.json | 8 - .../examples/gcp_templates/jupyter/dataproc.json | 12 - .../gcp_templates/jupyter/jupyter-notebook.json | 7 - .../gcp_templates/jupyter/spark_cluster.json | 8 - .../examples/gcp_templates/rstudio/dataproc.json | 12 - .../gcp_templates/rstudio/rstudio-notebook.json | 7 - .../gcp_templates/rstudio/spark_cluster.json | 8 - .../examples/gcp_templates/tensor/dataproc.json | 12 - .../gcp_templates/tensor/spark_cluster.json | 8 - .../gcp_templates/tensor/tensor-notebook.json | 7 - .../examples/gcp_templates/zeppelin/dataproc.json | 12 - .../gcp_templates/zeppelin/spark_cluster.json | 8 - .../gcp_templates/zeppelin/zeppelin-notebook.json | 7 - .../scenario_deeplearning/deeplearning_tests.py | 141 ---- .../examples/scenario_jupyter/jupyter_tests.py | 102 --- .../examples/scenario_rstudio/rstudio_tests.py | 95 --- .../examples/scenario_tensor/tensor_tests.py | 91 --- .../examples/scenario_zeppelin/zeppelin_tests.py | 180 ----- .../test_libs/deeplearning/lib_groups.json | 5 - .../examples/test_libs/deeplearning/lib_list.json | 14 - .../examples/test_libs/jupyter/lib_groups.json | 6 - .../examples/test_libs/jupyter/lib_list.json | 18 - .../examples/test_libs/lib_groups.json | 5 - integration-tests/examples/test_libs/lib_list.json | 6 - .../examples/test_libs/rstudio/lib_groups.json | 6 - .../examples/test_libs/rstudio/lib_list.json | 18 - .../examples/test_libs/tensor/lib_groups.json | 5 - .../examples/test_libs/tensor/lib_list.json | 14 - .../examples/test_libs/zeppelin/lib_groups.json | 6 - .../examples/test_libs/zeppelin/lib_list.json | 19 - .../examples/test_templates/README.txt | 2 - .../test_templates/deeplearning/cat_gray.jpg | Bin 92726 -> 0 bytes .../test_templates/deeplearning/conv.prototxt | 48 -- .../deeplearning/template_caffe.ipynb | 99 --- .../deeplearning/template_caffe2.ipynb | 619 ---------------- .../deeplearning/template_cntk.ipynb | 78 -- .../deeplearning/template_keras.ipynb | 64 -- .../deeplearning/template_mxnet.ipynb | 108 --- .../deeplearning/template_preparation_tensor.ipynb | 285 -------- .../deeplearning/template_theano.ipynb | 45 -- .../deeplearning/template_torch.ipynb | 73 -- .../template_visualization_tensor.ipynb | 139 ---- .../jupyter/template_preparation_pyspark.ipynb | 198 ------ .../jupyter/template_preparation_spark.ipynb | 103 --- .../jupyter/template_preparation_sparkr.ipynb | 111 --- .../jupyter/template_visualization_pyspark.ipynb | 243 ------- .../jupyter/template_visualization_sparkr.ipynb | 249 ------- .../test_templates/rstudio/template_preparation.r | 44 -- .../rstudio/template_visualization.r | 122 ---- .../tensor/template_preparation_tensor.ipynb | 285 -------- .../tensor/template_visualization_tensor.ipynb | 139 ---- .../zeppelin/template_preparation_pyspark.json | 1 - .../zeppelin/template_preparation_spark.json | 1 - .../zeppelin/template_preparation_sparkr.json | 1 - .../zeppelin/template_visualization_pyspark.json | 1 - .../zeppelin/template_visualization_sparkr.json | 1 - .../cloud/VirtualMachineStatusChecker.java | 131 ---- .../datalab/automation/cloud/aws/AmazonHelper.java | 153 ---- .../automation/cloud/aws/AmazonInstanceState.java | 32 - .../automation/cloud/azure/AzureHelper.java | 154 ---- .../datalab/automation/cloud/gcp/GcpHelper.java | 245 ------- .../automation/cloud/gcp/GcpInstanceState.java | 33 - .../epam/datalab/automation/docker/AckStatus.java | 53 -- .../com/epam/datalab/automation/docker/Bridge.java | 160 ----- .../com/epam/datalab/automation/docker/Docker.java | 111 --- .../datalab/automation/docker/DockerContainer.java | 171 ----- .../epam/datalab/automation/docker/HostConfig.java | 37 - .../com/epam/datalab/automation/docker/Labels.java | 23 - .../datalab/automation/docker/NetworkSettings.java | 36 - .../epam/datalab/automation/docker/Networks.java | 34 - .../epam/datalab/automation/docker/SSHConnect.java | 136 ---- .../automation/exceptions/CloudException.java | 29 - .../automation/exceptions/DockerException.java | 29 - .../automation/exceptions/JenkinsException.java | 29 - .../automation/exceptions/LoadFailException.java | 29 - .../datalab/automation/helper/CloudHelper.java | 219 ------ .../datalab/automation/helper/CloudProvider.java | 31 - .../automation/helper/ConfigPropertyValue.java | 387 ---------- .../datalab/automation/helper/NamingHelper.java | 283 -------- .../automation/helper/PropertiesResolver.java | 217 ------ .../datalab/automation/helper/WaitForStatus.java | 205 ------ .../com/epam/datalab/automation/http/ApiPath.java | 70 -- .../epam/datalab/automation/http/ContentType.java | 30 - .../epam/datalab/automation/http/HttpRequest.java | 91 --- .../datalab/automation/http/HttpStatusCode.java | 31 - .../jenkins/JenkinsConfigProperties.java | 48 -- .../jenkins/JenkinsResponseElements.java | 29 - .../datalab/automation/jenkins/JenkinsService.java | 225 ------ .../datalab/automation/jenkins/JenkinsUrls.java | 30 - .../automation/model/CreateNotebookDto.java | 86 --- .../datalab/automation/model/DeployClusterDto.java | 67 -- .../automation/model/DeployDataProcDto.java | 109 --- .../datalab/automation/model/DeployEMRDto.java | 109 --- .../datalab/automation/model/DeploySparkDto.java | 64 -- .../automation/model/ExploratoryImageDto.java | 72 -- .../epam/datalab/automation/model/ImageDto.java | 92 --- .../datalab/automation/model/JsonMapperDto.java | 61 -- .../com/epam/datalab/automation/model/Lib.java | 66 -- .../epam/datalab/automation/model/LoginDto.java | 59 -- .../datalab/automation/model/NotebookConfig.java | 209 ------ .../src/main/resources/application.properties | 36 - integration-tests/src/main/resources/log4j2.xml | 63 -- .../epam/datalab/automation/test/TestCallable.java | 783 --------------------- .../automation/test/TestDataEngineService.java | 419 ----------- .../epam/datalab/automation/test/TestServices.java | 327 --------- .../test/libs/LibraryNotFoundException.java | 29 - .../datalab/automation/test/libs/LibsHelper.java | 62 -- .../automation/test/libs/TestDescription.java | 31 - .../automation/test/libs/TestLibGroupStep.java | 104 --- .../automation/test/libs/TestLibInstallStep.java | 162 ----- .../automation/test/libs/TestLibListStep.java | 129 ---- .../datalab/automation/test/libs/TestLibStep.java | 50 -- .../test/libs/models/LibInstallRequest.java | 55 -- .../test/libs/models/LibSearchRequest.java | 53 -- .../test/libs/models/LibStatusResponse.java | 87 --- .../test/libs/models/LibToSearchData.java | 37 - .../automation/test/libs/models/LibraryStatus.java | 95 --- integration-tests/src/test/resources/log4j2.xml | 63 -- .../org.mockito.plugins.MockMaker | 1 - .../core/commands/CommandExecutorMock.java | 3 +- .../backendapi/domain/NotebookTemplate.java | 1 + .../service/impl/LibraryServiceImpl.java | 2 +- .../epam/datalab/backendapi/util/BillingUtils.java | 2 +- .../src/main/resources/mongo/aws/mongo_roles.json | 12 + .../resources-grid/resources-grid.component.html | 6 +- 191 files changed, 972 insertions(+), 12201 deletions(-) diff --cc infrastructure-provisioning/src/general/conf/datalab.ini index fe84416,ae1ac4f..3f9d8e5 --- a/infrastructure-provisioning/src/general/conf/datalab.ini +++ b/infrastructure-provisioning/src/general/conf/datalab.ini @@@ -362,8 -350,8 +362,10 @@@ gcs_connector_version = 2.0. setuptools_version = 54.1.1 ### Roxygen2 version roxygen2_version = 7.1.1 +### Nbconvert version +nbconvert_version = 5.6.1 + ### jupyterlab version + jupyterlab_version = 3.2.9 #--- [emr] section contains all parameters that are using for emr provisioning ---# [emr] diff --cc services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java index ad5e954,62a3ab8..fe83453 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java @@@ -27,11 -27,12 +27,12 @@@ import lombok.Getter public enum NotebookTemplate { JUPYTER("Jupyter notebook 6.1.6"), JUPYTER_LAB("JupyterLab 0.35.6"), - ZEPPELIN("Apache Zeppelin 0.9.0"), + ZEPPELIN("Apache Zeppelin 0.9.1"), DEEP_LEARNING("Deep Learning 2.4"), TENSOR("Jupyter with TensorFlow 2.5.0"), + TENSOR_JUPYTERLAB("Jupyterlab with TensorFlow 2.5.0"), TENSOR_RSTUDIO("RStudio with TensorFlow 2.3.2"), - RSTUDIO("RStudio 1.4.1103"), + RSTUDIO("RStudio 2021.09.1-372"), TENSOR_GCP("Jupyter with TensorFlow 2.1.0"), DEEP_LEARNING_GCP("Deeplearning notebook"), DEEP_LEARNING_AWS("Deep Learning AMI Version 42.1"), --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
