This is an automated email from the ASF dual-hosted git repository. areusch pushed a commit to branch areusch/freeze-dependencies in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 0d3d8bc81ad57c06857186435f5a99887d15dc9d Author: Andrew Reusch <areu...@gmail.com> AuthorDate: Thu May 19 15:16:15 2022 -0700 Align Python and package install process in all containers. --- docker/Dockerfile.ci_arm | 13 ++-- docker/Dockerfile.ci_cpu | 48 ++++++++----- docker/Dockerfile.ci_gpu | 45 ++++++------ docker/Dockerfile.ci_hexagon | 13 ++-- docker/Dockerfile.ci_i386 | 19 +++--- docker/Dockerfile.ci_lint | 14 ++-- docker/Dockerfile.ci_qemu | 26 +++---- docker/Dockerfile.ci_wasm | 9 ++- docker/install/ubuntu1804_install_python.sh | 21 +++--- docker/install/ubuntu1804_install_python_venv.sh | 31 --------- docker/install/ubuntu2004_install_python.sh | 20 +++--- docker/install/ubuntu_install_caffe.sh | 3 - docker/install/ubuntu_install_core.sh | 3 +- docker/install/ubuntu_install_coreml.sh | 23 ------- docker/install/ubuntu_install_darknet.sh | 29 -------- docker/install/ubuntu_install_mxnet.sh | 23 ------- docker/install/ubuntu_install_oneflow.sh | 25 ------- docker/install/ubuntu_install_onnx.sh | 41 ----------- docker/install/ubuntu_install_paddle.sh | 23 ------- docker/install/ubuntu_install_python_package.sh | 41 ++++------- docker/install/ubuntu_install_redis.sh | 2 - docker/install/ubuntu_install_sphinx.sh | 33 --------- docker/install/ubuntu_install_tensorflow.sh | 26 ------- .../install/ubuntu_install_tensorflow_aarch64.sh | 8 --- docker/install/ubuntu_install_tflite.sh | 79 ---------------------- docker/install/ubuntu_install_vela.sh | 23 ------- 26 files changed, 134 insertions(+), 507 deletions(-) diff --git a/docker/Dockerfile.ci_arm b/docker/Dockerfile.ci_arm index c19f1ff5a4..11973aeca5 100644 --- a/docker/Dockerfile.ci_arm +++ b/docker/Dockerfile.ci_arm @@ -44,26 +44,23 @@ ENV PATH /opt/sccache:$PATH COPY install/ubuntu_install_llvm.sh /install/ubuntu_install_llvm.sh RUN bash /install/ubuntu_install_llvm.sh +ENV TVM_VENV /venv/apache-tvm-py3.7 +COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh RUN bash /install/ubuntu1804_install_python.sh -# Globally disable pip cache -RUN pip config set global.no-cache-dir false - COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh RUN bash /install/ubuntu_install_cmake_source.sh +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh RUN bash /install/ubuntu_install_python_package.sh -# TensorFlow deps +# TensorFlow C++ deps COPY install/ubuntu_install_tensorflow_aarch64.sh /install/ubuntu_install_tensorflow_aarch64.sh RUN bash /install/ubuntu_install_tensorflow_aarch64.sh -# TFLite deps -COPY install/ubuntu_install_tflite.sh /install/ubuntu_install_tflite.sh -RUN bash /install/ubuntu_install_tflite.sh - # Caffe deps COPY install/ubuntu_install_boost.sh /install/ubuntu_install_boost.sh RUN bash /install/ubuntu_install_boost.sh diff --git a/docker/Dockerfile.ci_cpu b/docker/Dockerfile.ci_cpu index 45943334a0..7eb8e2083f 100644 --- a/docker/Dockerfile.ci_cpu +++ b/docker/Dockerfile.ci_cpu @@ -18,22 +18,36 @@ # CI docker CPU env FROM ubuntu:18.04 -RUN apt-get update --fix-missing - COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh RUN bash /install/ubuntu_install_core.sh COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh RUN bash /install/ubuntu_install_googletest.sh +ENV TVM_VENV /venv/apache-tvm-py3.7 +COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh RUN bash /install/ubuntu1804_install_python.sh -# Globally disable pip cache -RUN pip config set global.no-cache-dir false - +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh -RUN bash /install/ubuntu_install_python_package.sh +RUN bash /install/ubuntu_install_python_package.sh \ + -E ethosu \ + -E importer-caffe \ + -E importer-caffe2 \ + -E importer-coreml \ + -E importer-darknet \ + -E importer-keras \ + -E importer-oneflow \ + -E importer-onnx \ + -E importer-paddle \ + -E importer-pytorch \ + -E importer-tensorflow \ + -E importer-tflite \ + -E tvmc \ + -E xgboost \ + ; COPY install/ubuntu1804_install_llvm.sh /install/ubuntu1804_install_llvm.sh RUN bash /install/ubuntu1804_install_llvm.sh @@ -44,9 +58,9 @@ RUN bash /install/ubuntu_install_dnnl.sh COPY install/ubuntu_install_papi.sh /install/ubuntu_install_papi.sh RUN bash /install/ubuntu_install_papi.sh "" -# Install MxNet for access to Gluon Model Zoo. -COPY install/ubuntu_install_mxnet.sh /install/ubuntu_install_mxnet.sh -RUN bash /install/ubuntu_install_mxnet.sh +# # Install MxNet for access to Gluon Model Zoo. +# COPY install/ubuntu_install_mxnet.sh /install/ubuntu_install_mxnet.sh +# RUN bash /install/ubuntu_install_mxnet.sh # Rust env (build early; takes a while) COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh @@ -89,12 +103,12 @@ COPY install/ubuntu_install_verilator.sh /install/ubuntu_install_verilator.sh RUN bash /install/ubuntu_install_verilator.sh # TensorFlow deps -COPY install/ubuntu_install_tensorflow.sh /install/ubuntu_install_tensorflow.sh -RUN bash /install/ubuntu_install_tensorflow.sh +# COPY install/ubuntu_install_tensorflow.sh /install/ubuntu_install_tensorflow.sh +# RUN bash /install/ubuntu_install_tensorflow.sh # TFLite deps -COPY install/ubuntu_install_tflite.sh /install/ubuntu_install_tflite.sh -RUN bash /install/ubuntu_install_tflite.sh +# COPY install/ubuntu_install_tflite.sh /install/ubuntu_install_tflite.sh +# RUN bash /install/ubuntu_install_tflite.sh # Compute Library COPY install/ubuntu_download_arm_compute_lib_binaries.sh /install/ubuntu_download_arm_compute_lib_binaries.sh @@ -132,15 +146,15 @@ COPY install/ubuntu_install_ethosu_driver_stack.sh /install/ubuntu_install_ethos RUN bash /install/ubuntu_install_ethosu_driver_stack.sh # Install Vela compiler -COPY install/ubuntu_install_vela.sh /install/ubuntu_install_vela.sh -RUN bash /install/ubuntu_install_vela.sh +# COPY install/ubuntu_install_vela.sh /install/ubuntu_install_vela.sh +# RUN bash /install/ubuntu_install_vela.sh # Update PATH ENV PATH /opt/arm/gcc-arm-none-eabi/bin:/opt/arm/FVP_Corstone_SSE-300/models/Linux64_GCC-6.4:$PATH # PaddlePaddle deps -COPY install/ubuntu_install_paddle.sh /install/ubuntu_install_paddle.sh -RUN bash /install/ubuntu_install_paddle.sh +# COPY install/ubuntu_install_paddle.sh /install/ubuntu_install_paddle.sh +# RUN bash /install/ubuntu_install_paddle.sh # sccache COPY install/ubuntu_install_sccache.sh /install/ubuntu_install_sccache.sh diff --git a/docker/Dockerfile.ci_gpu b/docker/Dockerfile.ci_gpu index 73d13007f1..c7c5d1b56e 100644 --- a/docker/Dockerfile.ci_gpu +++ b/docker/Dockerfile.ci_gpu @@ -25,7 +25,7 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/ # Base scripts RUN rm /etc/apt/sources.list.d/nvidia-ml.list && apt-get clean -RUN apt-get update --fix-missing +# NOTE: apt-get update --fix-missing is run by ubuntu_install_core.sh. COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh RUN bash /install/ubuntu_install_core.sh @@ -33,12 +33,11 @@ RUN bash /install/ubuntu_install_core.sh COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh RUN bash /install/ubuntu_install_googletest.sh +ENV TVM_VENV /venv/apache-tvm-py3.7 +COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh RUN bash /install/ubuntu1804_install_python.sh -# Globally disable pip cache -RUN pip config set global.no-cache-dir false - COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh RUN bash /install/ubuntu_install_cmake_source.sh @@ -48,11 +47,13 @@ RUN bash /install/ubuntu1804_install_llvm.sh COPY install/ubuntu_install_opencl.sh /install/ubuntu_install_opencl.sh RUN bash /install/ubuntu_install_opencl.sh +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh RUN bash /install/ubuntu_install_python_package.sh -COPY install/ubuntu_install_sphinx.sh /install/ubuntu_install_sphinx.sh -RUN bash /install/ubuntu_install_sphinx.sh +# COPY install/ubuntu_install_sphinx.sh /install/ubuntu_install_sphinx.sh +# RUN bash /install/ubuntu_install_sphinx.sh # Enable doxygen for c++ doc build RUN apt-get update && apt-get install -y doxygen libprotobuf-dev protobuf-compiler @@ -67,29 +68,29 @@ COPY install/ubuntu_install_rocm.sh /install/ubuntu_install_rocm.sh RUN bash /install/ubuntu_install_rocm.sh # DL Frameworks -COPY install/ubuntu_install_mxnet.sh /install/ubuntu_install_mxnet.sh -RUN bash /install/ubuntu_install_mxnet.sh +# COPY install/ubuntu_install_mxnet.sh /install/ubuntu_install_mxnet.sh +# RUN bash /install/ubuntu_install_mxnet.sh COPY install/ubuntu_install_gluoncv.sh /install/ubuntu_install_gluoncv.sh RUN bash /install/ubuntu_install_gluoncv.sh -COPY install/ubuntu_install_coreml.sh /install/ubuntu_install_coreml.sh -RUN bash /install/ubuntu_install_coreml.sh +# COPY install/ubuntu_install_coreml.sh /install/ubuntu_install_coreml.sh +# RUN bash /install/ubuntu_install_coreml.sh -COPY install/ubuntu_install_tensorflow.sh /install/ubuntu_install_tensorflow.sh -RUN bash /install/ubuntu_install_tensorflow.sh +# COPY install/ubuntu_install_tensorflow.sh /install/ubuntu_install_tensorflow.sh +# RUN bash /install/ubuntu_install_tensorflow.sh -COPY install/ubuntu_install_darknet.sh /install/ubuntu_install_darknet.sh -RUN bash /install/ubuntu_install_darknet.sh +# COPY install/ubuntu_install_darknet.sh /install/ubuntu_install_darknet.sh +# RUN bash /install/ubuntu_install_darknet.sh -COPY install/ubuntu_install_onnx.sh /install/ubuntu_install_onnx.sh -RUN bash /install/ubuntu_install_onnx.sh +# COPY install/ubuntu_install_onnx.sh /install/ubuntu_install_onnx.sh +# RUN bash /install/ubuntu_install_onnx.sh COPY install/ubuntu_install_libtorch.sh /install/ubuntu_install_libtorch.sh RUN bash /install/ubuntu_install_libtorch.sh -COPY install/ubuntu_install_tflite.sh /install/ubuntu_install_tflite.sh -RUN bash /install/ubuntu_install_tflite.sh +# COPY install/ubuntu_install_tflite.sh /install/ubuntu_install_tflite.sh +# RUN bash /install/ubuntu_install_tflite.sh COPY install/ubuntu_install_dgl.sh /install/ubuntu_install_dgl.sh RUN bash /install/ubuntu_install_dgl.sh @@ -98,12 +99,12 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,graphics,utility COPY install/ubuntu_install_vulkan.sh /install/ubuntu_install_vulkan.sh RUN bash /install/ubuntu_install_vulkan.sh -COPY install/ubuntu_install_paddle.sh /install/ubuntu_install_paddle.sh -RUN bash /install/ubuntu_install_paddle.sh +# COPY install/ubuntu_install_paddle.sh /install/ubuntu_install_paddle.sh +# RUN bash /install/ubuntu_install_paddle.sh # OneFlow deps -COPY install/ubuntu_install_oneflow.sh /install/ubuntu_install_oneflow.sh -RUN bash /install/ubuntu_install_oneflow.sh +# COPY install/ubuntu_install_oneflow.sh /install/ubuntu_install_oneflow.sh +# RUN bash /install/ubuntu_install_oneflow.sh # Rust env (build early; takes a while) COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh diff --git a/docker/Dockerfile.ci_hexagon b/docker/Dockerfile.ci_hexagon index ddca5c6c2e..d0a2d2facc 100644 --- a/docker/Dockerfile.ci_hexagon +++ b/docker/Dockerfile.ci_hexagon @@ -19,21 +19,20 @@ # tag: v0.02 FROM tvmcihexagon/ci-hexagon-base:v0.02_SDK4.5.0.3 -RUN apt-get update --fix-missing -RUN apt-get install -y ca-certificates gnupg2 libxml2-dev - COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh RUN bash /install/ubuntu_install_core.sh +# TODO: why do we need this? +# RUN apt-get install -y ca-certificates gnupg2 libxml2-dev + COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh RUN bash /install/ubuntu_install_googletest.sh +ENV TVM_VENV /venv/apache-tvm-py3.7 +COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles COPY install/ubuntu2004_install_python.sh /install/ubuntu2004_install_python.sh RUN bash /install/ubuntu2004_install_python.sh -# Globally disable pip cache -RUN pip config set global.cache-dir false - # Rust env (build early; takes a while) COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh RUN bash /install/ubuntu_install_rust.sh @@ -41,6 +40,8 @@ ENV RUSTUP_HOME /opt/rust ENV CARGO_HOME /opt/rust ENV PATH $PATH:$CARGO_HOME/bin +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh RUN bash /install/ubuntu_install_python_package.sh diff --git a/docker/Dockerfile.ci_i386 b/docker/Dockerfile.ci_i386 index 61ba064ff3..25eba3d7c4 100644 --- a/docker/Dockerfile.ci_i386 +++ b/docker/Dockerfile.ci_i386 @@ -20,20 +20,12 @@ FROM i386/ubuntu:18.04 -RUN apt-get update --fix-missing && apt-get install -y ca-certificates - COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh RUN bash /install/ubuntu_install_core.sh COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh RUN bash /install/ubuntu_install_googletest.sh -COPY install/ubuntu_install_llvm.sh /install/ubuntu_install_llvm.sh -RUN bash /install/ubuntu_install_llvm.sh - -COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh -RUN bash /install/ubuntu1804_install_python.sh - # Rust env (build early; takes a while) COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh RUN bash /install/ubuntu_install_rust.sh @@ -41,12 +33,19 @@ ENV RUSTUP_HOME /opt/rust ENV CARGO_HOME /opt/rust ENV PATH $PATH:$CARGO_HOME/bin -# Globally disable pip cache -RUN pip config set global.no-cache-dir false +ENV TVM_VENV /venv/apache-tvm-py3.7 +COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles +COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh +RUN bash /install/ubuntu1804_install_python.sh + +COPY install/ubuntu_install_llvm.sh /install/ubuntu_install_llvm.sh +RUN bash /install/ubuntu_install_llvm.sh COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh RUN bash /install/ubuntu_install_cmake_source.sh +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh RUN bash /install/ubuntu_install_python_package.sh diff --git a/docker/Dockerfile.ci_lint b/docker/Dockerfile.ci_lint index 1d0c984c61..6da29415f2 100644 --- a/docker/Dockerfile.ci_lint +++ b/docker/Dockerfile.ci_lint @@ -20,20 +20,20 @@ # tag: v0.60 FROM ubuntu:18.04 -RUN apt-get update --fix-missing - -RUN apt-get update && apt-get install -y wget git sudo make parallel +RUN apt-get update --fix-missing && apt-get install -y wget git sudo make parallel +ENV TVM_VENV /venv/apache-tvm-py3.7 +COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh RUN bash /install/ubuntu1804_install_python.sh -# Globally disable pip cache -RUN pip config set global.no-cache-dir false +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock +COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh +RUN bash /install/ubuntu_install_python_package.sh RUN apt-get update && apt-get install -y doxygen graphviz curl shellcheck -RUN pip3 install cpplint pylint==2.4.4 mypy==0.902 black==22.3.0 flake8==3.9.2 blocklint==0.2.3 jinja2==3.0.3 - # Rust env (build early; takes a while) COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh RUN bash /install/ubuntu_install_rust.sh diff --git a/docker/Dockerfile.ci_qemu b/docker/Dockerfile.ci_qemu index 28bfd8962d..0bbd1e454e 100644 --- a/docker/Dockerfile.ci_qemu +++ b/docker/Dockerfile.ci_qemu @@ -19,24 +19,19 @@ # tag: v0.62 FROM ubuntu:18.04 -RUN apt-get update --fix-missing - COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh RUN bash /install/ubuntu_install_core.sh COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh RUN bash /install/ubuntu_install_googletest.sh +ENV TVM_VENV /venv/apache-tvm-py3.7 +COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh RUN bash /install/ubuntu1804_install_python.sh -COPY install/ubuntu1804_install_python_venv.sh /install/ubuntu1804_install_python_venv.sh -RUN bash /install/ubuntu1804_install_python_venv.sh -ENV PATH=/opt/tvm-venv/bin:/opt/zephyr-sdk/sysroots/x86_64-pokysdk-linux/usr/bin:$PATH - -# Globally disable pip cache -RUN pip config set global.no-cache-dir false - +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh RUN bash /install/ubuntu_install_python_package.sh @@ -59,12 +54,12 @@ COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh RUN bash /install/ubuntu_install_java.sh # TensorFlow deps -COPY install/ubuntu_install_tensorflow.sh /install/ubuntu_install_tensorflow.sh -RUN bash /install/ubuntu_install_tensorflow.sh +# COPY install/ubuntu_install_tensorflow.sh /install/ubuntu_install_tensorflow.sh +# RUN bash /install/ubuntu_install_tensorflow.sh # TFLite deps -COPY install/ubuntu_install_tflite.sh /install/ubuntu_install_tflite.sh -RUN bash /install/ubuntu_install_tflite.sh +# COPY install/ubuntu_install_tflite.sh /install/ubuntu_install_tflite.sh +# RUN bash /install/ubuntu_install_tflite.sh # sccache COPY install/ubuntu_install_sccache.sh /install/ubuntu_install_sccache.sh @@ -77,6 +72,7 @@ COPY install/ubuntu_init_zephyr_project.sh /install/ubuntu_init_zephyr_project.s COPY install/ubuntu_install_zephyr_sdk.sh /install/ubuntu_install_zephyr_sdk.sh RUN bash /install/ubuntu_install_zephyr.sh ENV ZEPHYR_BASE=/opt/zephyrproject/zephyr +ENV PATH=/opt/zephyr-sdk/sysroots/x86_64-pokysdk-linux/usr/bin:$PATH # FreeRTOS deps COPY install/ubuntu_install_freertos.sh /install/ubuntu_install_freertos.sh @@ -92,8 +88,8 @@ COPY install/ubuntu_install_arduino.sh /install/ubuntu_install_arduino.sh RUN bash /install/ubuntu_install_arduino.sh # Install ONNX -COPY install/ubuntu_install_onnx.sh /install/ubuntu_install_onnx.sh -RUN bash /install/ubuntu_install_onnx.sh +# COPY install/ubuntu_install_onnx.sh /install/ubuntu_install_onnx.sh +# RUN bash /install/ubuntu_install_onnx.sh # Install CMSIS_NN COPY install/ubuntu_install_cmsis.sh /install/ubuntu_install_cmsis.sh diff --git a/docker/Dockerfile.ci_wasm b/docker/Dockerfile.ci_wasm index 1c7d3eb59b..243c3d53f7 100644 --- a/docker/Dockerfile.ci_wasm +++ b/docker/Dockerfile.ci_wasm @@ -16,20 +16,19 @@ # under the License. FROM ubuntu:18.04 -RUN apt-get update --fix-missing - COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh RUN bash /install/ubuntu_install_core.sh COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh RUN bash /install/ubuntu_install_googletest.sh +ENV TVM_VENV /venv/apache-tvm-py3.7 +COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh RUN bash /install/ubuntu1804_install_python.sh -# Globally disable pip cache -RUN pip config set global.no-cache-dir false - +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh RUN bash /install/ubuntu_install_python_package.sh diff --git a/docker/install/ubuntu1804_install_python.sh b/docker/install/ubuntu1804_install_python.sh index 94d316199d..cdef3389d5 100755 --- a/docker/install/ubuntu1804_install_python.sh +++ b/docker/install/ubuntu1804_install_python.sh @@ -32,15 +32,16 @@ trap cleanup 0 # instead modify install_python_package.sh apt-get update apt-get install -y software-properties-common -apt-get install -y python3.7 python3.7-dev python3-pip +# NOTE: There is no python3.7-pip, but whatever installed is upgraded anyway in a few lines. +apt-get install -y python3.7 python3.7-dev python3-pip python3.7-venv update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 -# Pin pip and setuptools versions -# Hashes generated via: -# $ pip download <package>==<version> -# $ pip hash --algorithm sha512 <package>.whl -cat <<EOF > base-requirements.txt -pip==19.3.1 --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7 -setuptools==58.4.0 --hash=sha256:e8b1d3127a0441fb99a130bcc3c2bf256c2d3ead3aba8fd400e5cbbaf788e036 -EOF -pip3 install -r base-requirements.txt +# Update pip to match version used to produce requirements-hashed.txt. This step +# is necessary so that pip's dependency solver is recent. +pip_spec=$(cat /install/python/bootstrap/lockfiles/constraints.txt | grep 'pip==') +pip3 install -U --require-hashes -r <(echo "${pip_spec}") \ + -c /install/python/bootstrap/lockfiles/constraints.txt +pip3 config set global.no-cache-dir false + +# Now install the remaining base packages. +pip3 install -r /install/python/bootstrap/lockfiles/constraints.txt diff --git a/docker/install/ubuntu1804_install_python_venv.sh b/docker/install/ubuntu1804_install_python_venv.sh deleted file mode 100755 index 5dc5efea76..0000000000 --- a/docker/install/ubuntu1804_install_python_venv.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u -set -o pipefail - -# install python and pip, don't modify this, modify install_python_package.sh -apt-get update -apt-get install -y software-properties-common -apt-get install -y python3.7-dev python3-setuptools python3.7-venv - -python3 -mvenv /opt/tvm-venv - -# Pin pip and setuptools versions -/opt/tvm-venv/bin/pip3 install pip==19.3.1 setuptools==58.4.0 diff --git a/docker/install/ubuntu2004_install_python.sh b/docker/install/ubuntu2004_install_python.sh index 5b87a74061..dbe6cfca33 100755 --- a/docker/install/ubuntu2004_install_python.sh +++ b/docker/install/ubuntu2004_install_python.sh @@ -31,15 +31,15 @@ trap cleanup 0 # instead modify install_python_package.sh apt-get update apt-get install -y software-properties-common -apt-get install -y python3.8 python3.8-dev python3-pip +apt-get install -y python3.8 python3.8-dev python3-pip python3.8-venv update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 -# Pin pip and setuptools versions -# Hashes generated via: -# $ pip download <package>==<version> -# $ pip hash --algorithm sha256 <package>.whl -cat <<EOF > base-requirements.txt -pip==22.0.4 --hash=sha256:c6aca0f2f081363f689f041d90dab2a07a9a07fb840284db2218117a52da800b -setuptools==58.4.0 --hash=sha256:e8b1d3127a0441fb99a130bcc3c2bf256c2d3ead3aba8fd400e5cbbaf788e036 -EOF -pip3 install -r base-requirements.txt +# Update pip to match version used to produce requirements-hashed.txt. This step +# is necessary so that pip's dependency solver is recent. +pip_spec=$(cat /install/python/bootstrap/lockfiles/constraints.txt | grep 'pip==') +pip3 install -U --require-hashes -r <(echo "${pip_spec}") \ + -c /install/python/bootstrap/lockfiles/constraints.txt +pip3 config set global.no-cache-dir false + +# Now install the remaining base packages. +pip3 install -r /install/python/bootstrap/lockfiles/constraints.txt diff --git a/docker/install/ubuntu_install_caffe.sh b/docker/install/ubuntu_install_caffe.sh index ab71eab54a..6867993c58 100755 --- a/docker/install/ubuntu_install_caffe.sh +++ b/docker/install/ubuntu_install_caffe.sh @@ -25,9 +25,6 @@ apt-get install -y --no-install-recommends protobuf-compiler \ libprotobuf-dev libhdf5-serial-dev libopenblas-dev libgflags-dev libgoogle-glog-dev -# install python packages -pip install "numpy" "protobuf" "scikit-image" "six" - # Build the Caffe and the python wrapper echo "Downloading Caffe" CAFFE_HOME="/opt/caffe" diff --git a/docker/install/ubuntu_install_core.sh b/docker/install/ubuntu_install_core.sh index 5593d61ea5..b577370f56 100755 --- a/docker/install/ubuntu_install_core.sh +++ b/docker/install/ubuntu_install_core.sh @@ -21,7 +21,8 @@ set -u set -o pipefail # install libraries for building c++ core on ubuntu -apt-get update && apt-get install -y --no-install-recommends \ +export DEBIAN_FRONTEND=noninteractive # avoid tzdata interactive config. +apt-get update --fix-missing && apt-get install -y --no-install-recommends \ apt-transport-https \ ca-certificates \ cmake \ diff --git a/docker/install/ubuntu_install_coreml.sh b/docker/install/ubuntu_install_coreml.sh deleted file mode 100755 index cbdc87666b..0000000000 --- a/docker/install/ubuntu_install_coreml.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u -set -o pipefail - -pip3 install coremltools diff --git a/docker/install/ubuntu_install_darknet.sh b/docker/install/ubuntu_install_darknet.sh deleted file mode 100755 index 8020899f8b..0000000000 --- a/docker/install/ubuntu_install_darknet.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u -set -o pipefail - -#install the necessary dependancies, cffi, opencv -wget -q 'https://github.com/siju-samuel/darknet/blob/master/lib/libdarknet.so?raw=true' -O libdarknet.so -debian_version=`cat /etc/debian_version` - -pip3 install \ - cffi \ - opencv-python diff --git a/docker/install/ubuntu_install_mxnet.sh b/docker/install/ubuntu_install_mxnet.sh deleted file mode 100755 index aa04d4c191..0000000000 --- a/docker/install/ubuntu_install_mxnet.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u -set -o pipefail - -pip3 install mxnet==1.6.0 diff --git a/docker/install/ubuntu_install_oneflow.sh b/docker/install/ubuntu_install_oneflow.sh deleted file mode 100755 index 3eb6b7d89b..0000000000 --- a/docker/install/ubuntu_install_oneflow.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u -set -o pipefail - -pip3 install flowvision==0.1.0 - -python3 -m pip install -f https://release.oneflow.info oneflow==0.7.0+cpu diff --git a/docker/install/ubuntu_install_onnx.sh b/docker/install/ubuntu_install_onnx.sh deleted file mode 100755 index 6a41a55740..0000000000 --- a/docker/install/ubuntu_install_onnx.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u -set -o pipefail - -# We need to fix the onnx version because changing versions tends to break tests -# TODO(mbrookhart): periodically update - -# onnx 1.9 removed onnx optimizer from the main repo (see -# https://github.com/onnx/onnx/pull/2834). When updating the CI image -# to onnx>=1.9, onnxoptimizer should also be installed. -pip3 install \ - onnx==1.10.2 \ - onnxruntime==1.9.0 \ - onnxoptimizer==0.2.6 - -# torch depends on a number of other packages, but unhelpfully, does -# not expose that in the wheel!!! -pip3 install future - -pip3 install \ - torch==1.11.0 \ - torchvision==0.12.0 \ - --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/docker/install/ubuntu_install_paddle.sh b/docker/install/ubuntu_install_paddle.sh deleted file mode 100755 index c7f9d43a3b..0000000000 --- a/docker/install/ubuntu_install_paddle.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u -set -o pipefail - -pip install paddlepaddle==2.1.3 diff --git a/docker/install/ubuntu_install_python_package.sh b/docker/install/ubuntu_install_python_package.sh index 0353814efc..6fa4b5cfc0 100755 --- a/docker/install/ubuntu_install_python_package.sh +++ b/docker/install/ubuntu_install_python_package.sh @@ -16,31 +16,18 @@ # specific language governing permissions and limitations # under the License. -set -e -set -u -set -o pipefail +set -euo pipefail +set -x -# install libraries for python package on ubuntu -pip3 install --upgrade \ - attrs \ - cloudpickle \ - cython \ - decorator \ - mypy \ - numpy~=1.19.5 \ - orderedset \ - packaging \ - Pillow==9.1.0 \ - psutil \ - pytest \ - tlcpack-sphinx-addon==0.2.1 \ - pytest-profiling \ - pytest-xdist \ - requests \ - scipy \ - Jinja2 \ - synr==0.6.0 \ - junitparser==2.4.2 \ - six \ - tornado \ - pytest-lazy-fixture +if [ -z "${TVM_VENV+x}" ]; then + echo "install script error: must set TVM_VENV to the path to the venv symlink" + exit 2 +fi + +cd $(dirname $0)/python +poetry config cache-dir /tmp/poetry-cache +poetry config virtualenvs.path /venv + +poetry install --no-root "$@" +VENV_ROOT=$(ls -d1 /venv/apache-tvm-*-py3.7) +ln -s "${VENV_ROOT}" "${TVM_VENV}" diff --git a/docker/install/ubuntu_install_redis.sh b/docker/install/ubuntu_install_redis.sh index 8678c20501..615415f557 100755 --- a/docker/install/ubuntu_install_redis.sh +++ b/docker/install/ubuntu_install_redis.sh @@ -21,5 +21,3 @@ set -u set -o pipefail apt-get update && apt-get install -y redis-server -pip3 install \ - xgboost==1.4.2 diff --git a/docker/install/ubuntu_install_sphinx.sh b/docker/install/ubuntu_install_sphinx.sh deleted file mode 100755 index 12ca25b22b..0000000000 --- a/docker/install/ubuntu_install_sphinx.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u -set -o pipefail - -# NOTE: install docutils < 0.17 to work around https://github.com/readthedocs/sphinx_rtd_theme/issues/1115 -pip3 install \ - autodocsumm \ - "commonmark>=0.7.3" \ - "docutils>=0.11,<0.17" \ - Image \ - matplotlib \ - sphinx==4.2.0 \ - sphinx_autodoc_annotation \ - sphinx-gallery==0.4.0 \ - sphinx_rtd_theme diff --git a/docker/install/ubuntu_install_tensorflow.sh b/docker/install/ubuntu_install_tensorflow.sh deleted file mode 100755 index eaf89ffcf8..0000000000 --- a/docker/install/ubuntu_install_tensorflow.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u -set -o pipefail - -pip3 install \ - "h5py==3.1.0" \ - keras==2.6 \ - tensorflow==2.6.2 diff --git a/docker/install/ubuntu_install_tensorflow_aarch64.sh b/docker/install/ubuntu_install_tensorflow_aarch64.sh index 6acf8b7270..cfb05ac1de 100755 --- a/docker/install/ubuntu_install_tensorflow_aarch64.sh +++ b/docker/install/ubuntu_install_tensorflow_aarch64.sh @@ -20,11 +20,3 @@ set -euxo pipefail # Build dependencies apt-get install -y --no-install-recommends libhdf5-dev - -# We're only using the TensorFlow wheel snapshot here as the -# h5py wheel tries to use the wrong .so file -pip3 install \ - "h5py==3.1.0" \ - keras==2.6 \ - tensorflow-aarch64==2.6.2 \ - -f https://snapshots.linaro.org/ldcg/python-cache/tensorflow-aarch64/ diff --git a/docker/install/ubuntu_install_tflite.sh b/docker/install/ubuntu_install_tflite.sh deleted file mode 100755 index 8a394302fd..0000000000 --- a/docker/install/ubuntu_install_tflite.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u -set -o pipefail - -# The tflite version should have matched versions to the tensorflow -# version installed from pip in ubuntu_install_tensorflow.sh -TENSORFLOW_VERSION=$(python3 -c "import tensorflow; print(tensorflow.__version__)" 2> /dev/null) - -# Download, build and install flatbuffers -git clone --branch=v1.12.0 --depth=1 --recursive https://github.com/google/flatbuffers.git -cd flatbuffers -cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -make install -j8 -cd .. - -# Install flatbuffers python packages. -pip3 install flatbuffers - -# Build the TFLite static library, necessary for building with TFLite ON. -# The library is built at: -# tensorflow/tensorflow/lite/tools/make/gen/*/lib/libtensorflow-lite.a. -git clone https://github.com/tensorflow/tensorflow --branch=v${TENSORFLOW_VERSION} --depth 1 -./tensorflow/tensorflow/lite/tools/make/download_dependencies.sh -./tensorflow/tensorflow/lite/tools/make/build_lib.sh - -# Setup tflite from schema -mkdir tflite -cp tensorflow/tensorflow/lite/schema/schema.fbs tflite -cd tflite -flatc --python schema.fbs - -cat <<EOM >setup.py -import setuptools - -setuptools.setup( - name="tflite", - version="${TENSORFLOW_VERSION}", - author="google", - author_email="goo...@google.com", - description="TFLite", - long_description="TFLite", - long_description_content_type="text/markdown", - url="https://www.tensorflow.org/lite", - packages=setuptools.find_packages(), - classifiers=[ - "Programming Language :: Python :: 2", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - ], -) -EOM - -cat <<EOM >__init__.py -name = "tflite" -EOM - -# Install tflite over python3 -python3 setup.py install - -cd .. -rm -rf tflite diff --git a/docker/install/ubuntu_install_vela.sh b/docker/install/ubuntu_install_vela.sh deleted file mode 100755 index c72d118233..0000000000 --- a/docker/install/ubuntu_install_vela.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u -set -o pipefail - -pip3 install ethos-u-vela==3.2.0