This is an automated email from the ASF dual-hosted git repository. areusch pushed a commit to branch areusch/freeze-dependencies in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 705558fa2abfe877c3327e7f052154ea4c0309d9 Author: Andrew Reusch <areu...@gmail.com> AuthorDate: Thu May 19 15:01:03 2022 -0700 Add freeze_deps tool and modify gen_requirements to work with it. --- docker/python/ci-constraints.txt | 40 ++++ docker/python/freeze-dependencies.sh | 13 ++ docker/python/freeze_deps.py | 260 ++++++++++++++++++++++++++ pyproject.toml | 21 +++ python/gen_requirements.py | 343 +++++++++++++++++++++++------------ 5 files changed, 559 insertions(+), 118 deletions(-) diff --git a/docker/python/ci-constraints.txt b/docker/python/ci-constraints.txt new file mode 100644 index 0000000000..258d4bd4bd --- /dev/null +++ b/docker/python/ci-constraints.txt @@ -0,0 +1,40 @@ +# This file lists packages we intentionally hold back in CI for no reason other than that +# updates outside of these bounds require a considerable amount of work, and allowing them to float +# freely would mean that small changes to the TVM dependency set could be held up behind large +# migration tasks if a new version of these packages were to be released. Holding packages back +# here allows us to decide when to tackle such migration work. +#keras = "^2.6.0" +#mxnet = "^1.6.0" + +black = "<21.8b0" # Breaks tensorflow-gpu. Revisit when tensorflow is upgraded. +blocklint = "==0.2.3" +#commonmark = ">=0.7.3" +cpplint = "==1.6.0" +#docutils = ">=0.11,<0.17" +#ethos-u-vela = "==3.2.0" +flake8 = "==3.9.2" +flowvision = "==0.1.0" +#h5py = "==3.1.0" +keras = "==2.6" +jinja2 = "==3.0.3" +mxnet = "==1.6.0" +mypy = "==0.902" +oneflow = "==0.7.0" +onnx = "==1.10.2" +onnxruntime = "==1.9.0" +onnxoptimizer = "==0.2.6" +numpy = "==1.19.3" +paddlepaddle = "==2.1.3" +pillow = "==9.1.0" +pylint = "==2.4.4" +scipy = "==1.7.3" +sphinx = "==4.2.0" +sphinx-gallery = "==0.4.0" +tensorflow = "==2.6.2" +tensorflow-aarch64 = "==2.6.2" +tensorflow-estimator = "==2.6.0" +tensorflow-gpu = "==2.6.2" +tflite = "==2.4.0" +torch = "==1.11.0" +torchvision = "==0.12.0" +#xgboost = "==1.4.2" diff --git a/docker/python/freeze-dependencies.sh b/docker/python/freeze-dependencies.sh new file mode 100755 index 0000000000..9aabb7feb3 --- /dev/null +++ b/docker/python/freeze-dependencies.sh @@ -0,0 +1,13 @@ +#!/bin/bash -eux + +# Build base images (one per Python architecture) used in building the remaining TVM docker images. +set -eux + +cd "$(dirname "$0")/../.." + +# NOTE: working dir inside docker is repo root. +docker/bash.sh -i "${BUILD_TAG}.ci_py_deps:latest" python3 docker/python/freeze_deps.py \ + --ci-constraints=docker/python/ci-constraints.txt \ + --gen-requirements-py=python/gen_requirements.py \ + --template-pyproject-toml=pyproject.toml \ + --output-base=docker/python/build diff --git a/docker/python/freeze_deps.py b/docker/python/freeze_deps.py new file mode 100644 index 0000000000..01abff5ee6 --- /dev/null +++ b/docker/python/freeze_deps.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +import argparse +import importlib +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import typing + + +SECTION_RE = re.compile(r'\[([^]]+)\].*') + + +def remove_sections(lines : typing.List[str], section_names : typing.List[str]) -> typing.List[str]: + """Remove .toml sections from a list of lines. + + Parameters + ---------- + lines : list[str] + A list containing the lines of the toml file. + section_names : list[str] + A list of names of sections which should be removed. + + Returns + ------- + (removed, insert_points): + A 2-tuple. `removed` is a new list of strings with those sections removed. `insert_points` is + a dict containing an entry for each section removed; key is the section name and value is the + index into `removed` where that section would have been. + """ + removed = [] + insert_points = {} + drop_line = False + for line in lines: + m = SECTION_RE.match(line) + if m: + drop_line = m.group(1) in section_names + insert_points[m.group(1)] = len(removed) + + if not drop_line: + removed.append(line) + + return removed, insert_points + + +def write_dependencies(requirements_by_piece : dict, constraints : dict, output_f): + """Write the [tool.poetry.dependencies] section of pyproject.toml. + + Parameters + ---------- + requirements_by_piece : dict + The REQUIREMENTS_BY_PIECE dict from gen_requirements.py module. + constraints : dict + The CONSTRAINTS dict from gen_requirements.py module, updated with additional constraints from + ci-constraints.txt. + output_f : File + A file-like object where the section should be written. + """ + output_f.write("[tool.poetry.dependencies]\n" + 'python = ">=3.7, <3.9"\n') + core_packages = set(requirements_by_piece["core"][1]) + dev_packages = set(requirements_by_piece["dev"][1]) + + for package, constraint in constraints.items(): + if package in dev_packages: + continue + + optional = package not in core_packages + marker = f', markers = "{constraint.environment_marker}"' if constraint.environment_marker else '' + output_f.write( + f"{package} = {{ version = \"{constraint.constraint or '*'}\", optional = {str(optional).lower()}{marker} }}\n") + + output_f.write("\n") + + +def write_dev_dependencies(requirements_by_piece : dict, constraints : dict, output_f): + """Write the [tool.poetry.dev-dependencies] section of pyproject.toml. + + Parameters + ---------- + requirements_by_piece : dict + The REQUIREMENTS_BY_PIECE dict from gen_requirements.py module. + constraints : dict + The CONSTRAINTS dict from gen_requirements.py module, updated with additional constraints from + ci-constraints.txt. + output_f : File + A file-like object where the section should be written. + """ + output_f.write("[tool.poetry.dev-dependencies]\n") + dev_packages = set(requirements_by_piece["dev"][1]) + + for package, constraint in constraints.items(): + if package not in dev_packages: + continue + + output_f.write(f"{package} = \"{constraint.constraint or '*'}\"\n") + + output_f.write("\n") + + +def write_extras(requirements_by_piece : dict, constraints : dict, output_f): + """Write the [tool.poetry.extras] section of pyproject.toml. + + Parameters + ---------- + requirements_by_piece : dict + The REQUIREMENTS_BY_PIECE dict from gen_requirements.py module. + constraints : dict + The CONSTRAINTS dict from gen_requirements.py module, updated with additional constraints from + ci-constraints.txt. + output_f : File + A file-like object where the section should be written. + """ + output_f.write("[tool.poetry.extras]\n") + + for piece, (description, packages) in requirements_by_piece.items(): + if piece in ("core", "dev"): + # These pieces do not need an extras declaration. + continue + + output_f.write(f"# {description}\n") + package_list = ", ".join(f'"{p}"' for p in sorted(packages)) + output_f.write(f"{piece} = [{package_list}]\n\n") + + output_f.write("\n") + + +# List of all the emitted sections in order they are to be emitted. +SECTION_ORDER = ("tool.poetry.dependencies", "tool.poetry.dev-dependencies", "tool.poetry.extras") + + +CI_CONSTRAINTS_RE = re.compile(r'(?P<package_name>[a-zA-Z0-9_-]+) = "(?P<version>[^"]+)".*') + + +def generate_pyproject_toml(ci_constraints_txt : pathlib.Path, gen_requirements_py : pathlib.Path, + template_pyproject_toml : pathlib.Path, + output_pyproject_toml : pathlib.Path): + """Generate poetry dependencies sections in pyproject.toml from gen_requirements.py. + + Existing [tool.poetry.dev-dependencies], [tool.poetry.dependencies], and [tool.poetry.extras] + sections are overwritten. + + Parameters + ---------- + ci_constraints_txt : pathlib.Path + Path to ci-constraints.txt. + gen_requirements_py : pathlib.Path + Path to the python/gen_requirements.py file in TVM. + template_pyproject_toml : pathlib.Path + Path to a pyproject.toml whose [{dev-,}dependencies] sections should be replaced with those from + gen_requirements.py. In production, this is expected to be the checked-in pyproject.toml at + the root of the TVM repo. + output_pyproject_toml : pathlib.Path + Non-existent path to the revised pyproject.toml. + """ + with open(template_pyproject_toml) as template_f: + pyproject_toml, insert_points = remove_sections(template_f, SECTION_ORDER) + + insert_points = {s: insert_points.get(s, len(pyproject_toml)) for s in SECTION_ORDER} + + sys.path.insert(0, str(gen_requirements_py.resolve().parent)) + gen_requirements = importlib.import_module(gen_requirements_py.stem) + sys.path.pop(0) + + constraints_list = [] + for pkg, constraint in gen_requirements.CONSTRAINTS: + gen_requirements.parse_constraint_entry(pkg, constraint, None, constraints_list) + + constraints = {r.package: r for r in constraints_list} + with open(ci_constraints_txt) as ci_constraints_f: + for i, line in enumerate(ci_constraints_f): + if not line.strip(): + continue + + m = CI_CONSTRAINTS_RE.match(line) + if not m: + if line.startswith("#"): + continue + print(f"{ci_constraints_txt}: {i}: Malformed line {line}") + sys.exit(2) + + package_name = m.group("package_name") + if package_name not in constraints: + print(f"{ci_constraints_txt}: {i}: Package {package_name} not listed in gen_requirements.py") + sys.exit(2) + + constraint = constraints[package_name] + if constraint.constraint != "==*": + print(f"{ci_constraints_txt}: {i}: Package {package_name} already functionally constrained in gen_requirements.py") + sys.exit(2) + + constraints[package_name] = gen_requirements.Requirement(constraint.package, m.group("version"), constraint.environment_marker) + + stop_points = list(sorted([(v, k) for k, v in insert_points.items()], key=lambda x: (x[0], SECTION_ORDER.index(x[1])))) + next_stop = stop_points.pop(0) + with open(output_pyproject_toml, "w") as output_f: + def _write(next_stop, i): + while next_stop[0] == i: + writer_function_name = f"write_{next_stop[1][len('tool.poetry.'):].replace('-', '_')}" + globals()[writer_function_name](dict(gen_requirements.REQUIREMENTS_BY_PIECE), constraints, output_f) + next_stop = stop_points.pop(0) if stop_points else (None, "") + + return next_stop + + for i, line in enumerate(pyproject_toml): + next_stop = _write(next_stop, i) + output_f.write(line) + + next_stop = _write(next_stop, len(pyproject_toml)) + assert next_stop[0] is None, f"Did not write all sections. Remaining: {next_stop}" + + +def freeze_deps(output_pyproject_toml): + with open(output_pyproject_toml.parent / "poetry-lock.log", "w") as f: + # Disable parallel fetching which tends to result in "Connection aborted" errors. + # https://github.com/python-poetry/poetry/issues/3219 + subprocess.check_call(["poetry", "config", "installer.parallel", "false"], cwd=output_pyproject_toml.parent) + subprocess.check_call(["poetry", "lock", "-vv"], stdout=f, stderr=subprocess.STDOUT, cwd=output_pyproject_toml.parent) + + +REPO_ROOT = pathlib.Path(__file__).parent.parent + + +def parse_args(argv : typing.List[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(argv[0], usage="Create a pyproject.toml containing the information in python/gen_requirements.py") + parser.add_argument("--ci-constraints", + type=pathlib.Path, default=REPO_ROOT / "docker/ci-constraints.txt", + help=("Path to a file describing packages held back in " + "CI to make routine package updates possible.")) + parser.add_argument("--gen-requirements-py", + type=pathlib.Path, default=REPO_ROOT / "python" / "gen_requirements.py", + help="Path to python/gen_requirements.py in the TVM repo") + parser.add_argument("--template-pyproject-toml", + type=pathlib.Path, + help="Path to the pyproject.toml to use as a basis for the updated pyproject.toml.") + parser.add_argument("--output-base", + type=pathlib.Path, + help="Path where the updated pyproject.toml and poetry.lock should be written.") + + return parser.parse_args(argv[1:]) + + +def main(argv : typing.List[str]): + args = parse_args(argv) + + if args.output_base.exists(): + shutil.rmtree(args.output_base) + args.output_base.mkdir(parents=True) + + pyproject_toml = pathlib.Path(args.output_base) / "pyproject.toml" + generate_pyproject_toml(args.ci_constraints, args.gen_requirements_py, args.template_pyproject_toml, pyproject_toml) + with open(pyproject_toml) as f: + print(f.read()) + freeze_deps(pyproject_toml) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/pyproject.toml b/pyproject.toml index 5cca711ddb..65444820e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,3 +46,24 @@ exclude = ''' )/ ) ''' + +[tool.poetry] +name = "apache-tvm" +authors = [] +version = "0.8.0" +description = "Open source Deep Learning compliation toolkit" + +[[tool.poetry.source]] +name = "oneflow" +url = "https://release.oneflow.info" +secondary = true + +#[[tool.poetry.source]] +#name = "onnx" +#url = "https://download.pytorch.org/whl/cpu" +#secondary = true + +[[tool.poetry.source]] +name = "tensorflow-aarch64" +url = "https://snapshots.linaro.org/ldcg/python-cache" +secondary = true diff --git a/python/gen_requirements.py b/python/gen_requirements.py index 6cb92921f3..45aab10295 100755 --- a/python/gen_requirements.py +++ b/python/gen_requirements.py @@ -45,14 +45,15 @@ The data representing each piece is contained in the two maps below. import argparse import collections +import dataclasses import os import re import textwrap import sys -import typing +from typing import Dict, List, Pattern, Tuple, Union -RequirementsByPieceType = typing.List[typing.Tuple[str, typing.Tuple[str, typing.List[str]]]] +RequirementsByPieceType = List[Tuple[str, Tuple[str, List[str]]]] # Maps named TVM piece (see description above) to a list of names of Python packages. Please use @@ -85,6 +86,13 @@ REQUIREMENTS_BY_PIECE: RequirementsByPieceType = [ ], ), ), + ( + "gpu", + ( + "Requirements for working with GPUs", + [], # NOTE: tensorflow-gpu installed via environment marker. + ), + ), # Relay frontends. ( "importer-caffe", @@ -112,7 +120,17 @@ REQUIREMENTS_BY_PIECE: RequirementsByPieceType = [ ("importer-darknet", ("Requirements for the DarkNet importer", ["opencv-python"])), ( "importer-keras", - ("Requirements for the Keras importer", ["tensorflow", "tensorflow-estimator"]), + ("Requirements for the Keras importer", ["keras", "tensorflow", "tensorflow-estimator"]), + ), + ( + "importer-oneflow", + ( + "Requirements for the OneFlow importer", + [ + "flowvision", + "oneflow", + ], + ), ), ( "importer-onnx", @@ -128,6 +146,15 @@ REQUIREMENTS_BY_PIECE: RequirementsByPieceType = [ ], ), ), + ( + "importer-mxnet", + ( + "Requirements for the mxnet importer", + [ + "mxnet", + ], + ), + ), ( "importer-paddle", ("Requirements for the PaddlePaddle importer", ["paddlepaddle"]), @@ -170,17 +197,6 @@ REQUIREMENTS_BY_PIECE: RequirementsByPieceType = [ ], ), ), - # Vitis AI requirements - ( - "vitis-ai", - ( - "Requirements for the Vitis AI codegen", - [ - "h5py", - "progressbar", - ], - ), - ), # XGBoost, useful for autotuning on some targets. ( "xgboost", @@ -202,28 +218,45 @@ REQUIREMENTS_BY_PIECE: RequirementsByPieceType = [ "astroid", # pylint requirement, listed so a hard constraint can be included. "autodocsumm", "black", + "blocklint", "commonmark", "cpplint", "docutils", + "flake8", "image", + "jinja2", "matplotlib", + "mypy", "pillow", "pylint", "sphinx", - "sphinx_autodoc_annotation", - "sphinx_gallery", - "sphinx_rtd_theme", + "sphinx-autodoc-annotation", + "sphinx-gallery", + "sphinx-rtd-theme", "types-psutil", ], ), ), ] -ConstraintsType = typing.List[typing.Tuple[str, typing.Union[None, str]]] +ConstraintsType = List[Tuple[str, Union[Tuple[str]]]] -# Maps a named Python package (which should appear in REQUIREMENTS_BY_PIECE above) to a -# semver or pip version constraint. Semver constraints are translated into requirements.txt-friendly -# constraints. +# Maps a named Python package (which should appear in REQUIREMENTS_BY_PIECE above) to one or more +# constraint specifications matching the following form: +# +# [<replacement-package-name>]<constraint>[; <pep496 environment marker>] +# +# Where each field is defined as: +# <replacement-package-name>: Valid only when <pep496 environment marker> is present. If given, +# uses this package name in place of the original when the environment marker condition is +# met. +# <constraint>: A semantic version (semver.org) (expressed as "^a.b.c") or a pip version constarint. +# <pep496 environment maker>: A PEP406-compatible environment marker specifying the conditions under +# which this conraint and package should be used. +# +# A few limitations on replacement-package-name: +# 1. It can't be mentioned in REQUIRMENTS_BY_NAME. +# 2. It can't be mentioned as <replacement-package-name> in a constraint for different package name. # # These constraints serve only to record technical reasons why a particular version can't be used. # They are the default install_requires used in setup.py. These can be further narrowed to restrict @@ -234,74 +267,95 @@ ConstraintsType = typing.List[typing.Tuple[str, typing.Union[None, str]]] # 2. If TVM will functionally break against an old version of a dependency, specify a >= relation # here. Include a comment linking to context or explaining why the constraint is in place. CONSTRAINTS = [ - ("astroid", None), - ("attrs", None), - ("autodocsumm", None), - ("black", "==20.8b1"), - ("cloudpickle", None), - ("commonmark", ">=0.7.3"), # From PR #213. - ("coremltools", None), - ("cpplint", None), - ("decorator", None), + ("astroid", []), + ("attrs", []), + ("autodocsumm", []), + ("black", []), + ("blocklint", []), + ("cloudpickle", []), + ("commonmark", [">=0.7.3"]), # From PR #213. + ("coremltools", []), + ("cpplint", []), + ("decorator", []), ( "docutils", - "<0.17", + [">=0.11,<0.17"], ), # Work around https://github.com/readthedocs/sphinx_rtd_theme/issues/1115 - ("ethos-u-vela", "==3.2.0"), - ("future", None), - ("h5py", "==2.10.0"), - ("image", None), - ("matplotlib", None), - ("numpy", None), - ("onnx", None), - ("onnxoptimizer", None), - ("onnxruntime", None), - ("opencv-python", None), - ("paddlepaddle", None), - ("pillow", None), - ("progressbar", None), - ("protobuf", None), - ("psutil", None), - ("pylint", None), - ("scikit-image", None), - ("scipy", None), - ("six", None), - ("sphinx", None), - ("sphinx_autodoc_annotation", None), - ("sphinx_gallery", None), - ("sphinx_rtd_theme", None), - ("synr", "==0.6.0"), - ("tensorflow", None), - ("tensorflow-estimator", None), - ("tflite", None), - ("torch", None), - ("torchvision", None), - ("tornado", None), - ("xgboost", ">=1.1.0"), # From PR #4953. + ("ethos-u-vela", ["==3.2.0"]), + ("flake8", []), + ("flowvision", []), + ("future", []), + ("image", []), + ("jinja2", []), + ("keras", []), + ("matplotlib", []), + ("mxnet", []), + ("mypy", []), + ("numpy", []), + ("oneflow", []), + ("onnx", []), + ("onnxoptimizer", []), + ("onnxruntime", []), + ("opencv-python", []), + ("paddlepaddle", ["==*; 'importer-tensorflow' not in extra and 'importer-tflite' not in extra"]), + ("pillow", []), + ("protobuf", []), + ("psutil", []), + ("pylint", []), + ("scikit-image", []), + ("scipy", []), + ("six", []), + ("sphinx", []), + ("sphinx-autodoc-annotation", []), + ("sphinx-gallery", []), + ("sphinx-rtd-theme", []), + ("synr", ["==0.6.0"]), + ("tensorflow", [ + "tensorflow==*; platform_machine not in 'aarch64' and 'gpu' not in extra and 'importer-paddle' not in extra", + "tensorflow-aarch64==*; platform_machine in 'aarch64' and 'importer-paddle' not in extra", + "tensorflow-gpu==*; platform_machine not in 'aarch64' and 'gpu' in extra and 'importer-paddle' not in extra", + ]), + ("tensorflow-estimator", []), + ("tflite", []), + ("torch", []), + ("torchvision", []), + ("tornado", []), + ("xgboost", [">=1.1.0"]), # From PR #4953. ] + ################################################################################ # End of configuration options. ################################################################################ # Required keys in REQUIREMENTS_BY_PIECE. -REQUIRED_PIECES: typing.List[str] = ["core", "dev"] +REQUIRED_PIECES: List[str] = ["core", "dev"] # Regex to validates piece names. -PIECE_REGEX: typing.Pattern = re.compile(r"^[a-z0-9][a-z0-9-]*", re.IGNORECASE) +PIECE_REGEX: Pattern = re.compile(r"^[a-z0-9][a-z0-9-]*", re.IGNORECASE) # Regex to match a constraint specification. Multiple constraints are not supported. -CONSTRAINT_REGEX: typing.Pattern = re.compile(r"(?:\^|\<|(?:~=)|(?:<=)|(?:==)|(?:>=)|\>)[^<>=\^,]+") +CONSTRAINT_REGEX: Pattern = re.compile(r"(?:\^|\<|(?:~=)|(?:<=)|(?:==)|(?:>=)|\>)[^<>=\^,;]+") # Regex for parsing semantic versions. See # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string -SEMVER_REGEX: typing.Pattern = re.compile( +SEMVER_REGEX: Pattern = re.compile( r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" ) -def validate_requirements_by_piece() -> typing.List[str]: +CONSTRAINT_SPEC_REGEX: Pattern = re.compile( + r"(?P<package>[a-z0-9_-]+)?" + + r"(?P<constraint>(?:" + CONSTRAINT_REGEX.pattern + r")" + + r"|(?:" + SEMVER_REGEX.pattern + r")" + + r"|(?:==\*))" + + r"(?:;[\s]*(?P<environment_marker>.+))?") + +print("CSR", CONSTRAINT_SPEC_REGEX.pattern) + + +def validate_requirements_by_piece() -> List[str]: """Validate REQUIREMENTS_BY_PIECE, returning a list of problems. Returns @@ -393,8 +447,8 @@ def validate_requirements_by_piece() -> typing.List[str]: def parse_semver( - package: str, constraint: str, problems: typing.List[str] -) -> typing.Tuple[typing.List[str], int, int]: + package: str, constraint: str, problems: List[str] +) -> Tuple[List[str], int, int]: """Parse a semantic versioning constraint of the form "^X.[.Y[.Z[...]]]]" Parameters @@ -447,7 +501,101 @@ def parse_semver( return min_ver_parts, 0, 0 -def validate_constraints() -> typing.List[str]: +@dataclasses.dataclass(eq=True, frozen=True) +class Requirement: + package: str + constraint: str + environment_marker: Union[str, None] + + def to_requirement(self): + return f'{self.package}{self.constraint}{self.environment_marker or ""}' + + +def semver_to_requirements(dep: str, constraint: str, problems: List[str], joined_deps: Union[None, List[Requirement]]): + """Convert a SemVer-style constraint to a setuptools-compatible constraint. + + Parameters + ---------- + dep : str + Name of the PyPI package to depend on. + constraint : str + The SemVer constraint, of the form "^<semver constraint>" + problems : List[str] + A list of the validation problems encountered when parsing this semver. + joined_deps : Union[None, List[str]] + Either: + 1. A list of strings, each a setuptools-compatible constraint which could be written to + a line in requirements.txt. The converted constraint is appended to this list. + 2. None, in which case only validation is performed. + """ + min_ver_parts, fixed_index, fixed_part = parse_semver(dep, constraint, problems) + if joined_deps is not None: + text_problems = "\n" + "\n".join(f" * {p}" for p in problems) + assert ( + not problems + ), f"should not happen: validated semver {constraint} parses with problems:{text_problems}" + + max_ver_parts = ( + min_ver_parts[:fixed_index] + + [str(fixed_part + 1)] + + ["0" for _ in min_ver_parts[fixed_index + 1 :]] + ) + joined_deps.append(Requirement(package=dep, constraint=f'{".".join(min_ver_parts)},<{".".join(max_ver_parts)}')) + + +def parse_constraint_entry(package: str, constraints: str, problems: List[str], + requirements: Union[None, List[str]]): + """Parse an entry in CONSTRAINTS into requirements.txt entries. + + When requirements is None, assert-fails if any validation problems occur. + + Parameters + ---------- + package : str + The key of this entry in CONSTRAINTS. + constraints : str + The value of + Either the value in CONSTRAINTS (if said value is a str) or one item from the value (if said + value is a list of strings) which should be converted into a requirement. + problems : List[str] + A list of the validation problems encountered when parsing the entry. + requirements : Union[None, List[str]] + Either: + 1. A list of strings, each a setuptools-compatible constraint which could be written to a + line in requirements.txt The converted constraint is appended to this list. + 2. None, in which case the constraint will be only validated. + """ + def _parse_one(c): + print("PARSE_ONE", package, not c) + if not c: + if requirements is not None: + requirements.append(Requirement(package=package, constraint="==*", environment_marker=None)) + return + + m = CONSTRAINT_SPEC_REGEX.match(c) + if m is None: + problems.append( + f'{package}: constraint "{c}" does not look like a valid constraint' + ) + + if c[0] == "^": + semver_to_requirements(package, c, problems, requirements) + elif requirements is not None: + groups = m.groupdict() + requirement_package = groups.get('package') or package + requirements.append(Requirement(package=requirement_package, + constraint=groups.get('constraint', '==*'), + environment_marker=groups.get('environment_marker'))) + + if not constraints: + _parse_one(constraints) + return + + for constraint in constraints: + _parse_one(constraint) + + +def validate_constraints() -> List[str]: """Validate CONSTRAINTS, returning a list of problems found. Returns @@ -477,13 +625,7 @@ def validate_constraints() -> typing.List[str]: if constraint is None: # None is just a placeholder that allows for comments. continue - if not CONSTRAINT_REGEX.match(constraint): - problems.append( - f'{package}: constraint "{constraint}" does not look like a valid constraint' - ) - - if constraint.startswith("^"): - parse_semver(package, constraint, problems) + parse_constraint_entry(package, constraint, problems, None) all_constrained_packages = [p for (p, _) in CONSTRAINTS] sorted_constrained_packages = list(sorted(all_constrained_packages)) @@ -499,7 +641,7 @@ class ValidationError(Exception): """Raised when a validation error occurs.""" @staticmethod - def format_problems(config: str, problems: typing.List[str]) -> str: + def format_problems(config: str, problems: List[str]) -> str: """Format a list of problems with a global config variable into human-readable output. Parameters @@ -527,7 +669,7 @@ class ValidationError(Exception): return "\n".join(formatted) - def __init__(self, config: str, problems: typing.List[str]): + def __init__(self, config: str, problems: List[str]): """Describes an error that occurs validating one of the global config variables. Parameters @@ -551,35 +693,7 @@ def validate_or_raise(): raise ValidationError("CONSTRAINTS", problems) -def semver_to_requirements(dep: str, constraint: str, joined_deps: typing.List[str]): - """Convert a SemVer-style constraint to a setuptools-compatible constraint. - - Parameters - ---------- - dep : str - Name of the PyPI package to depend on. - constraint : str - The SemVer constraint, of the form "^<semver constraint>" - joined_deps : list[str] - A list of strings, each a setuptools-compatible constraint which could be written to - a line in requirements.txt. The converted constraint is appended to this list. - """ - problems: typing.List[str] = [] - min_ver_parts, fixed_index, fixed_part = parse_semver(dep, constraint, problems) - text_problems = "\n" + "\n".join(f" * {p}" for p in problems) - assert ( - not problems - ), f"should not happen: validated semver {constraint} parses with problems:{text_problems}" - - max_ver_parts = ( - min_ver_parts[:fixed_index] - + [str(fixed_part + 1)] - + ["0" for _ in min_ver_parts[fixed_index + 1 :]] - ) - joined_deps.append(f'{dep}>={".".join(min_ver_parts)},<{".".join(max_ver_parts)}') - - -def join_requirements() -> typing.Dict[str, typing.Tuple[str, typing.List[str]]]: +def join_requirements() -> Dict[str, Tuple[str, List[str]]]: """Validate, then join REQUIRMENTS_BY_PIECE against CONSTRAINTS and return the result. Returns @@ -597,14 +711,7 @@ def join_requirements() -> typing.Dict[str, typing.Tuple[str, typing.List[str]]] joined_deps = [] for d in deps: constraint = constraints_map.get(d.lower()) - if constraint is None: - joined_deps.append(d) - continue - - if constraint[0] == "^": - semver_to_requirements(d, constraint, joined_deps) - else: - joined_deps.append(f"{d}{constraint}") + parse_constraint_entry(d, constraint, None, joined_deps) if piece != "dev": all_deps.update(joined_deps) @@ -613,7 +720,7 @@ def join_requirements() -> typing.Dict[str, typing.Tuple[str, typing.List[str]]] to_return["all-prod"] = ( "Combined dependencies for all TVM pieces, excluding dev", - list(sorted(all_deps)), + list(sorted(all_deps, key=lambda r: r.package)), ) return to_return @@ -648,7 +755,7 @@ def join_and_write_requirements(args: argparse.Namespace): f"# {description}{os.linesep}" ) for d in deps: - f.write(f"{d}{os.linesep}") + f.write(f"{d!s}{os.linesep}") def parse_args() -> argparse.Namespace: