This is an automated email from the ASF dual-hosted git repository.
mshr-h pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new b473185418 [Relax][ONNX] Add ONNX Backend Tests for systematic
frontend coverage (#19515)
b473185418 is described below
commit b473185418cc8e7c27590d3294bc86d97cc8027d
Author: HoYi <[email protected]>
AuthorDate: Wed May 13 21:08:56 2026 +0800
[Relax][ONNX] Add ONNX Backend Tests for systematic frontend coverage
(#19515)
## Summary
Introduce a test runner that reuses the official ONNX Backend Test Suite
to systematically verify the Relax ONNX importer. This complements the
existing hand-written tests in `test_frontend_onnx.py` by providing
spec-aligned coverage of standard ONNX operator semantics.
Towards #19505
## Motivation
The existing `test_frontend_onnx.py` has 187 hand-written tests that
validate TVM-specific importer behavior (parameter handling, name
sanitization, dynamic shapes, Relax IR structure). However, it relies on
ONNX Runtime as the reference and cannot systematically cover all edge
cases defined in the ONNX specification.
The ONNX Backend Test Suite provides 1653+ node-level tests with
protobuf reference inputs/outputs. It is the industry standard for
validating ONNX importers/exporters (used by ONNX Runtime, TensorFlow,
PyTorch). Reusing it gives Relax a living, upstream-aligned correctness
baseline.
## What this PR adds
- `tests/python/relax/test_frontend_onnx_backend.py` — a backend adapter
(`TVMRelaxBackend`) that implements the `onnx.backend.base.Backend`
interface, wiring `from_onnx()` → `DecomposeOpsForInference()` →
`LegalizeOps()` → `tvm.compile()` → `VirtualMachine`.
## Coverage
72 operators with 388 test cases, all passing. Only operators where
every ONNX node test passes are included — no xfail markers.
Operators not yet covered include: cast (exotic dtypes), reduce ops
(edge cases), reshape/resize/attention (complex behavior), quantization,
and several others with known importer gaps. These can be added
incrementally as the importer improves.
## Test results
388 passed, 3216 skipped (CUDA variants + operators not yet in
allowlist), 0 failed, 0 xfailed
## CI impact
- New test file is not added to any existing CI test shard by default
- Full suite (388 tests) is lightweight on CPU-only runners
## Design decisions
- **Coexistence with existing tests**: `test_frontend_onnx.py` remains
unchanged. Backend tests cover standard ONNX semantics; hand-written
tests continue to cover TVM-specific behavior (dynamic shapes, Relax IR
structure, importer options).
- **Public API only**: uses `backend_test.include()` with `^`-anchored
regex patterns. No access to private ONNX APIs.
- **No xfail**: only include operators that fully pass. Uncovered
operators are documented in code comments and this PR description.
Follow-up PRs can expand coverage as importer gaps are fixed.
- **Prefix conflict handling**: `include()` patterns use
`^test_{op}(?:_.*)?(?:_cpu|_cuda)$`, which can cause false matches when
a short op name is a prefix of a longer one (e.g. `log` vs
`log_softmax`). Affected ops (`log`, `max`, `relu`) are excluded until a
more precise matching strategy is adopted.
---
tests/python/relax/test_frontend_onnx_backend.py | 167 +++++++++++++++++++++++
1 file changed, 167 insertions(+)
diff --git a/tests/python/relax/test_frontend_onnx_backend.py
b/tests/python/relax/test_frontend_onnx_backend.py
new file mode 100644
index 0000000000..3eb63f1535
--- /dev/null
+++ b/tests/python/relax/test_frontend_onnx_backend.py
@@ -0,0 +1,167 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=invalid-name
+"""
+ONNX Backend Tests
+===================
+Systematically verify the Relax ONNX importer using the official ONNX
+Backend Test Suite (node-level tests only). Each test loads a small
+ONNX model with protobuf reference inputs/outputs and checks that the
+Relax-imported model produces numerically correct results.
+
+Only ``onnx.backend.test.data.node`` tests are registered here; real,
+simple, and PyTorch model tests are out of scope for importer-level
+semantic verification.
+
+"""
+
+import numpy as np
+import onnx
+import onnx.backend.test
+from onnx.backend.base import Backend, BackendRep
+
+import tvm
+from tvm import relax
+from tvm.relax.frontend.onnx import from_onnx
+
+# ---------------------------------------------------------------------------
+# Backend adapter
+# ---------------------------------------------------------------------------
+
+
+class TVMRelaxBackendRep(BackendRep):
+ """Compiled Relax VM representation for running an ONNX model."""
+
+ def __init__(self, mod, params, func_param_names, graph_input_names):
+ super().__init__()
+ self._params = params
+ self._func_param_names = func_param_names
+ self._graph_input_names = graph_input_names
+
+ with tvm.transform.PassContext(opt_level=3):
+ ex = tvm.compile(mod, target="llvm")
+ self._vm = relax.VirtualMachine(ex, tvm.cpu())
+
+ def run(self, inputs, **kwargs):
+ # Map positional inputs to names. The runner loads one .pb per
+ # non-initializer input, aligned with model.graph.input order.
+ input_map = {}
+ for i, arr in enumerate(inputs):
+ if i < len(self._graph_input_names):
+ input_map[self._graph_input_names[i]] = arr
+
+ # Build the argument list matching the Relax function's param order:
+ # user inputs first, then weight params from self._params.
+ input_list = []
+ for name in self._func_param_names:
+ if name in input_map:
+ input_list.append(input_map[name])
+ if self._params and "main" in self._params:
+ input_list += self._params["main"]
+
+ self._vm.set_input("main", *input_list)
+ self._vm.invoke_stateful("main")
+ output = self._vm.get_outputs("main")
+
+ if isinstance(output, (tvm.runtime.Tensor, np.ndarray)):
+ return (output.numpy() if hasattr(output, "numpy") else output,)
+ if isinstance(output, (tuple, list)):
+ return tuple(
+ o.numpy() if hasattr(o, "numpy") else np.array(o) for o in
output
+ )
+ return (np.array(output),)
+
+
+class TVMRelaxBackend(Backend):
+ """ONNX backend that imports models through Relax's ONNX frontend."""
+
+ @classmethod
+ def is_compatible(cls, model, device="CPU", **kwargs):
+ return True
+
+ @classmethod
+ def prepare(cls, model, device="CPU", **kwargs):
+ opset = None
+ for opset_import in model.opset_import:
+ if opset_import.domain in ("", "ai.onnx"):
+ opset = opset_import.version
+ break
+
+ tvm_model = from_onnx(model, opset=opset, keep_params_in_input=True)
+ tvm_model = relax.transform.DecomposeOpsForInference()(tvm_model)
+ tvm_model = relax.transform.LegalizeOps()(tvm_model)
+ tvm_model, params = relax.frontend.detach_params(tvm_model)
+
+ func = tvm_model["main"]
+ func_param_names = [p.name_hint for p in func.params]
+ graph_input_names = [inp.name for inp in model.graph.input]
+
+ return TVMRelaxBackendRep(
+ tvm_model, params, func_param_names, graph_input_names
+ )
+
+ @classmethod
+ def supports_device(cls, device: str) -> bool:
+ return device == "CPU"
+
+
+# ---------------------------------------------------------------------------
+# Test registration
+# ---------------------------------------------------------------------------
+
+backend_test = onnx.backend.test.BackendTest(TVMRelaxBackend, __name__)
+
+# Operators where ALL ONNX node tests pass on the Relax importer.
+# Each prefix covers the base test and all its variants
+# (e.g. test_add, test_add_bcast, test_add_uint8).
+#
+# Operators not listed here have known importer gaps or have not yet been
+# validated against the ONNX Backend Test Suite. They can be added
+# incrementally as the importer improves.
+_INCLUDE_OPS = [
+ "abs", "acos", "acosh", "add", "and", "argmax", "argmin",
+ "averagepool", "bitshift",
+ "bitwise_and", "bitwise_not", "bitwise_or", "bitwise_xor",
+ "ceil", "clip", "compress", "concat",
+ "conv", "cos", "cosh",
+ "depthtospace", "div",
+ "einsum", "erf", "exp",
+ "flatten", "floor",
+ "gathernd", "gemm",
+ "globalaveragepool", "globalmaxpool", "greater", "greater_equal",
+ "hardmax", "hardswish",
+ "isnan",
+ "less", "less_equal", "lrn",
+ "matmul", "matmulinteger", "mean", "min", "mod", "mul", "neg",
+ "nonzero", "not",
+ "or",
+ "reciprocal",
+ "round",
+ "scatternd",
+ "sigmoid", "sign",
+ "sin", "sinh", "size", "slice",
+ "spacetodepth",
+ "sqrt", "squeeze", "sub", "sum",
+ "tan", "tanh", "tile", "transpose",
+ "unique", "unsqueeze",
+ "where", "xor",
+]
+
+for _op in _INCLUDE_OPS:
+ backend_test.include(rf"^test_{_op}(?:_.*)?(?:_cpu|_cuda)$")
+
+globals().update(backend_test.test_cases)
