This is an automated email from the ASF dual-hosted git repository.
mshr-h pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new 8f01d0d314 [Relax][Frontend][ONNX] Add QLinearMatMul operator
converter (#20951)
8f01d0d314 is described below
commit 8f01d0d314754afcbe60527fab2a5ef7d4a80938
Author: Chaitanya140904 <[email protected]>
AuthorDate: Tue Nov 17 19:18:03 2026 +0530
[Relax][Frontend][ONNX] Add QLinearMatMul operator converter (#20951)
This PR adds support for the ONNX `QLinearMatMul` operator in the Relax
ONNX frontend, resolving #20958.
### Key Changes
- In `python/tvm/relax/frontend/onnx/onnx_frontend.py`:
- Implemented `QLinearMatMul` op converter supporting opset 10 and 21.
- Widens inputs to `int32` before zero-point subtraction to prevent
arithmetic overflow.
- Computes matrix multiplication via `relax.op.matmul(...,
out_dtype="int32")` for 32-bit integer accumulation.
- Supports per-row and per-column scale and zero-point broadcasting.
- Handles requantization: `round(scaled) + y_zero_point` and clamps to
output data type range (`uint8`, `int8`, `uint16`, `int16`).
- Registered `"QLinearMatMul": QLinearMatMul` in `_get_convert_map()`.
- In `tests/python/relax/test_frontend_onnx.py`:
- Added `test_qlinearmatmul` testing 2D uint8, 2D int8, 3D batched
int8/uint8, and opset 10/21, validated against ONNX Runtime.
Closes #20958.
---------
Co-authored-by: Masahiro Hiramori <[email protected]>
---
python/tvm/relax/frontend/onnx/onnx_frontend.py | 216 ++++++++++++
tests/python/relax/test_frontend_onnx.py | 447 ++++++++++++++++++++++++
2 files changed, 663 insertions(+)
diff --git a/python/tvm/relax/frontend/onnx/onnx_frontend.py
b/python/tvm/relax/frontend/onnx/onnx_frontend.py
index ba684062d7..95c26aa55f 100644
--- a/python/tvm/relax/frontend/onnx/onnx_frontend.py
+++ b/python/tvm/relax/frontend/onnx/onnx_frontend.py
@@ -6206,6 +6206,221 @@ class MatMulInteger(OnnxOpConverter):
return relax.op.matmul(a, b, out_dtype="int32") # Output is int32 per
ONNX spec
+class QLinearMatMul(OnnxOpConverter):
+ """Converts an onnx QLinearMatMul node into an equivalent Relax
expression."""
+
+ @classmethod
+ def _squeeze_singleton(cls, bb, param):
+ """Squeeze singleton tensors (all dimensions equal to 1, e.g. [1], [1,
1]) to 0D scalars.
+
+ Per-tensor scales and zero points in ONNX can be provided as scalars
(0D) or singleton
+ tensors (1D [1], 2D [1, 1], etc.). Normalizing them to scalars
prevents them from
+ accidentally expanding output ranks during matrix-vector and
vector-matrix multiplications.
+ """
+ if param is None:
+ return None
+ param_norm = bb.normalize(param)
+ if not hasattr(param_norm.ty, "shape"):
+ return param
+ p_shape = param_norm.ty.shape
+ if len(p_shape) > 0 and all(
+ isinstance(d, int | tvm_tirx.ir.IntImm) and int(d) == 1 for d in
p_shape
+ ):
+ return relax.op.squeeze(param)
+ return param
+
+ @classmethod
+ def _align_output_param(cls, bb, param, a_norm, b_norm):
+ if param is None:
+ return None
+ param = cls._squeeze_singleton(bb, param)
+ param_norm = bb.normalize(param)
+ if not hasattr(param_norm.ty, "shape"):
+ return param
+ p_shape = param_norm.ty.shape
+ p_ndim = len(p_shape)
+ if p_ndim == 0:
+ return param
+
+ a_shape = a_norm.ty.shape if hasattr(a_norm.ty, "shape") else None
+ b_shape = b_norm.ty.shape if hasattr(b_norm.ty, "shape") else None
+ a_rank = len(a_shape) if a_shape is not None else 2
+ b_rank = len(b_shape) if b_shape is not None else 2
+
+ # When both a and b are 1D vectors, matmul output is scalar (0D)
+ if a_rank == 1 and b_rank == 1:
+ return relax.op.squeeze(param)
+
+ # When b_rank == 1, output of matmul is [..., M], no trailing column
axis
+ if b_rank == 1:
+ if (
+ p_ndim >= 2
+ and isinstance(p_shape[-1], int | tvm_tirx.ir.IntImm)
+ and int(p_shape[-1]) == 1
+ ):
+ return relax.op.squeeze(param, axis=[-1])
+ return param
+
+ # When a_rank == 1, output of matmul is [..., N], no leading row axis
+ if a_rank == 1:
+ if (
+ p_ndim >= 2
+ and isinstance(p_shape[-2], int | tvm_tirx.ir.IntImm)
+ and int(p_shape[-2]) == 1
+ ):
+ return relax.op.squeeze(param, axis=[-2])
+ return param
+
+ # Both a and b have rank >= 2, output is [..., M, N]
+ if p_ndim == 1:
+ # Check if it corresponds to row dimension M
+ if a_shape is not None and a_rank >= 2:
+ row_dim = a_shape[-2]
+ if isinstance(row_dim, int | tvm_tirx.ir.IntImm) and
int(row_dim) == int(
+ p_shape[0]
+ ):
+ return relax.op.expand_dims(param, axis=-1)
+ # Check if it corresponds to col dimension N
+ if b_shape is not None and b_rank >= 2:
+ col_dim = b_shape[-1]
+ if isinstance(col_dim, int | tvm_tirx.ir.IntImm) and
int(col_dim) == int(
+ p_shape[0]
+ ):
+ return relax.op.expand_dims(param, axis=0)
+
+ return param
+
+ @classmethod
+ def _impl_v10(cls, bb, inputs, attr, params):
+ a = inputs[0]
+ a_scale = inputs[1]
+ a_zero_point = inputs[2] if len(inputs) > 2 and inputs[2] is not None
else None
+ b = inputs[3]
+ b_scale = inputs[4]
+ b_zero_point = inputs[5] if len(inputs) > 5 and inputs[5] is not None
else None
+ y_scale = inputs[6]
+ y_zero_point = inputs[7] if len(inputs) > 7 and inputs[7] is not None
else None
+
+ if y_zero_point is None:
+ y_zero_point = relax.const(0, "uint8")
+ out_dtype = y_zero_point.ty.dtype.dtype
+
+ # Reject float8 input and output dtypes
+ for name, val in [("A", a), ("B", b), ("Y", y_zero_point)]:
+ if val is not None and hasattr(val, "ty") and hasattr(val.ty,
"dtype"):
+ dtype_str = str(val.ty.dtype.dtype)
+ if dtype_str.startswith("float8") or "e4m3" in dtype_str or
"e5m2" in dtype_str:
+ raise NotImplementedError(
+ f"Float8 operands and outputs ({name} has dtype
{dtype_str}) "
+ "are currently not supported in QLinearMatMul."
+ )
+
+ # Normalize singleton [1]-shape and [1, 1]-shape parameters to 0D
scalars
+ # BEFORE any rank expansion or alignment, so they do not accidentally
+ # increase the output rank in vector-matrix, matrix-vector, or dot
products.
+ a_scale = cls._squeeze_singleton(bb, a_scale)
+ a_zero_point = cls._squeeze_singleton(bb, a_zero_point)
+ b_scale = cls._squeeze_singleton(bb, b_scale)
+ b_zero_point = cls._squeeze_singleton(bb, b_zero_point)
+ y_scale = cls._squeeze_singleton(bb, y_scale)
+ y_zero_point = cls._squeeze_singleton(bb, y_zero_point)
+
+ a_norm = bb.normalize(a)
+ b_norm = bb.normalize(b)
+ a_shape = a_norm.ty.shape if hasattr(a_norm.ty, "shape") else None
+ b_shape = b_norm.ty.shape if hasattr(b_norm.ty, "shape") else None
+ a_rank = len(a_shape) if a_shape is not None else 2
+ b_rank = len(b_shape) if b_shape is not None else 2
+
+ # Widen to int32 before arithmetic to prevent overflow
+ a = relax.op.astype(a, "int32")
+ b = relax.op.astype(b, "int32")
+
+ if a_zero_point is not None:
+ a_zp = relax.op.astype(a_zero_point, "int32")
+ a_zp = bb.normalize(a_zp)
+ if a_rank >= 2 and len(a_zp.ty.shape) == 1:
+ a_zp = relax.op.expand_dims(a_zp, axis=-1)
+ a = relax.op.subtract(a, a_zp)
+
+ if b_zero_point is not None:
+ b_zp = relax.op.astype(b_zero_point, "int32")
+ b_zp = bb.normalize(b_zp)
+ if b_rank >= 2 and len(b_zp.ty.shape) == 1:
+ b_zp = relax.op.expand_dims(b_zp, axis=0)
+ b = relax.op.subtract(b, b_zp)
+
+ # 32-bit integer accumulation
+ matmul_int32 = relax.op.matmul(a, b, out_dtype="int32")
+
+ # Broadcast a_scale: when both a and b are rank >= 2, expand row
dimension to broadcast over columns
+ a_scale_norm = bb.normalize(a_scale)
+ if hasattr(a_scale_norm.ty, "shape"):
+ as_shape = a_scale_norm.ty.shape
+ if a_rank >= 2 and b_rank >= 2:
+ if len(as_shape) == 1:
+ a_scale = relax.op.expand_dims(a_scale, axis=-1)
+ elif b_rank == 1:
+ if (
+ len(as_shape) >= 2
+ and isinstance(as_shape[-1], int | tvm_tirx.ir.IntImm)
+ and int(as_shape[-1]) == 1
+ ):
+ a_scale = relax.op.squeeze(a_scale, axis=[-1])
+
+ # Broadcast b_scale: when both a and b are rank >= 2, expand column
dimension to broadcast over rows
+ b_scale_norm = bb.normalize(b_scale)
+ if hasattr(b_scale_norm.ty, "shape"):
+ bs_shape = b_scale_norm.ty.shape
+ if a_rank >= 2 and b_rank >= 2:
+ if len(bs_shape) == 1:
+ b_scale = relax.op.expand_dims(b_scale, axis=0)
+ elif a_rank == 1:
+ if (
+ len(bs_shape) >= 2
+ and isinstance(bs_shape[-2], int | tvm_tirx.ir.IntImm)
+ and int(bs_shape[-2]) == 1
+ ):
+ b_scale = relax.op.squeeze(b_scale, axis=[-2])
+
+ # Ensure any resulting singleton scale or output parameter is squeezed
+ a_scale = cls._squeeze_singleton(bb, a_scale)
+ b_scale = cls._squeeze_singleton(bb, b_scale)
+ y_scale = cls._squeeze_singleton(bb, y_scale)
+ y_zero_point = cls._squeeze_singleton(bb, y_zero_point)
+
+ y_scale = cls._align_output_param(bb, y_scale, a_norm, b_norm)
+ y_zero_point = cls._align_output_param(bb, y_zero_point, a_norm,
b_norm)
+
+ matmul_float = relax.op.astype(matmul_int32, "float32")
+ a_scale_f = relax.op.astype(a_scale, "float32")
+ b_scale_f = relax.op.astype(b_scale, "float32")
+ y_scale_f = relax.op.astype(y_scale, "float32")
+
+ scale_mult = relax.op.divide(relax.op.multiply(a_scale_f, b_scale_f),
y_scale_f)
+ scaled = relax.op.multiply(matmul_float, scale_mult)
+ rounded = relax.op.round(scaled)
+ y_val = relax.op.add(rounded, relax.op.astype(y_zero_point, "float32"))
+
+ if out_dtype == "uint8":
+ min_val, max_val = 0, 255
+ elif out_dtype == "int8":
+ min_val, max_val = -128, 127
+ elif out_dtype == "uint16":
+ min_val, max_val = 0, 65535
+ elif out_dtype == "int16":
+ min_val, max_val = -32768, 32767
+ else:
+ min_val, max_val = 0, 255
+
+ clipped = relax.op.clip(y_val, min_val, max_val)
+ return relax.op.astype(clipped, out_dtype)
+
+ @classmethod
+ def _impl_v21(cls, bb, inputs, attr, params):
+ return cls._impl_v10(bb, inputs, attr, params)
+
+
class DFT(OnnxOpConverter):
"""Converts an onnx DFT node into an equivalent Relax expression."""
@@ -6457,6 +6672,7 @@ def _get_convert_map():
"MatMul": MatMul,
"MatMulInteger": MatMulInteger,
"MatMulInteger16": MatMulInteger16,
+ "QLinearMatMul": QLinearMatMul,
"Reshape": Reshape,
"Sigmoid": Sigmoid,
"Softmax": Softmax,
diff --git a/tests/python/relax/test_frontend_onnx.py
b/tests/python/relax/test_frontend_onnx.py
index ca88a9259b..1a5113ca2d 100644
--- a/tests/python/relax/test_frontend_onnx.py
+++ b/tests/python/relax/test_frontend_onnx.py
@@ -14812,6 +14812,453 @@ def test_quantizelinear_output_dtype_mismatch(opset):
from_onnx(model, opset=opset, keep_params_in_input=True)
+def test_qlinearmatmul():
+ """Verify numerical correctness of QLinearMatMul against ONNX Runtime and
reference."""
+
+ np_dtype_map = {
+ "uint8": (np.uint8, TensorProto.UINT8),
+ "int8": (np.int8, TensorProto.INT8),
+ }
+
+ def verify_qlinearmatmul(
+ a_shape,
+ b_shape,
+ a_dtype="uint8",
+ b_dtype="uint8",
+ y_dtype="uint8",
+ a_scale_shape=(),
+ b_scale_shape=(),
+ y_scale_shape=(),
+ opset=10,
+ seed=42,
+ ):
+ rng = np.random.RandomState(seed)
+ a_np_dtype, a_proto = np_dtype_map[a_dtype]
+ b_np_dtype, b_proto = np_dtype_map[b_dtype]
+ y_np_dtype, y_proto = np_dtype_map[y_dtype]
+
+ if a_dtype == "uint8":
+ a = rng.randint(120, 136, size=a_shape).astype(a_np_dtype)
+ a_zp = np.full(a_scale_shape, 128, dtype=a_np_dtype)
+ else:
+ a = rng.randint(-8, 8, size=a_shape).astype(a_np_dtype)
+ a_zp = np.full(a_scale_shape, 0, dtype=a_np_dtype)
+
+ if b_dtype == "uint8":
+ b = rng.randint(120, 136, size=b_shape).astype(b_np_dtype)
+ b_zp = np.full(b_scale_shape, 128, dtype=b_np_dtype)
+ else:
+ b = rng.randint(-8, 8, size=b_shape).astype(b_np_dtype)
+ b_zp = np.full(b_scale_shape, 0, dtype=b_np_dtype)
+
+ if y_dtype == "uint8":
+ y_zp = np.full(y_scale_shape, 128, dtype=y_np_dtype)
+ else:
+ y_zp = np.full(y_scale_shape, 0, dtype=y_np_dtype)
+
+ a_scale = np.full(a_scale_shape, 0.5, dtype=np.float32)
+ b_scale = np.full(b_scale_shape, 0.25, dtype=np.float32)
+ y_scale = np.full(y_scale_shape, 2.0, dtype=np.float32)
+
+ node = helper.make_node(
+ "QLinearMatMul",
+ inputs=["a", "a_scale", "a_zp", "b", "b_scale", "b_zp", "y_scale",
"y_zp"],
+ outputs=["y"],
+ )
+
+ y_shape = list(np.matmul(np.zeros(a_shape), np.zeros(b_shape)).shape)
+
+ graph = helper.make_graph(
+ [node],
+ "qlinearmatmul_test",
+ inputs=[
+ helper.make_tensor_value_info("a", a_proto, a_shape),
+ helper.make_tensor_value_info("a_scale", TensorProto.FLOAT,
list(a_scale_shape)),
+ helper.make_tensor_value_info("a_zp", a_proto,
list(a_scale_shape)),
+ helper.make_tensor_value_info("b", b_proto, b_shape),
+ helper.make_tensor_value_info("b_scale", TensorProto.FLOAT,
list(b_scale_shape)),
+ helper.make_tensor_value_info("b_zp", b_proto,
list(b_scale_shape)),
+ helper.make_tensor_value_info("y_scale", TensorProto.FLOAT,
list(y_scale_shape)),
+ helper.make_tensor_value_info("y_zp", y_proto,
list(y_scale_shape)),
+ ],
+ outputs=[helper.make_tensor_value_info("y", y_proto, y_shape)],
+ )
+ model = helper.make_model(graph,
opset_imports=[helper.make_opsetid("", opset)])
+ inputs = {
+ "a": a,
+ "a_scale": a_scale,
+ "a_zp": a_zp,
+ "b": b,
+ "b_scale": b_scale,
+ "b_zp": b_zp,
+ "y_scale": y_scale,
+ "y_zp": y_zp,
+ }
+ check_correctness(model, inputs=inputs, opset=opset)
+
+ def verify_qlinearmatmul_ref(
+ a_shape,
+ b_shape,
+ a_dtype="uint8",
+ b_dtype="uint8",
+ y_dtype="uint8",
+ a_scale_shape=(),
+ b_scale_shape=(),
+ y_scale_shape=(),
+ opset=10,
+ seed=42,
+ ):
+ rng = np.random.RandomState(seed)
+ a_np_dtype, a_proto = np_dtype_map[a_dtype]
+ b_np_dtype, b_proto = np_dtype_map[b_dtype]
+ y_np_dtype, y_proto = np_dtype_map[y_dtype]
+
+ if a_dtype == "uint8":
+ a = rng.randint(120, 136, size=a_shape).astype(a_np_dtype)
+ a_zp = np.full(a_scale_shape, 128, dtype=a_np_dtype)
+ else:
+ a = rng.randint(-8, 8, size=a_shape).astype(a_np_dtype)
+ a_zp = np.full(a_scale_shape, 0, dtype=a_np_dtype)
+
+ if b_dtype == "uint8":
+ b = rng.randint(120, 136, size=b_shape).astype(b_np_dtype)
+ b_zp = np.full(b_scale_shape, 128, dtype=b_np_dtype)
+ else:
+ b = rng.randint(-8, 8, size=b_shape).astype(b_np_dtype)
+ b_zp = np.full(b_scale_shape, 0, dtype=b_np_dtype)
+
+ if y_dtype == "uint8":
+ y_zp = np.full(y_scale_shape, 128, dtype=y_np_dtype)
+ else:
+ y_zp = np.full(y_scale_shape, 0, dtype=y_np_dtype)
+
+ a_scale = np.full(a_scale_shape, 0.5, dtype=np.float32)
+ b_scale = np.full(b_scale_shape, 0.25, dtype=np.float32)
+ y_scale = np.full(y_scale_shape, 2.0, dtype=np.float32)
+
+ # Reference calculation
+ a_int32 = a.astype(np.int32)
+ b_int32 = b.astype(np.int32)
+ a_zp_i = a_zp.astype(np.int32)
+ b_zp_i = b_zp.astype(np.int32)
+ if a.ndim >= 2 and a_zp_i.ndim == 1:
+ a_zp_i = np.expand_dims(a_zp_i, axis=-1)
+ if b.ndim >= 2 and b_zp_i.ndim == 1:
+ b_zp_i = np.expand_dims(b_zp_i, axis=0)
+
+ matmul_int32 = np.matmul(a_int32 - a_zp_i, b_int32 - b_zp_i)
+ as_f = a_scale.copy()
+ bs_f = b_scale.copy()
+ ys_f = y_scale.copy()
+ yz_f = y_zp.astype(np.float32)
+
+ if b.ndim >= 2 and as_f.ndim == 1:
+ as_f = np.expand_dims(as_f, axis=-1)
+ elif b.ndim == 1 and as_f.ndim >= 2 and as_f.shape[-1] == 1:
+ as_f = np.squeeze(as_f, axis=-1)
+
+ if a.ndim >= 2 and bs_f.ndim == 1:
+ bs_f = np.expand_dims(bs_f, axis=0)
+ elif a.ndim == 1 and bs_f.ndim >= 2 and bs_f.shape[-2] == 1:
+ bs_f = np.squeeze(bs_f, axis=-2)
+
+ if b.ndim >= 2 and a.ndim >= 2 and ys_f.ndim == 1:
+ if a.shape[-2] == ys_f.shape[0]:
+ ys_f = np.expand_dims(ys_f, axis=-1)
+ yz_f = np.expand_dims(yz_f, axis=-1)
+ elif b.shape[-1] == ys_f.shape[0]:
+ ys_f = np.expand_dims(ys_f, axis=0)
+ yz_f = np.expand_dims(yz_f, axis=0)
+
+ scaled = matmul_int32.astype(np.float32) * (as_f * bs_f) / ys_f
+ rounded = np.round(scaled) + yz_f
+ min_v, max_v = (0, 255) if y_dtype == "uint8" else (-128, 127)
+ ref_out = np.clip(rounded, min_v, max_v).astype(y_np_dtype)
+
+ node = helper.make_node(
+ "QLinearMatMul",
+ inputs=["a", "a_scale", "a_zp", "b", "b_scale", "b_zp", "y_scale",
"y_zp"],
+ outputs=["y"],
+ )
+ y_shape = list(ref_out.shape)
+
+ graph = helper.make_graph(
+ [node],
+ "qlinearmatmul_ref_test",
+ inputs=[
+ helper.make_tensor_value_info("a", a_proto, a_shape),
+ helper.make_tensor_value_info("a_scale", TensorProto.FLOAT,
list(a_scale_shape)),
+ helper.make_tensor_value_info("a_zp", a_proto,
list(a_scale_shape)),
+ helper.make_tensor_value_info("b", b_proto, b_shape),
+ helper.make_tensor_value_info("b_scale", TensorProto.FLOAT,
list(b_scale_shape)),
+ helper.make_tensor_value_info("b_zp", b_proto,
list(b_scale_shape)),
+ helper.make_tensor_value_info("y_scale", TensorProto.FLOAT,
list(y_scale_shape)),
+ helper.make_tensor_value_info("y_zp", y_proto,
list(y_scale_shape)),
+ ],
+ outputs=[helper.make_tensor_value_info("y", y_proto, y_shape)],
+ )
+ model = helper.make_model(graph,
opset_imports=[helper.make_opsetid("", opset)])
+ inputs = {
+ "a": a,
+ "a_scale": a_scale,
+ "a_zp": a_zp,
+ "b": b,
+ "b_scale": b_scale,
+ "b_zp": b_zp,
+ "y_scale": y_scale,
+ "y_zp": y_zp,
+ }
+ tvm_out = run_in_tvm(model, inputs=inputs, opset=opset)
+ tvm_val = tvm_out[0].numpy() if isinstance(tvm_out, list | tuple) else
tvm_out.numpy()
+ assert tvm_val.shape == ref_out.shape
+ np.testing.assert_allclose(tvm_val, ref_out, atol=1e-5)
+
+ # 2D uint8 with in-range output
+ verify_qlinearmatmul(
+ [2, 3], [3, 4], a_dtype="uint8", b_dtype="uint8", y_dtype="uint8",
opset=10
+ )
+ # 2D int8
+ verify_qlinearmatmul([3, 4], [4, 2], a_dtype="int8", b_dtype="int8",
y_dtype="int8", opset=10)
+ # 3D batched int8
+ verify_qlinearmatmul(
+ [2, 3, 4], [2, 4, 5], a_dtype="int8", b_dtype="int8", y_dtype="int8",
opset=10
+ )
+ # 3D batched uint8
+ verify_qlinearmatmul(
+ [2, 3, 4], [2, 4, 5], a_dtype="uint8", b_dtype="uint8",
y_dtype="uint8", opset=10
+ )
+ # Opset 21 uint8
+ verify_qlinearmatmul(
+ [2, 3], [3, 4], a_dtype="uint8", b_dtype="uint8", y_dtype="uint8",
opset=21
+ )
+ # Opset 21 int8
+ verify_qlinearmatmul([2, 3], [3, 4], a_dtype="int8", b_dtype="int8",
y_dtype="int8", opset=21)
+ # Mixed signedness uint8/int8
+ verify_qlinearmatmul([2, 3], [3, 4], a_dtype="uint8", b_dtype="int8",
y_dtype="uint8", opset=10)
+ # Matrix-vector (2D x 1D -> 1D)
+ verify_qlinearmatmul([2, 3], [3], a_dtype="uint8", b_dtype="uint8",
y_dtype="uint8", opset=10)
+ # Vector-matrix (1D x 2D -> 1D)
+ verify_qlinearmatmul([3], [3, 4], a_dtype="uint8", b_dtype="uint8",
y_dtype="uint8", opset=10)
+ # Batched matrix-vector (3D x 1D -> 2D)
+ verify_qlinearmatmul(
+ [2, 3, 4], [4], a_dtype="uint8", b_dtype="uint8", y_dtype="uint8",
opset=10
+ )
+ # Per-column parameter on B
+ verify_qlinearmatmul(
+ [2, 3], [3, 4], a_dtype="uint8", b_dtype="uint8", y_dtype="uint8",
b_scale_shape=(4,)
+ )
+ # Batched per-column parameter on B
+ verify_qlinearmatmul(
+ [2, 3, 4],
+ [2, 4, 5],
+ a_dtype="uint8",
+ b_dtype="uint8",
+ y_dtype="uint8",
+ b_scale_shape=(2, 1, 5),
+ )
+
+ # Reference tests for cases unsupported by ORT (per-row A and int8/uint8)
+ verify_qlinearmatmul_ref(
+ [2, 3], [3, 4], a_dtype="uint8", b_dtype="uint8", y_dtype="uint8",
a_scale_shape=(2,)
+ )
+ verify_qlinearmatmul_ref(
+ [2, 3, 4],
+ [2, 4, 5],
+ a_dtype="uint8",
+ b_dtype="uint8",
+ y_dtype="uint8",
+ a_scale_shape=(2, 3, 1),
+ )
+ verify_qlinearmatmul_ref([2, 3], [3, 4], a_dtype="int8", b_dtype="uint8",
y_dtype="int8")
+
+
+def test_qlinearmatmul_float8_unsupported():
+ """Verify that float8 inputs and outputs in QLinearMatMul are rejected
with NotImplementedError."""
+ node = helper.make_node(
+ "QLinearMatMul",
+ inputs=["a", "a_scale", "a_zp", "b", "b_scale", "b_zp", "y_scale",
"y_zp"],
+ outputs=["y"],
+ )
+ # TensorProto.FLOAT8E4M3FN is type 17
+ graph = helper.make_graph(
+ [node],
+ "test_float8",
+ inputs=[
+ helper.make_tensor_value_info("a", 17, [2, 3]),
+ helper.make_tensor_value_info("a_scale", TensorProto.FLOAT, []),
+ helper.make_tensor_value_info("a_zp", 17, []),
+ helper.make_tensor_value_info("b", 17, [3, 4]),
+ helper.make_tensor_value_info("b_scale", TensorProto.FLOAT, []),
+ helper.make_tensor_value_info("b_zp", 17, []),
+ helper.make_tensor_value_info("y_scale", TensorProto.FLOAT, []),
+ helper.make_tensor_value_info("y_zp", 17, []),
+ ],
+ outputs=[helper.make_tensor_value_info("y", 17, [2, 4])],
+ )
+ model = helper.make_model(graph, opset_imports=[helper.make_opsetid("",
21)])
+ with pytest.raises(NotImplementedError, match="Float8 operands and
outputs"):
+ from_onnx(model, opset=21, keep_params_in_input=True)
+
+
+def test_qlinearmatmul_singleton_scales_output_shape():
+ """Singleton [1]-shape scales and zero points must not alter the MatMul
output rank.
+
+ A valid a_scale:[1] on [3] @ [3,4] must still produce output [4], not
[1,4].
+ """
+ a = np.array([2, 3, 4], dtype=np.uint8)
+ b = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0]], dtype=np.uint8)
+ # All quantization parameters as [1]-shape (singleton) arrays
+ a_scale = np.array([0.5], dtype=np.float32)
+ a_zp = np.array([0], dtype=np.uint8)
+ b_scale = np.array([0.5], dtype=np.float32)
+ b_zp = np.array([0], dtype=np.uint8)
+ y_scale = np.array([1.0], dtype=np.float32)
+ y_zp = np.array([0], dtype=np.uint8)
+
+ node = helper.make_node(
+ "QLinearMatMul",
+ inputs=["a", "a_scale", "a_zp", "b", "b_scale", "b_zp", "y_scale",
"y_zp"],
+ outputs=["y"],
+ )
+ graph = helper.make_graph(
+ [node],
+ "singleton_test",
+ inputs=[
+ helper.make_tensor_value_info("a", TensorProto.UINT8, [3]),
+ helper.make_tensor_value_info("a_scale", TensorProto.FLOAT, [1]),
+ helper.make_tensor_value_info("a_zp", TensorProto.UINT8, [1]),
+ helper.make_tensor_value_info("b", TensorProto.UINT8, [3, 4]),
+ helper.make_tensor_value_info("b_scale", TensorProto.FLOAT, [1]),
+ helper.make_tensor_value_info("b_zp", TensorProto.UINT8, [1]),
+ helper.make_tensor_value_info("y_scale", TensorProto.FLOAT, [1]),
+ helper.make_tensor_value_info("y_zp", TensorProto.UINT8, [1]),
+ ],
+ outputs=[helper.make_tensor_value_info("y", TensorProto.UINT8, [4])],
+ )
+ model = helper.make_model(graph, opset_imports=[helper.make_opsetid("",
10)])
+ inputs = {
+ "a": a,
+ "a_scale": a_scale,
+ "a_zp": a_zp,
+ "b": b,
+ "b_scale": b_scale,
+ "b_zp": b_zp,
+ "y_scale": y_scale,
+ "y_zp": y_zp,
+ }
+ tvm_out = run_in_tvm(model, inputs=inputs, opset=10)
+ tvm_val = tvm_out[0].numpy() if isinstance(tvm_out, list | tuple) else
tvm_out.numpy()
+ # Output must be rank-1, shape [4], not rank-2 [1,4]
+ assert tvm_val.shape == (4,), f"Expected shape (4,) but got
{tvm_val.shape}"
+ # Reference: (a - a_zp) @ (b - b_zp) * (a_scale * b_scale / y_scale) + y_zp
+ ref = np.clip(
+ np.round(
+ np.matmul(a.astype(np.int32),
b.astype(np.int32)).astype(np.float32) * (0.5 * 0.5 / 1.0)
+ ),
+ 0,
+ 255,
+ ).astype(np.uint8)
+ np.testing.assert_array_equal(tvm_val, ref)
+
+ # Matrix-vector case: [4, 3] @ [3] -> [4]
+ a_mv = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [1, 0, 1]],
dtype=np.uint8)
+ b_mv = np.array([2, 1, 0], dtype=np.uint8)
+ node_mv = helper.make_node(
+ "QLinearMatMul",
+ inputs=["a", "a_scale", "a_zp", "b", "b_scale", "b_zp", "y_scale",
"y_zp"],
+ outputs=["y"],
+ )
+ graph_mv = helper.make_graph(
+ [node_mv],
+ "singleton_mv_test",
+ inputs=[
+ helper.make_tensor_value_info("a", TensorProto.UINT8, [4, 3]),
+ helper.make_tensor_value_info("a_scale", TensorProto.FLOAT, [1]),
+ helper.make_tensor_value_info("a_zp", TensorProto.UINT8, [1]),
+ helper.make_tensor_value_info("b", TensorProto.UINT8, [3]),
+ helper.make_tensor_value_info("b_scale", TensorProto.FLOAT, [1]),
+ helper.make_tensor_value_info("b_zp", TensorProto.UINT8, [1]),
+ helper.make_tensor_value_info("y_scale", TensorProto.FLOAT, [1]),
+ helper.make_tensor_value_info("y_zp", TensorProto.UINT8, [1]),
+ ],
+ outputs=[helper.make_tensor_value_info("y", TensorProto.UINT8, [4])],
+ )
+ model_mv = helper.make_model(graph_mv,
opset_imports=[helper.make_opsetid("", 10)])
+ inputs_mv = {
+ "a": a_mv,
+ "a_scale": a_scale,
+ "a_zp": a_zp,
+ "b": b_mv,
+ "b_scale": b_scale,
+ "b_zp": b_zp,
+ "y_scale": y_scale,
+ "y_zp": y_zp,
+ }
+ tvm_out_mv = run_in_tvm(model_mv, inputs=inputs_mv, opset=10)
+ tvm_val_mv = (
+ tvm_out_mv[0].numpy() if isinstance(tvm_out_mv, list | tuple) else
tvm_out_mv.numpy()
+ )
+ assert tvm_val_mv.shape == (4,), f"Expected shape (4,) but got
{tvm_val_mv.shape}"
+ ref_mv = np.clip(
+ np.round(
+ np.matmul(a_mv.astype(np.int32),
b_mv.astype(np.int32)).astype(np.float32)
+ * (0.5 * 0.5 / 1.0)
+ ),
+ 0,
+ 255,
+ ).astype(np.uint8)
+ np.testing.assert_array_equal(tvm_val_mv, ref_mv)
+
+ # Vector-vector dot product case: [3] @ [3] -> ()
+ a_vv = np.array([2, 3, 4], dtype=np.uint8)
+ b_vv = np.array([1, 2, 1], dtype=np.uint8)
+ node_vv = helper.make_node(
+ "QLinearMatMul",
+ inputs=["a", "a_scale", "a_zp", "b", "b_scale", "b_zp", "y_scale",
"y_zp"],
+ outputs=["y"],
+ )
+ graph_vv = helper.make_graph(
+ [node_vv],
+ "singleton_vv_test",
+ inputs=[
+ helper.make_tensor_value_info("a", TensorProto.UINT8, [3]),
+ helper.make_tensor_value_info("a_scale", TensorProto.FLOAT, [1]),
+ helper.make_tensor_value_info("a_zp", TensorProto.UINT8, [1]),
+ helper.make_tensor_value_info("b", TensorProto.UINT8, [3]),
+ helper.make_tensor_value_info("b_scale", TensorProto.FLOAT, [1]),
+ helper.make_tensor_value_info("b_zp", TensorProto.UINT8, [1]),
+ helper.make_tensor_value_info("y_scale", TensorProto.FLOAT, [1]),
+ helper.make_tensor_value_info("y_zp", TensorProto.UINT8, [1]),
+ ],
+ outputs=[helper.make_tensor_value_info("y", TensorProto.UINT8, [])],
+ )
+ model_vv = helper.make_model(graph_vv,
opset_imports=[helper.make_opsetid("", 10)])
+ inputs_vv = {
+ "a": a_vv,
+ "a_scale": a_scale,
+ "a_zp": a_zp,
+ "b": b_vv,
+ "b_scale": b_scale,
+ "b_zp": b_zp,
+ "y_scale": y_scale,
+ "y_zp": y_zp,
+ }
+ tvm_out_vv = run_in_tvm(model_vv, inputs=inputs_vv, opset=10)
+ tvm_val_vv = (
+ tvm_out_vv[0].numpy() if isinstance(tvm_out_vv, list | tuple) else
tvm_out_vv.numpy()
+ )
+ assert tvm_val_vv.shape == (), f"Expected scalar shape () but got
{tvm_val_vv.shape}"
+ ref_vv = np.clip(
+ np.round(
+ np.matmul(a_vv.astype(np.int32),
b_vv.astype(np.int32)).astype(np.float32)
+ * (0.5 * 0.5 / 1.0)
+ ),
+ 0,
+ 255,
+ ).astype(np.uint8)
+ np.testing.assert_array_equal(tvm_val_vv, ref_vv)
+
+
def test_dft():
"""Verify numerical correctness of DFT against ONNX Runtime."""