This is an automated email from the ASF dual-hosted git repository.

mshr-h pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git


The following commit(s) were added to refs/heads/main by this push:
     new 8f01d0d314 [Relax][Frontend][ONNX] Add QLinearMatMul operator 
converter (#20951)
8f01d0d314 is described below

commit 8f01d0d314754afcbe60527fab2a5ef7d4a80938
Author: Chaitanya140904 <[email protected]>
AuthorDate: Tue Nov 17 19:18:03 2026 +0530

    [Relax][Frontend][ONNX] Add QLinearMatMul operator converter (#20951)
    
    This PR adds support for the ONNX `QLinearMatMul` operator in the Relax
    ONNX frontend, resolving #20958.
    
    ### Key Changes
    - In `python/tvm/relax/frontend/onnx/onnx_frontend.py`:
      - Implemented `QLinearMatMul` op converter supporting opset 10 and 21.
    - Widens inputs to `int32` before zero-point subtraction to prevent
    arithmetic overflow.
    - Computes matrix multiplication via `relax.op.matmul(...,
    out_dtype="int32")` for 32-bit integer accumulation.
      - Supports per-row and per-column scale and zero-point broadcasting.
    - Handles requantization: `round(scaled) + y_zero_point` and clamps to
    output data type range (`uint8`, `int8`, `uint16`, `int16`).
      - Registered `"QLinearMatMul": QLinearMatMul` in `_get_convert_map()`.
    - In `tests/python/relax/test_frontend_onnx.py`:
    - Added `test_qlinearmatmul` testing 2D uint8, 2D int8, 3D batched
    int8/uint8, and opset 10/21, validated against ONNX Runtime.
    
    Closes #20958.
    
    ---------
    
    Co-authored-by: Masahiro Hiramori <[email protected]>
---
 python/tvm/relax/frontend/onnx/onnx_frontend.py | 216 ++++++++++++
 tests/python/relax/test_frontend_onnx.py        | 447 ++++++++++++++++++++++++
 2 files changed, 663 insertions(+)

diff --git a/python/tvm/relax/frontend/onnx/onnx_frontend.py 
b/python/tvm/relax/frontend/onnx/onnx_frontend.py
index ba684062d7..95c26aa55f 100644
--- a/python/tvm/relax/frontend/onnx/onnx_frontend.py
+++ b/python/tvm/relax/frontend/onnx/onnx_frontend.py
@@ -6206,6 +6206,221 @@ class MatMulInteger(OnnxOpConverter):
         return relax.op.matmul(a, b, out_dtype="int32")  # Output is int32 per 
ONNX spec
 
 
+class QLinearMatMul(OnnxOpConverter):
+    """Converts an onnx QLinearMatMul node into an equivalent Relax 
expression."""
+
+    @classmethod
+    def _squeeze_singleton(cls, bb, param):
+        """Squeeze singleton tensors (all dimensions equal to 1, e.g. [1], [1, 
1]) to 0D scalars.
+
+        Per-tensor scales and zero points in ONNX can be provided as scalars 
(0D) or singleton
+        tensors (1D [1], 2D [1, 1], etc.). Normalizing them to scalars 
prevents them from
+        accidentally expanding output ranks during matrix-vector and 
vector-matrix multiplications.
+        """
+        if param is None:
+            return None
+        param_norm = bb.normalize(param)
+        if not hasattr(param_norm.ty, "shape"):
+            return param
+        p_shape = param_norm.ty.shape
+        if len(p_shape) > 0 and all(
+            isinstance(d, int | tvm_tirx.ir.IntImm) and int(d) == 1 for d in 
p_shape
+        ):
+            return relax.op.squeeze(param)
+        return param
+
+    @classmethod
+    def _align_output_param(cls, bb, param, a_norm, b_norm):
+        if param is None:
+            return None
+        param = cls._squeeze_singleton(bb, param)
+        param_norm = bb.normalize(param)
+        if not hasattr(param_norm.ty, "shape"):
+            return param
+        p_shape = param_norm.ty.shape
+        p_ndim = len(p_shape)
+        if p_ndim == 0:
+            return param
+
+        a_shape = a_norm.ty.shape if hasattr(a_norm.ty, "shape") else None
+        b_shape = b_norm.ty.shape if hasattr(b_norm.ty, "shape") else None
+        a_rank = len(a_shape) if a_shape is not None else 2
+        b_rank = len(b_shape) if b_shape is not None else 2
+
+        # When both a and b are 1D vectors, matmul output is scalar (0D)
+        if a_rank == 1 and b_rank == 1:
+            return relax.op.squeeze(param)
+
+        # When b_rank == 1, output of matmul is [..., M], no trailing column 
axis
+        if b_rank == 1:
+            if (
+                p_ndim >= 2
+                and isinstance(p_shape[-1], int | tvm_tirx.ir.IntImm)
+                and int(p_shape[-1]) == 1
+            ):
+                return relax.op.squeeze(param, axis=[-1])
+            return param
+
+        # When a_rank == 1, output of matmul is [..., N], no leading row axis
+        if a_rank == 1:
+            if (
+                p_ndim >= 2
+                and isinstance(p_shape[-2], int | tvm_tirx.ir.IntImm)
+                and int(p_shape[-2]) == 1
+            ):
+                return relax.op.squeeze(param, axis=[-2])
+            return param
+
+        # Both a and b have rank >= 2, output is [..., M, N]
+        if p_ndim == 1:
+            # Check if it corresponds to row dimension M
+            if a_shape is not None and a_rank >= 2:
+                row_dim = a_shape[-2]
+                if isinstance(row_dim, int | tvm_tirx.ir.IntImm) and 
int(row_dim) == int(
+                    p_shape[0]
+                ):
+                    return relax.op.expand_dims(param, axis=-1)
+            # Check if it corresponds to col dimension N
+            if b_shape is not None and b_rank >= 2:
+                col_dim = b_shape[-1]
+                if isinstance(col_dim, int | tvm_tirx.ir.IntImm) and 
int(col_dim) == int(
+                    p_shape[0]
+                ):
+                    return relax.op.expand_dims(param, axis=0)
+
+        return param
+
+    @classmethod
+    def _impl_v10(cls, bb, inputs, attr, params):
+        a = inputs[0]
+        a_scale = inputs[1]
+        a_zero_point = inputs[2] if len(inputs) > 2 and inputs[2] is not None 
else None
+        b = inputs[3]
+        b_scale = inputs[4]
+        b_zero_point = inputs[5] if len(inputs) > 5 and inputs[5] is not None 
else None
+        y_scale = inputs[6]
+        y_zero_point = inputs[7] if len(inputs) > 7 and inputs[7] is not None 
else None
+
+        if y_zero_point is None:
+            y_zero_point = relax.const(0, "uint8")
+        out_dtype = y_zero_point.ty.dtype.dtype
+
+        # Reject float8 input and output dtypes
+        for name, val in [("A", a), ("B", b), ("Y", y_zero_point)]:
+            if val is not None and hasattr(val, "ty") and hasattr(val.ty, 
"dtype"):
+                dtype_str = str(val.ty.dtype.dtype)
+                if dtype_str.startswith("float8") or "e4m3" in dtype_str or 
"e5m2" in dtype_str:
+                    raise NotImplementedError(
+                        f"Float8 operands and outputs ({name} has dtype 
{dtype_str}) "
+                        "are currently not supported in QLinearMatMul."
+                    )
+
+        # Normalize singleton [1]-shape and [1, 1]-shape parameters to 0D 
scalars
+        # BEFORE any rank expansion or alignment, so they do not accidentally
+        # increase the output rank in vector-matrix, matrix-vector, or dot 
products.
+        a_scale = cls._squeeze_singleton(bb, a_scale)
+        a_zero_point = cls._squeeze_singleton(bb, a_zero_point)
+        b_scale = cls._squeeze_singleton(bb, b_scale)
+        b_zero_point = cls._squeeze_singleton(bb, b_zero_point)
+        y_scale = cls._squeeze_singleton(bb, y_scale)
+        y_zero_point = cls._squeeze_singleton(bb, y_zero_point)
+
+        a_norm = bb.normalize(a)
+        b_norm = bb.normalize(b)
+        a_shape = a_norm.ty.shape if hasattr(a_norm.ty, "shape") else None
+        b_shape = b_norm.ty.shape if hasattr(b_norm.ty, "shape") else None
+        a_rank = len(a_shape) if a_shape is not None else 2
+        b_rank = len(b_shape) if b_shape is not None else 2
+
+        # Widen to int32 before arithmetic to prevent overflow
+        a = relax.op.astype(a, "int32")
+        b = relax.op.astype(b, "int32")
+
+        if a_zero_point is not None:
+            a_zp = relax.op.astype(a_zero_point, "int32")
+            a_zp = bb.normalize(a_zp)
+            if a_rank >= 2 and len(a_zp.ty.shape) == 1:
+                a_zp = relax.op.expand_dims(a_zp, axis=-1)
+            a = relax.op.subtract(a, a_zp)
+
+        if b_zero_point is not None:
+            b_zp = relax.op.astype(b_zero_point, "int32")
+            b_zp = bb.normalize(b_zp)
+            if b_rank >= 2 and len(b_zp.ty.shape) == 1:
+                b_zp = relax.op.expand_dims(b_zp, axis=0)
+            b = relax.op.subtract(b, b_zp)
+
+        # 32-bit integer accumulation
+        matmul_int32 = relax.op.matmul(a, b, out_dtype="int32")
+
+        # Broadcast a_scale: when both a and b are rank >= 2, expand row 
dimension to broadcast over columns
+        a_scale_norm = bb.normalize(a_scale)
+        if hasattr(a_scale_norm.ty, "shape"):
+            as_shape = a_scale_norm.ty.shape
+            if a_rank >= 2 and b_rank >= 2:
+                if len(as_shape) == 1:
+                    a_scale = relax.op.expand_dims(a_scale, axis=-1)
+            elif b_rank == 1:
+                if (
+                    len(as_shape) >= 2
+                    and isinstance(as_shape[-1], int | tvm_tirx.ir.IntImm)
+                    and int(as_shape[-1]) == 1
+                ):
+                    a_scale = relax.op.squeeze(a_scale, axis=[-1])
+
+        # Broadcast b_scale: when both a and b are rank >= 2, expand column 
dimension to broadcast over rows
+        b_scale_norm = bb.normalize(b_scale)
+        if hasattr(b_scale_norm.ty, "shape"):
+            bs_shape = b_scale_norm.ty.shape
+            if a_rank >= 2 and b_rank >= 2:
+                if len(bs_shape) == 1:
+                    b_scale = relax.op.expand_dims(b_scale, axis=0)
+            elif a_rank == 1:
+                if (
+                    len(bs_shape) >= 2
+                    and isinstance(bs_shape[-2], int | tvm_tirx.ir.IntImm)
+                    and int(bs_shape[-2]) == 1
+                ):
+                    b_scale = relax.op.squeeze(b_scale, axis=[-2])
+
+        # Ensure any resulting singleton scale or output parameter is squeezed
+        a_scale = cls._squeeze_singleton(bb, a_scale)
+        b_scale = cls._squeeze_singleton(bb, b_scale)
+        y_scale = cls._squeeze_singleton(bb, y_scale)
+        y_zero_point = cls._squeeze_singleton(bb, y_zero_point)
+
+        y_scale = cls._align_output_param(bb, y_scale, a_norm, b_norm)
+        y_zero_point = cls._align_output_param(bb, y_zero_point, a_norm, 
b_norm)
+
+        matmul_float = relax.op.astype(matmul_int32, "float32")
+        a_scale_f = relax.op.astype(a_scale, "float32")
+        b_scale_f = relax.op.astype(b_scale, "float32")
+        y_scale_f = relax.op.astype(y_scale, "float32")
+
+        scale_mult = relax.op.divide(relax.op.multiply(a_scale_f, b_scale_f), 
y_scale_f)
+        scaled = relax.op.multiply(matmul_float, scale_mult)
+        rounded = relax.op.round(scaled)
+        y_val = relax.op.add(rounded, relax.op.astype(y_zero_point, "float32"))
+
+        if out_dtype == "uint8":
+            min_val, max_val = 0, 255
+        elif out_dtype == "int8":
+            min_val, max_val = -128, 127
+        elif out_dtype == "uint16":
+            min_val, max_val = 0, 65535
+        elif out_dtype == "int16":
+            min_val, max_val = -32768, 32767
+        else:
+            min_val, max_val = 0, 255
+
+        clipped = relax.op.clip(y_val, min_val, max_val)
+        return relax.op.astype(clipped, out_dtype)
+
+    @classmethod
+    def _impl_v21(cls, bb, inputs, attr, params):
+        return cls._impl_v10(bb, inputs, attr, params)
+
+
 class DFT(OnnxOpConverter):
     """Converts an onnx DFT node into an equivalent Relax expression."""
 
@@ -6457,6 +6672,7 @@ def _get_convert_map():
         "MatMul": MatMul,
         "MatMulInteger": MatMulInteger,
         "MatMulInteger16": MatMulInteger16,
+        "QLinearMatMul": QLinearMatMul,
         "Reshape": Reshape,
         "Sigmoid": Sigmoid,
         "Softmax": Softmax,
diff --git a/tests/python/relax/test_frontend_onnx.py 
b/tests/python/relax/test_frontend_onnx.py
index ca88a9259b..1a5113ca2d 100644
--- a/tests/python/relax/test_frontend_onnx.py
+++ b/tests/python/relax/test_frontend_onnx.py
@@ -14812,6 +14812,453 @@ def test_quantizelinear_output_dtype_mismatch(opset):
         from_onnx(model, opset=opset, keep_params_in_input=True)
 
 
+def test_qlinearmatmul():
+    """Verify numerical correctness of QLinearMatMul against ONNX Runtime and 
reference."""
+
+    np_dtype_map = {
+        "uint8": (np.uint8, TensorProto.UINT8),
+        "int8": (np.int8, TensorProto.INT8),
+    }
+
+    def verify_qlinearmatmul(
+        a_shape,
+        b_shape,
+        a_dtype="uint8",
+        b_dtype="uint8",
+        y_dtype="uint8",
+        a_scale_shape=(),
+        b_scale_shape=(),
+        y_scale_shape=(),
+        opset=10,
+        seed=42,
+    ):
+        rng = np.random.RandomState(seed)
+        a_np_dtype, a_proto = np_dtype_map[a_dtype]
+        b_np_dtype, b_proto = np_dtype_map[b_dtype]
+        y_np_dtype, y_proto = np_dtype_map[y_dtype]
+
+        if a_dtype == "uint8":
+            a = rng.randint(120, 136, size=a_shape).astype(a_np_dtype)
+            a_zp = np.full(a_scale_shape, 128, dtype=a_np_dtype)
+        else:
+            a = rng.randint(-8, 8, size=a_shape).astype(a_np_dtype)
+            a_zp = np.full(a_scale_shape, 0, dtype=a_np_dtype)
+
+        if b_dtype == "uint8":
+            b = rng.randint(120, 136, size=b_shape).astype(b_np_dtype)
+            b_zp = np.full(b_scale_shape, 128, dtype=b_np_dtype)
+        else:
+            b = rng.randint(-8, 8, size=b_shape).astype(b_np_dtype)
+            b_zp = np.full(b_scale_shape, 0, dtype=b_np_dtype)
+
+        if y_dtype == "uint8":
+            y_zp = np.full(y_scale_shape, 128, dtype=y_np_dtype)
+        else:
+            y_zp = np.full(y_scale_shape, 0, dtype=y_np_dtype)
+
+        a_scale = np.full(a_scale_shape, 0.5, dtype=np.float32)
+        b_scale = np.full(b_scale_shape, 0.25, dtype=np.float32)
+        y_scale = np.full(y_scale_shape, 2.0, dtype=np.float32)
+
+        node = helper.make_node(
+            "QLinearMatMul",
+            inputs=["a", "a_scale", "a_zp", "b", "b_scale", "b_zp", "y_scale", 
"y_zp"],
+            outputs=["y"],
+        )
+
+        y_shape = list(np.matmul(np.zeros(a_shape), np.zeros(b_shape)).shape)
+
+        graph = helper.make_graph(
+            [node],
+            "qlinearmatmul_test",
+            inputs=[
+                helper.make_tensor_value_info("a", a_proto, a_shape),
+                helper.make_tensor_value_info("a_scale", TensorProto.FLOAT, 
list(a_scale_shape)),
+                helper.make_tensor_value_info("a_zp", a_proto, 
list(a_scale_shape)),
+                helper.make_tensor_value_info("b", b_proto, b_shape),
+                helper.make_tensor_value_info("b_scale", TensorProto.FLOAT, 
list(b_scale_shape)),
+                helper.make_tensor_value_info("b_zp", b_proto, 
list(b_scale_shape)),
+                helper.make_tensor_value_info("y_scale", TensorProto.FLOAT, 
list(y_scale_shape)),
+                helper.make_tensor_value_info("y_zp", y_proto, 
list(y_scale_shape)),
+            ],
+            outputs=[helper.make_tensor_value_info("y", y_proto, y_shape)],
+        )
+        model = helper.make_model(graph, 
opset_imports=[helper.make_opsetid("", opset)])
+        inputs = {
+            "a": a,
+            "a_scale": a_scale,
+            "a_zp": a_zp,
+            "b": b,
+            "b_scale": b_scale,
+            "b_zp": b_zp,
+            "y_scale": y_scale,
+            "y_zp": y_zp,
+        }
+        check_correctness(model, inputs=inputs, opset=opset)
+
+    def verify_qlinearmatmul_ref(
+        a_shape,
+        b_shape,
+        a_dtype="uint8",
+        b_dtype="uint8",
+        y_dtype="uint8",
+        a_scale_shape=(),
+        b_scale_shape=(),
+        y_scale_shape=(),
+        opset=10,
+        seed=42,
+    ):
+        rng = np.random.RandomState(seed)
+        a_np_dtype, a_proto = np_dtype_map[a_dtype]
+        b_np_dtype, b_proto = np_dtype_map[b_dtype]
+        y_np_dtype, y_proto = np_dtype_map[y_dtype]
+
+        if a_dtype == "uint8":
+            a = rng.randint(120, 136, size=a_shape).astype(a_np_dtype)
+            a_zp = np.full(a_scale_shape, 128, dtype=a_np_dtype)
+        else:
+            a = rng.randint(-8, 8, size=a_shape).astype(a_np_dtype)
+            a_zp = np.full(a_scale_shape, 0, dtype=a_np_dtype)
+
+        if b_dtype == "uint8":
+            b = rng.randint(120, 136, size=b_shape).astype(b_np_dtype)
+            b_zp = np.full(b_scale_shape, 128, dtype=b_np_dtype)
+        else:
+            b = rng.randint(-8, 8, size=b_shape).astype(b_np_dtype)
+            b_zp = np.full(b_scale_shape, 0, dtype=b_np_dtype)
+
+        if y_dtype == "uint8":
+            y_zp = np.full(y_scale_shape, 128, dtype=y_np_dtype)
+        else:
+            y_zp = np.full(y_scale_shape, 0, dtype=y_np_dtype)
+
+        a_scale = np.full(a_scale_shape, 0.5, dtype=np.float32)
+        b_scale = np.full(b_scale_shape, 0.25, dtype=np.float32)
+        y_scale = np.full(y_scale_shape, 2.0, dtype=np.float32)
+
+        # Reference calculation
+        a_int32 = a.astype(np.int32)
+        b_int32 = b.astype(np.int32)
+        a_zp_i = a_zp.astype(np.int32)
+        b_zp_i = b_zp.astype(np.int32)
+        if a.ndim >= 2 and a_zp_i.ndim == 1:
+            a_zp_i = np.expand_dims(a_zp_i, axis=-1)
+        if b.ndim >= 2 and b_zp_i.ndim == 1:
+            b_zp_i = np.expand_dims(b_zp_i, axis=0)
+
+        matmul_int32 = np.matmul(a_int32 - a_zp_i, b_int32 - b_zp_i)
+        as_f = a_scale.copy()
+        bs_f = b_scale.copy()
+        ys_f = y_scale.copy()
+        yz_f = y_zp.astype(np.float32)
+
+        if b.ndim >= 2 and as_f.ndim == 1:
+            as_f = np.expand_dims(as_f, axis=-1)
+        elif b.ndim == 1 and as_f.ndim >= 2 and as_f.shape[-1] == 1:
+            as_f = np.squeeze(as_f, axis=-1)
+
+        if a.ndim >= 2 and bs_f.ndim == 1:
+            bs_f = np.expand_dims(bs_f, axis=0)
+        elif a.ndim == 1 and bs_f.ndim >= 2 and bs_f.shape[-2] == 1:
+            bs_f = np.squeeze(bs_f, axis=-2)
+
+        if b.ndim >= 2 and a.ndim >= 2 and ys_f.ndim == 1:
+            if a.shape[-2] == ys_f.shape[0]:
+                ys_f = np.expand_dims(ys_f, axis=-1)
+                yz_f = np.expand_dims(yz_f, axis=-1)
+            elif b.shape[-1] == ys_f.shape[0]:
+                ys_f = np.expand_dims(ys_f, axis=0)
+                yz_f = np.expand_dims(yz_f, axis=0)
+
+        scaled = matmul_int32.astype(np.float32) * (as_f * bs_f) / ys_f
+        rounded = np.round(scaled) + yz_f
+        min_v, max_v = (0, 255) if y_dtype == "uint8" else (-128, 127)
+        ref_out = np.clip(rounded, min_v, max_v).astype(y_np_dtype)
+
+        node = helper.make_node(
+            "QLinearMatMul",
+            inputs=["a", "a_scale", "a_zp", "b", "b_scale", "b_zp", "y_scale", 
"y_zp"],
+            outputs=["y"],
+        )
+        y_shape = list(ref_out.shape)
+
+        graph = helper.make_graph(
+            [node],
+            "qlinearmatmul_ref_test",
+            inputs=[
+                helper.make_tensor_value_info("a", a_proto, a_shape),
+                helper.make_tensor_value_info("a_scale", TensorProto.FLOAT, 
list(a_scale_shape)),
+                helper.make_tensor_value_info("a_zp", a_proto, 
list(a_scale_shape)),
+                helper.make_tensor_value_info("b", b_proto, b_shape),
+                helper.make_tensor_value_info("b_scale", TensorProto.FLOAT, 
list(b_scale_shape)),
+                helper.make_tensor_value_info("b_zp", b_proto, 
list(b_scale_shape)),
+                helper.make_tensor_value_info("y_scale", TensorProto.FLOAT, 
list(y_scale_shape)),
+                helper.make_tensor_value_info("y_zp", y_proto, 
list(y_scale_shape)),
+            ],
+            outputs=[helper.make_tensor_value_info("y", y_proto, y_shape)],
+        )
+        model = helper.make_model(graph, 
opset_imports=[helper.make_opsetid("", opset)])
+        inputs = {
+            "a": a,
+            "a_scale": a_scale,
+            "a_zp": a_zp,
+            "b": b,
+            "b_scale": b_scale,
+            "b_zp": b_zp,
+            "y_scale": y_scale,
+            "y_zp": y_zp,
+        }
+        tvm_out = run_in_tvm(model, inputs=inputs, opset=opset)
+        tvm_val = tvm_out[0].numpy() if isinstance(tvm_out, list | tuple) else 
tvm_out.numpy()
+        assert tvm_val.shape == ref_out.shape
+        np.testing.assert_allclose(tvm_val, ref_out, atol=1e-5)
+
+    # 2D uint8 with in-range output
+    verify_qlinearmatmul(
+        [2, 3], [3, 4], a_dtype="uint8", b_dtype="uint8", y_dtype="uint8", 
opset=10
+    )
+    # 2D int8
+    verify_qlinearmatmul([3, 4], [4, 2], a_dtype="int8", b_dtype="int8", 
y_dtype="int8", opset=10)
+    # 3D batched int8
+    verify_qlinearmatmul(
+        [2, 3, 4], [2, 4, 5], a_dtype="int8", b_dtype="int8", y_dtype="int8", 
opset=10
+    )
+    # 3D batched uint8
+    verify_qlinearmatmul(
+        [2, 3, 4], [2, 4, 5], a_dtype="uint8", b_dtype="uint8", 
y_dtype="uint8", opset=10
+    )
+    # Opset 21 uint8
+    verify_qlinearmatmul(
+        [2, 3], [3, 4], a_dtype="uint8", b_dtype="uint8", y_dtype="uint8", 
opset=21
+    )
+    # Opset 21 int8
+    verify_qlinearmatmul([2, 3], [3, 4], a_dtype="int8", b_dtype="int8", 
y_dtype="int8", opset=21)
+    # Mixed signedness uint8/int8
+    verify_qlinearmatmul([2, 3], [3, 4], a_dtype="uint8", b_dtype="int8", 
y_dtype="uint8", opset=10)
+    # Matrix-vector (2D x 1D -> 1D)
+    verify_qlinearmatmul([2, 3], [3], a_dtype="uint8", b_dtype="uint8", 
y_dtype="uint8", opset=10)
+    # Vector-matrix (1D x 2D -> 1D)
+    verify_qlinearmatmul([3], [3, 4], a_dtype="uint8", b_dtype="uint8", 
y_dtype="uint8", opset=10)
+    # Batched matrix-vector (3D x 1D -> 2D)
+    verify_qlinearmatmul(
+        [2, 3, 4], [4], a_dtype="uint8", b_dtype="uint8", y_dtype="uint8", 
opset=10
+    )
+    # Per-column parameter on B
+    verify_qlinearmatmul(
+        [2, 3], [3, 4], a_dtype="uint8", b_dtype="uint8", y_dtype="uint8", 
b_scale_shape=(4,)
+    )
+    # Batched per-column parameter on B
+    verify_qlinearmatmul(
+        [2, 3, 4],
+        [2, 4, 5],
+        a_dtype="uint8",
+        b_dtype="uint8",
+        y_dtype="uint8",
+        b_scale_shape=(2, 1, 5),
+    )
+
+    # Reference tests for cases unsupported by ORT (per-row A and int8/uint8)
+    verify_qlinearmatmul_ref(
+        [2, 3], [3, 4], a_dtype="uint8", b_dtype="uint8", y_dtype="uint8", 
a_scale_shape=(2,)
+    )
+    verify_qlinearmatmul_ref(
+        [2, 3, 4],
+        [2, 4, 5],
+        a_dtype="uint8",
+        b_dtype="uint8",
+        y_dtype="uint8",
+        a_scale_shape=(2, 3, 1),
+    )
+    verify_qlinearmatmul_ref([2, 3], [3, 4], a_dtype="int8", b_dtype="uint8", 
y_dtype="int8")
+
+
+def test_qlinearmatmul_float8_unsupported():
+    """Verify that float8 inputs and outputs in QLinearMatMul are rejected 
with NotImplementedError."""
+    node = helper.make_node(
+        "QLinearMatMul",
+        inputs=["a", "a_scale", "a_zp", "b", "b_scale", "b_zp", "y_scale", 
"y_zp"],
+        outputs=["y"],
+    )
+    # TensorProto.FLOAT8E4M3FN is type 17
+    graph = helper.make_graph(
+        [node],
+        "test_float8",
+        inputs=[
+            helper.make_tensor_value_info("a", 17, [2, 3]),
+            helper.make_tensor_value_info("a_scale", TensorProto.FLOAT, []),
+            helper.make_tensor_value_info("a_zp", 17, []),
+            helper.make_tensor_value_info("b", 17, [3, 4]),
+            helper.make_tensor_value_info("b_scale", TensorProto.FLOAT, []),
+            helper.make_tensor_value_info("b_zp", 17, []),
+            helper.make_tensor_value_info("y_scale", TensorProto.FLOAT, []),
+            helper.make_tensor_value_info("y_zp", 17, []),
+        ],
+        outputs=[helper.make_tensor_value_info("y", 17, [2, 4])],
+    )
+    model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 
21)])
+    with pytest.raises(NotImplementedError, match="Float8 operands and 
outputs"):
+        from_onnx(model, opset=21, keep_params_in_input=True)
+
+
+def test_qlinearmatmul_singleton_scales_output_shape():
+    """Singleton [1]-shape scales and zero points must not alter the MatMul 
output rank.
+
+    A valid a_scale:[1] on [3] @ [3,4] must still produce output [4], not 
[1,4].
+    """
+    a = np.array([2, 3, 4], dtype=np.uint8)
+    b = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0]], dtype=np.uint8)
+    # All quantization parameters as [1]-shape (singleton) arrays
+    a_scale = np.array([0.5], dtype=np.float32)
+    a_zp = np.array([0], dtype=np.uint8)
+    b_scale = np.array([0.5], dtype=np.float32)
+    b_zp = np.array([0], dtype=np.uint8)
+    y_scale = np.array([1.0], dtype=np.float32)
+    y_zp = np.array([0], dtype=np.uint8)
+
+    node = helper.make_node(
+        "QLinearMatMul",
+        inputs=["a", "a_scale", "a_zp", "b", "b_scale", "b_zp", "y_scale", 
"y_zp"],
+        outputs=["y"],
+    )
+    graph = helper.make_graph(
+        [node],
+        "singleton_test",
+        inputs=[
+            helper.make_tensor_value_info("a", TensorProto.UINT8, [3]),
+            helper.make_tensor_value_info("a_scale", TensorProto.FLOAT, [1]),
+            helper.make_tensor_value_info("a_zp", TensorProto.UINT8, [1]),
+            helper.make_tensor_value_info("b", TensorProto.UINT8, [3, 4]),
+            helper.make_tensor_value_info("b_scale", TensorProto.FLOAT, [1]),
+            helper.make_tensor_value_info("b_zp", TensorProto.UINT8, [1]),
+            helper.make_tensor_value_info("y_scale", TensorProto.FLOAT, [1]),
+            helper.make_tensor_value_info("y_zp", TensorProto.UINT8, [1]),
+        ],
+        outputs=[helper.make_tensor_value_info("y", TensorProto.UINT8, [4])],
+    )
+    model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 
10)])
+    inputs = {
+        "a": a,
+        "a_scale": a_scale,
+        "a_zp": a_zp,
+        "b": b,
+        "b_scale": b_scale,
+        "b_zp": b_zp,
+        "y_scale": y_scale,
+        "y_zp": y_zp,
+    }
+    tvm_out = run_in_tvm(model, inputs=inputs, opset=10)
+    tvm_val = tvm_out[0].numpy() if isinstance(tvm_out, list | tuple) else 
tvm_out.numpy()
+    # Output must be rank-1, shape [4], not rank-2 [1,4]
+    assert tvm_val.shape == (4,), f"Expected shape (4,) but got 
{tvm_val.shape}"
+    # Reference: (a - a_zp) @ (b - b_zp) * (a_scale * b_scale / y_scale) + y_zp
+    ref = np.clip(
+        np.round(
+            np.matmul(a.astype(np.int32), 
b.astype(np.int32)).astype(np.float32) * (0.5 * 0.5 / 1.0)
+        ),
+        0,
+        255,
+    ).astype(np.uint8)
+    np.testing.assert_array_equal(tvm_val, ref)
+
+    # Matrix-vector case: [4, 3] @ [3] -> [4]
+    a_mv = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [1, 0, 1]], 
dtype=np.uint8)
+    b_mv = np.array([2, 1, 0], dtype=np.uint8)
+    node_mv = helper.make_node(
+        "QLinearMatMul",
+        inputs=["a", "a_scale", "a_zp", "b", "b_scale", "b_zp", "y_scale", 
"y_zp"],
+        outputs=["y"],
+    )
+    graph_mv = helper.make_graph(
+        [node_mv],
+        "singleton_mv_test",
+        inputs=[
+            helper.make_tensor_value_info("a", TensorProto.UINT8, [4, 3]),
+            helper.make_tensor_value_info("a_scale", TensorProto.FLOAT, [1]),
+            helper.make_tensor_value_info("a_zp", TensorProto.UINT8, [1]),
+            helper.make_tensor_value_info("b", TensorProto.UINT8, [3]),
+            helper.make_tensor_value_info("b_scale", TensorProto.FLOAT, [1]),
+            helper.make_tensor_value_info("b_zp", TensorProto.UINT8, [1]),
+            helper.make_tensor_value_info("y_scale", TensorProto.FLOAT, [1]),
+            helper.make_tensor_value_info("y_zp", TensorProto.UINT8, [1]),
+        ],
+        outputs=[helper.make_tensor_value_info("y", TensorProto.UINT8, [4])],
+    )
+    model_mv = helper.make_model(graph_mv, 
opset_imports=[helper.make_opsetid("", 10)])
+    inputs_mv = {
+        "a": a_mv,
+        "a_scale": a_scale,
+        "a_zp": a_zp,
+        "b": b_mv,
+        "b_scale": b_scale,
+        "b_zp": b_zp,
+        "y_scale": y_scale,
+        "y_zp": y_zp,
+    }
+    tvm_out_mv = run_in_tvm(model_mv, inputs=inputs_mv, opset=10)
+    tvm_val_mv = (
+        tvm_out_mv[0].numpy() if isinstance(tvm_out_mv, list | tuple) else 
tvm_out_mv.numpy()
+    )
+    assert tvm_val_mv.shape == (4,), f"Expected shape (4,) but got 
{tvm_val_mv.shape}"
+    ref_mv = np.clip(
+        np.round(
+            np.matmul(a_mv.astype(np.int32), 
b_mv.astype(np.int32)).astype(np.float32)
+            * (0.5 * 0.5 / 1.0)
+        ),
+        0,
+        255,
+    ).astype(np.uint8)
+    np.testing.assert_array_equal(tvm_val_mv, ref_mv)
+
+    # Vector-vector dot product case: [3] @ [3] -> ()
+    a_vv = np.array([2, 3, 4], dtype=np.uint8)
+    b_vv = np.array([1, 2, 1], dtype=np.uint8)
+    node_vv = helper.make_node(
+        "QLinearMatMul",
+        inputs=["a", "a_scale", "a_zp", "b", "b_scale", "b_zp", "y_scale", 
"y_zp"],
+        outputs=["y"],
+    )
+    graph_vv = helper.make_graph(
+        [node_vv],
+        "singleton_vv_test",
+        inputs=[
+            helper.make_tensor_value_info("a", TensorProto.UINT8, [3]),
+            helper.make_tensor_value_info("a_scale", TensorProto.FLOAT, [1]),
+            helper.make_tensor_value_info("a_zp", TensorProto.UINT8, [1]),
+            helper.make_tensor_value_info("b", TensorProto.UINT8, [3]),
+            helper.make_tensor_value_info("b_scale", TensorProto.FLOAT, [1]),
+            helper.make_tensor_value_info("b_zp", TensorProto.UINT8, [1]),
+            helper.make_tensor_value_info("y_scale", TensorProto.FLOAT, [1]),
+            helper.make_tensor_value_info("y_zp", TensorProto.UINT8, [1]),
+        ],
+        outputs=[helper.make_tensor_value_info("y", TensorProto.UINT8, [])],
+    )
+    model_vv = helper.make_model(graph_vv, 
opset_imports=[helper.make_opsetid("", 10)])
+    inputs_vv = {
+        "a": a_vv,
+        "a_scale": a_scale,
+        "a_zp": a_zp,
+        "b": b_vv,
+        "b_scale": b_scale,
+        "b_zp": b_zp,
+        "y_scale": y_scale,
+        "y_zp": y_zp,
+    }
+    tvm_out_vv = run_in_tvm(model_vv, inputs=inputs_vv, opset=10)
+    tvm_val_vv = (
+        tvm_out_vv[0].numpy() if isinstance(tvm_out_vv, list | tuple) else 
tvm_out_vv.numpy()
+    )
+    assert tvm_val_vv.shape == (), f"Expected scalar shape () but got 
{tvm_val_vv.shape}"
+    ref_vv = np.clip(
+        np.round(
+            np.matmul(a_vv.astype(np.int32), 
b_vv.astype(np.int32)).astype(np.float32)
+            * (0.5 * 0.5 / 1.0)
+        ),
+        0,
+        255,
+    ).astype(np.uint8)
+    np.testing.assert_array_equal(tvm_val_vv, ref_vv)
+
+
 def test_dft():
     """Verify numerical correctness of DFT against ONNX Runtime."""
 

Reply via email to