This is an automated email from the ASF dual-hosted git repository.
guan404ming pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/mahout.git
The following commit(s) were added to refs/heads/main by this push:
new 6a1ea6a31 fix(qdp): reject invalid non-amplitude benchmark combos
(#1303)
6a1ea6a31 is described below
commit 6a1ea6a315054975e0551135f4bf88de4e3e0f9a
Author: Vic Wen <[email protected]>
AuthorDate: Thu May 14 10:33:03 2026 +0800
fix(qdp): reject invalid non-amplitude benchmark combos (#1303)
* fix(qdp): reject invalid non-amplitude benchmark combos
* test(qdp): cover non-happy benchmark CLI cases
---
qdp/qdp-python/benchmark/README.md | 13 +-
qdp/qdp-python/benchmark/benchmark_latency.py | 20 +--
qdp/qdp-python/benchmark/benchmark_throughput.py | 20 +--
qdp/qdp-python/benchmark/utils.py | 35 ++++
testing/conftest.py | 1 +
.../qdp_python/test_benchmark_cli_validation.py | 185 +++++++++++++++++++++
testing/qdp_python/test_fallback.py | 5 +-
7 files changed, 253 insertions(+), 26 deletions(-)
diff --git a/qdp/qdp-python/benchmark/README.md
b/qdp/qdp-python/benchmark/README.md
index 348441fdc..0dbea81ee 100644
--- a/qdp/qdp-python/benchmark/README.md
+++ b/qdp/qdp-python/benchmark/README.md
@@ -80,7 +80,7 @@ Notes:
```bash
uv run --project qdp/qdp-python python
qdp/qdp-python/benchmark/benchmark_latency.py --qubits 16 --batches 200
--batch-size 64 --prefetch 16
uv run --project qdp/qdp-python python
qdp/qdp-python/benchmark/benchmark_latency.py --frameworks mahout,pennylane
-uv run --project qdp/qdp-python python
qdp/qdp-python/benchmark/benchmark_latency.py --encoding-method basis
+uv run --project qdp/qdp-python python
qdp/qdp-python/benchmark/benchmark_latency.py --frameworks mahout
--encoding-method basis
```
Notes:
@@ -88,6 +88,9 @@ Notes:
- `--frameworks` is a comma-separated list or `all`.
Options: `mahout`, `pennylane`, `qiskit-init`, `qiskit-statevector`.
- `--encoding-method` selects the encoding method: `amplitude` (default),
`angle`, `basis`, `iqp`, or `iqp-z`.
+- In these CLIs, non-`amplitude` benchmark modes are currently Mahout-only.
+ Cross-framework parity exists only for `amplitude`, so `angle`, `basis`,
+ `iqp`, and `iqp-z` must use `--frameworks mahout`.
- The latency test reports average milliseconds per vector.
- Flags:
- `--qubits`: controls the input length together with `--encoding-method`.
@@ -110,14 +113,18 @@ output.
```bash
uv run --project qdp/qdp-python python
qdp/qdp-python/benchmark/benchmark_throughput.py --qubits 16 --batches 200
--batch-size 64 --prefetch 16
uv run --project qdp/qdp-python python
qdp/qdp-python/benchmark/benchmark_throughput.py --frameworks mahout,pennylane
-uv run --project qdp/qdp-python python
qdp/qdp-python/benchmark/benchmark_throughput.py --encoding-method basis
+uv run --project qdp/qdp-python python
qdp/qdp-python/benchmark/benchmark_throughput.py --frameworks mahout
--encoding-method basis
```
Notes:
- `--frameworks` is a comma-separated list or `all`.
- Options: `mahout`, `pennylane`, `qiskit`.
+ Options: `mahout`, `mahout-amd`, `pennylane`, `pennylane-amdgpu`,
+ `pytorch-ref`, `qiskit`.
- `--encoding-method` selects the encoding method: `amplitude` (default),
`angle`, `basis`, `iqp`, or `iqp-z`.
+- In these CLIs, non-`amplitude` benchmark modes are currently Mahout-only.
+ Cross-framework parity exists only for `amplitude`, so `angle`, `basis`,
+ `iqp`, and `iqp-z` must use `--frameworks mahout`.
- For synthetic inputs, amplitude uses `2^qubits`, angle and `iqp-z` use
`qubits`,
basis uses one basis index, and `iqp` uses `qubits + qubits*(qubits-1)/2`.
- Throughput is reported in vectors/sec (higher is better).
diff --git a/qdp/qdp-python/benchmark/benchmark_latency.py
b/qdp/qdp-python/benchmark/benchmark_latency.py
index 3c736a3be..ab48faec4 100644
--- a/qdp/qdp-python/benchmark/benchmark_latency.py
+++ b/qdp/qdp-python/benchmark/benchmark_latency.py
@@ -31,7 +31,11 @@ import time
import torch
from qumat_qdp import QdpBenchmark
-from benchmark.utils import normalize_batch, prefetched_batches
+from benchmark.utils import (
+ normalize_batch,
+ prefetched_batches,
+ validate_framework_selection,
+)
BAR = "=" * 70
SEP = "-" * 70
@@ -267,16 +271,10 @@ def main() -> None:
except ValueError as exc:
parser.error(str(exc))
- # TODO: fix this with #1252 in the future.
- if args.encoding_method in {"iqp", "iqp-z"}:
- unsupported = [name for name in frameworks if name != "mahout"]
- if unsupported:
- print(
- "Warning: IQP benchmarks in this script currently support only
"
- "framework 'mahout'; skipping unsupported frameworks: "
- f"{', '.join(unsupported)}."
- )
- frameworks = ["mahout"]
+ try:
+ frameworks = validate_framework_selection(frameworks,
args.encoding_method)
+ except ValueError as exc:
+ parser.error(str(exc))
total_vectors = args.batches * args.batch_size
vector_len = _sample_dim(args.qubits, args.encoding_method)
diff --git a/qdp/qdp-python/benchmark/benchmark_throughput.py
b/qdp/qdp-python/benchmark/benchmark_throughput.py
index 481dca5c0..f174eab15 100644
--- a/qdp/qdp-python/benchmark/benchmark_throughput.py
+++ b/qdp/qdp-python/benchmark/benchmark_throughput.py
@@ -88,7 +88,11 @@ import numpy as np
import torch
from qumat_qdp import QdpBenchmark
-from benchmark.utils import normalize_batch, prefetched_batches
+from benchmark.utils import (
+ normalize_batch,
+ prefetched_batches,
+ validate_framework_selection,
+)
BAR = "=" * 70
SEP = "-" * 70
@@ -511,16 +515,10 @@ def main() -> None:
except ValueError as exc:
parser.error(str(exc))
- # TODO: fix this with #1252 in the future.
- if args.encoding_method in {"iqp", "iqp-z"}:
- unsupported = [name for name in frameworks if name != "mahout"]
- if unsupported:
- print(
- "Warning: IQP benchmarks in this script currently support only
"
- "framework 'mahout'; skipping unsupported frameworks: "
- f"{', '.join(unsupported)}."
- )
- frameworks = ["mahout"]
+ try:
+ frameworks = validate_framework_selection(frameworks,
args.encoding_method)
+ except ValueError as exc:
+ parser.error(str(exc))
total_vectors = args.batches * args.batch_size
vector_len = _sample_dim(args.qubits, args.encoding_method)
diff --git a/qdp/qdp-python/benchmark/utils.py
b/qdp/qdp-python/benchmark/utils.py
index bc9b577db..d013b73ed 100644
--- a/qdp/qdp-python/benchmark/utils.py
+++ b/qdp/qdp-python/benchmark/utils.py
@@ -31,6 +31,41 @@ import numpy as np
import torch
+def validate_framework_selection(
+ frameworks: list[str],
+ encoding_method: str,
+ *,
+ allowed_non_amplitude_frameworks: tuple[str, ...] = ("mahout",),
+) -> list[str]:
+ """
+ Validate that the selected frameworks provide a fair benchmark comparison.
+
+ These benchmark CLIs currently support non-amplitude modes only on the
+ Mahout path. Cross-framework parity is therefore available only for
+ amplitude encoding.
+ """
+ if encoding_method == "amplitude":
+ return frameworks
+
+ unsupported = [
+ framework
+ for framework in frameworks
+ if framework not in allowed_non_amplitude_frameworks
+ ]
+ if unsupported:
+ allowed = ", ".join(allowed_non_amplitude_frameworks)
+ requested = ", ".join(frameworks)
+ raise ValueError(
+ "These benchmark CLIs currently support non-amplitude encodings "
+ "only on the Mahout path; cross-framework parity exists only for "
+ "amplitude encoding. "
+ f"For encoding_method={encoding_method!r}, use frameworks limited
to: "
+ f"{allowed}. Requested: {requested}."
+ )
+
+ return frameworks
+
+
def build_sample(
seed: int, vector_len: int, encoding_method: str = "amplitude"
) -> np.ndarray:
diff --git a/testing/conftest.py b/testing/conftest.py
index 88e49deef..28c1e150d 100644
--- a/testing/conftest.py
+++ b/testing/conftest.py
@@ -63,6 +63,7 @@ def pytest_collection_modifyitems(config, items):
"test_torch_ref.py",
"test_fallback.py",
"test_benchmark_utils.py",
+ "test_benchmark_cli_validation.py",
}
for item in items:
diff --git a/testing/qdp_python/test_benchmark_cli_validation.py
b/testing/qdp_python/test_benchmark_cli_validation.py
new file mode 100644
index 000000000..6af902ea1
--- /dev/null
+++ b/testing/qdp_python/test_benchmark_cli_validation.py
@@ -0,0 +1,185 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+_QDP_PYTHON = Path(__file__).resolve().parents[2] / "qdp" / "qdp-python"
+if str(_QDP_PYTHON) not in sys.path:
+ sys.path.insert(0, str(_QDP_PYTHON))
+
+
+def _install_local_benchmark_package() -> None:
+ utils_path = _QDP_PYTHON / "benchmark" / "utils.py"
+ utils_spec = importlib.util.spec_from_file_location("benchmark.utils",
utils_path)
+ assert utils_spec is not None
+ assert utils_spec.loader is not None
+ utils_module = importlib.util.module_from_spec(utils_spec)
+ utils_spec.loader.exec_module(utils_module)
+
+ benchmark_package = types.ModuleType("benchmark")
+ benchmark_package.__path__ = [str(_QDP_PYTHON / "benchmark")]
+ setattr(benchmark_package, "utils", utils_module)
+
+ sys.modules["benchmark"] = benchmark_package
+ sys.modules["benchmark.utils"] = utils_module
+
+
+def _load_module(name: str, relative_path: str):
+ _install_local_benchmark_package()
+ module_path = _QDP_PYTHON / relative_path
+ spec = importlib.util.spec_from_file_location(name, module_path)
+ assert spec is not None
+ assert spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+benchmark_latency = _load_module(
+ "qdp_benchmark_latency", "benchmark/benchmark_latency.py"
+)
+benchmark_throughput = _load_module(
+ "qdp_benchmark_throughput", "benchmark/benchmark_throughput.py"
+)
+
+
[email protected](
+ ("module", "frameworks", "encoding_method"),
+ [
+ (benchmark_latency, ["mahout", "pennylane"], "angle"),
+ (benchmark_latency, ["mahout", "qiskit-init"], "basis"),
+ (benchmark_latency, ["mahout", "qiskit-statevector"], "iqp"),
+ (benchmark_throughput, ["mahout", "qiskit"], "angle"),
+ (benchmark_throughput, ["mahout", "mahout-amd"], "basis"),
+ (benchmark_throughput, ["mahout", "pytorch-ref"], "iqp-z"),
+ ],
+)
+def test_non_amplitude_cross_framework_combinations_are_rejected(
+ module, frameworks, encoding_method
+):
+ with pytest.raises(ValueError, match="currently support non-amplitude
encodings"):
+ module.validate_framework_selection(frameworks, encoding_method)
+
+
[email protected]("module", [benchmark_latency, benchmark_throughput])
[email protected]("encoding_method", ["angle", "basis", "iqp", "iqp-z"])
+def test_mahout_only_non_amplitude_runs_remain_allowed(module,
encoding_method):
+ assert module.validate_framework_selection(["mahout"], encoding_method) ==
[
+ "mahout"
+ ]
+
+
[email protected]("module", [benchmark_latency, benchmark_throughput])
+def test_amplitude_cross_framework_comparisons_remain_allowed(module):
+ frameworks = ["mahout", "pennylane"]
+
+ assert module.validate_framework_selection(frameworks, "amplitude") ==
frameworks
+
+
[email protected](
+ ("module", "framework"),
+ [
+ (benchmark_latency, "pennylane"),
+ (benchmark_latency, "qiskit-init"),
+ (benchmark_throughput, "qiskit"),
+ (benchmark_throughput, "mahout-amd"),
+ (benchmark_throughput, "pytorch-ref"),
+ ],
+)
+def test_non_amplitude_single_framework_runs_are_mahout_only(module,
framework):
+ with pytest.raises(ValueError, match="currently support non-amplitude
encodings"):
+ module.validate_framework_selection([framework], "basis")
+
+
+def test_latency_main_rejects_invalid_non_amplitude_cli_combo(monkeypatch,
capsys):
+ monkeypatch.setattr(benchmark_latency.torch.cuda, "is_available", lambda:
True)
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "benchmark_latency.py",
+ "--frameworks",
+ "mahout,pennylane",
+ "--encoding-method",
+ "angle",
+ ],
+ )
+
+ with pytest.raises(SystemExit) as exc_info:
+ benchmark_latency.main()
+
+ assert exc_info.value.code == 2
+ assert "currently support non-amplitude encodings" in
capsys.readouterr().err
+
+
+def test_throughput_main_rejects_invalid_non_amplitude_cli_combo(monkeypatch,
capsys):
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "benchmark_throughput.py",
+ "--frameworks",
+ "mahout,qiskit",
+ "--encoding-method",
+ "basis",
+ ],
+ )
+
+ with pytest.raises(SystemExit) as exc_info:
+ benchmark_throughput.main()
+
+ assert exc_info.value.code == 2
+ assert "currently support non-amplitude encodings" in
capsys.readouterr().err
+
+
+def test_latency_main_rejects_default_all_frameworks_for_non_amplitude(
+ monkeypatch, capsys
+):
+ monkeypatch.setattr(benchmark_latency.torch.cuda, "is_available", lambda:
True)
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ ["benchmark_latency.py", "--encoding-method", "iqp-z"],
+ )
+
+ with pytest.raises(SystemExit) as exc_info:
+ benchmark_latency.main()
+
+ assert exc_info.value.code == 2
+ assert "currently support non-amplitude encodings" in
capsys.readouterr().err
+
+
+def test_throughput_main_rejects_default_all_frameworks_for_non_amplitude(
+ monkeypatch, capsys
+):
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ ["benchmark_throughput.py", "--encoding-method", "basis"],
+ )
+
+ with pytest.raises(SystemExit) as exc_info:
+ benchmark_throughput.main()
+
+ assert exc_info.value.code == 2
+ assert "currently support non-amplitude encodings" in
capsys.readouterr().err
diff --git a/testing/qdp_python/test_fallback.py
b/testing/qdp_python/test_fallback.py
index 381fb6d95..9b34ff12b 100644
--- a/testing/qdp_python/test_fallback.py
+++ b/testing/qdp_python/test_fallback.py
@@ -93,7 +93,10 @@ class TestBackendDetection:
class TestLoaderPytorchBackend:
def test_loader_helpers_cover_iqp_family_edges(self):
- from qumat_qdp.loader import _build_sample, _sample_dim
+ from qumat_qdp import loader as loader_mod
+
+ _build_sample = getattr(loader_mod, "_build_sample")
+ _sample_dim = getattr(loader_mod, "_sample_dim")
assert _sample_dim(3, "basis") == 1
assert _sample_dim(3, "angle") == 3
