The single core forward performance test suite can run a variety of combinations of buffer sizes and number of descriptors. It would be beneficial if any possible combination of these parameters could be tested against a specific delta tolerance, rather than a globally defined tolerance for all testing. Along with this, adjusting traffic duration and test repetitions helps users define a more accurate test run if they wish to get a more refined average. Furthermore, the cores allotted to TREX can vary among systems and should be defined by the user.
Signed-off-by: Andrew Bailey <[email protected]> --- v2: * Updated commit description. * Made TREX cores optional with a default of 10. * Updated the config examples with new descriptions. dts/configurations/test_run.example.yaml | 3 ++ dts/configurations/tests_config.example.yaml | 14 +++++ dts/framework/config/test_run.py | 1 + .../testbed_model/traffic_generator/trex.py | 3 +- .../TestSuite_single_core_forward_perf.py | 52 ++++++++++++++----- 5 files changed, 60 insertions(+), 13 deletions(-) diff --git a/dts/configurations/test_run.example.yaml b/dts/configurations/test_run.example.yaml index 182ee377aa..cf169e88ea 100644 --- a/dts/configurations/test_run.example.yaml +++ b/dts/configurations/test_run.example.yaml @@ -33,6 +33,8 @@ # `traffic_generator_node`: # This field may be None When `port_topology` is specified as a no link topology, Otherwise it # is required. +# `cores`: +# The number of core to provide TREX traffic generator. If not provided, 10 will be used by default. # Define the test run environment dpdk: @@ -65,6 +67,7 @@ func_traffic_generator: # type: TREX # remote_path: "/opt/trex/v3.03" # The remote path of the traffic generator application. # config: "/opt/trex_config/trex_config.yaml" # Additional configuration files. (Leave blank if not required) +# cores: 10 # The number of cores to run TRex with. perf: false # disable performance testing func: true # enable functional testing crypto: false # disable cryptographic testing diff --git a/dts/configurations/tests_config.example.yaml b/dts/configurations/tests_config.example.yaml index ce67385b0b..a59fb9ca33 100644 --- a/dts/configurations/tests_config.example.yaml +++ b/dts/configurations/tests_config.example.yaml @@ -2,6 +2,17 @@ # Optional Fields: # All fields are optional, uncomment a test suite to run it with a specific configuration. +# `single_core_forward_perf`: +# `delta_tolerance`: +# Can be provided under `single_core_forward_perf` to apply to each configuration. +# Additionally, it can be provided within one list item to apply a unique delta tolerance +# to only that `frame_size` + `num_descriptors` combination. +# `test_duration`: +# Can be provided to define a duration in seconds to send traffic to the TG per config. If +# this field is not provided, 5 is used by default. +# `test_repetitions`: +# Can be provided to define how many times to repeat the performance test to calculate a more +# precise average. If this field is not provided, 5 is used by default. # Define the custom test suite configurations hello_world: @@ -11,12 +22,15 @@ hello_world: # - frame_size: 64 # num_descriptors: 512 # expected_mpps: 1.0 # Set millions of packets per second according to your devices expected throughput for this given frame size / descriptor count +# delta_tolerance: 0.06 # See `Optional Fields` # - frame_size: 64 # num_descriptors: 1024 # expected_mpps: 1.0 # - frame_size: 512 # num_descriptors: 1024 # expected_mpps: 1.0 +# test_duration: 5 # See `Optional Fields` +# test_repetitions: 5 # See `Optional Fields` # delta_tolerance: 0.05 # pmd_rss: # NUM_QUEUES: 4 diff --git a/dts/framework/config/test_run.py b/dts/framework/config/test_run.py index 97c9a7417a..b8eb5c888b 100644 --- a/dts/framework/config/test_run.py +++ b/dts/framework/config/test_run.py @@ -422,6 +422,7 @@ class TrexTrafficGeneratorConfig(TrafficGeneratorConfig): type: Literal[TrafficGeneratorType.TREX] remote_path: PurePath config: PurePath + cores: int = Field(default=10) #: A union type discriminating traffic generators by the `type` field. diff --git a/dts/framework/testbed_model/traffic_generator/trex.py b/dts/framework/testbed_model/traffic_generator/trex.py index 7ce1d40d79..b698b91c30 100644 --- a/dts/framework/testbed_model/traffic_generator/trex.py +++ b/dts/framework/testbed_model/traffic_generator/trex.py @@ -81,7 +81,7 @@ class TrexTrafficGenerator(PerformanceTrafficGenerator): _streaming_mode: TrexStatelessTXModes = TrexStatelessTXModes.STLTXCont - _tg_cores: int = 10 + _tg_cores: int _trex_app: BlockingApp @@ -104,6 +104,7 @@ def __init__(self, tg_node: Node, config: TrexTrafficGeneratorConfig) -> None: super().__init__(tg_node=tg_node, config=config) self._tg_node_config = tg_node.config self._tg_config = config + self._tg_cores = config.cores self._os_session = create_session(self._tg_node.config, "TRex", self._logger) diff --git a/dts/tests/TestSuite_single_core_forward_perf.py b/dts/tests/TestSuite_single_core_forward_perf.py index 4b22c5d091..ee54eeef59 100644 --- a/dts/tests/TestSuite_single_core_forward_perf.py +++ b/dts/tests/TestSuite_single_core_forward_perf.py @@ -32,14 +32,37 @@ class Config(BaseConfig): """Performance test metrics.""" test_parameters: list[dict[str, int | float]] = [ - {"frame_size": 64, "num_descriptors": 1024, "expected_mpps": 1.00}, - {"frame_size": 128, "num_descriptors": 1024, "expected_mpps": 1.00}, - {"frame_size": 256, "num_descriptors": 1024, "expected_mpps": 1.00}, - {"frame_size": 512, "num_descriptors": 1024, "expected_mpps": 1.00}, - {"frame_size": 1024, "num_descriptors": 1024, "expected_mpps": 1.00}, - {"frame_size": 1518, "num_descriptors": 1024, "expected_mpps": 1.00}, + {"frame_size": 64, "num_descriptors": 1024, "expected_mpps": 1.00, "delta_tolerance": 0.05}, + { + "frame_size": 128, + "num_descriptors": 1024, + "expected_mpps": 1.00, + }, + { + "frame_size": 256, + "num_descriptors": 1024, + "expected_mpps": 1.00, + }, + { + "frame_size": 512, + "num_descriptors": 1024, + "expected_mpps": 1.00, + }, + { + "frame_size": 1024, + "num_descriptors": 1024, + "expected_mpps": 1.00, + "delta_tolerance": 0.05, + }, + { + "frame_size": 1518, + "num_descriptors": 1024, + "expected_mpps": 1.00, + }, ] - delta_tolerance: float = 0.05 + traffic_duration: int = 5 + test_repetitions: int = 5 + delta_tolerance: float | None = 0.05 @requires_link_topology(LinkTopology.TWO_LINKS) @@ -51,6 +74,8 @@ class TestSingleCoreForwardPerf(TestSuite): def set_up_suite(self): """Set up the test suite.""" self.test_parameters = self.config.test_parameters + self.traffic_duration = self.config.traffic_duration + self.test_repetitions = self.config.test_repetitions self.delta_tolerance = self.config.delta_tolerance def _transmit(self, testpmd: TestPmd, frame_size: int, repetitions: int = 1) -> float: @@ -76,8 +101,7 @@ def _transmit(self, testpmd: TestPmd, frame_size: int, repetitions: int = 1) -> rx_avg = 0.0 for _ in range(repetitions): - # Transmit for 5 seconds. - stats = assess_performance_by_packet(packet=packet, duration=5) + stats = assess_performance_by_packet(packet=packet, duration=self.traffic_duration) rx_avg += stats.rx_pps return rx_avg / (repetitions * 1_000_000) @@ -134,21 +158,25 @@ def single_core_forward_perf(self) -> None: **driver_specific_testpmd_args, ) as testpmd: params["measured_mpps"] = round( - self._transmit(testpmd, frame_size, repetitions=5), 3 + self._transmit(testpmd, frame_size, repetitions=self.test_repetitions), 3 ) params["performance_delta"] = round( (float(params["measured_mpps"]) - float(params["expected_mpps"])) / float(params["expected_mpps"]), 3, ) - params["pass"] = float(params["performance_delta"]) >= -self.delta_tolerance + params["pass"] = float(params["performance_delta"]) >= -params.get( + "delta_tolerance", self.delta_tolerance + ) self._produce_stats_table(self.test_parameters) for params in self.test_parameters: verify( params["pass"] is True, - f"""Packets forwarded is less than {(1 - self.delta_tolerance) * 100}% + f"""Packets forwarded is less than { + (1 - params.get("delta_tolerance", self.delta_tolerance)) * 100 + }% of the expected baseline. Measured MPPS = {params["measured_mpps"]} Expected MPPS = {params["expected_mpps"]}""", -- 2.56.0
