Holger Freyther has uploaded this change for review. ( https://gerrit.osmocom.org/13029
Change subject: nitb_netreg_mass: Add code to declare a run successful/failure ...................................................................... nitb_netreg_mass: Add code to declare a run successful/failure Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86 --- M src/osmo_gsm_tester/ms_driver.py M src/osmo_ms_driver/location_update_test.py M suites/nitb_netreg_mass/register_default_mass.py 3 files changed, 70 insertions(+), 7 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-gsm-tester refs/changes/29/13029/1 diff --git a/src/osmo_gsm_tester/ms_driver.py b/src/osmo_gsm_tester/ms_driver.py index e8e543c..de2f967 100644 --- a/src/osmo_gsm_tester/ms_driver.py +++ b/src/osmo_gsm_tester/ms_driver.py @@ -146,6 +146,18 @@ """ self._test_case.print_stats() + def get_stats(self): + """ + Returns a statistical summary of the test. + """ + return self._test_case.get_stats() + + def get_result_values(self): + """ + Returns the raw result values of the test run in any order. + """ + return self._test_case.get_result_values() + def cleanup(self): """ Cleans up the driver (e.g. AF_UNIX files). diff --git a/src/osmo_ms_driver/location_update_test.py b/src/osmo_ms_driver/location_update_test.py index 8500607..f14bd28 100644 --- a/src/osmo_ms_driver/location_update_test.py +++ b/src/osmo_ms_driver/location_update_test.py @@ -23,9 +23,11 @@ from datetime import timedelta +import collections import time class LUResult(Results): + """Representation of a Location Updating Result.""" def __init__(self, name): super().__init__(name) @@ -44,6 +46,10 @@ def lu_delay(self): return self.lu_time() - self.start_time() + +LUStats = collections.namedtuple("LUStats", ["num_attempted", "num_completed", + "min_latency", "max_latency"]) + class MassUpdateLocationTest(log.Origin): """ A test to launch a configurable amount of MS and make them @@ -207,9 +213,24 @@ max_value = result.lu_delay() return min_value, max_value + def get_result_values(self): + """ + Returns the raw result values of the test run in any order. + """ + return self._results.values() + + def get_stats(self): + """ + Returns a statistical summary of the test. + """ + attempted = self._number_of_ms + completed = attempted - self._outstanding + min_latency, max_latency = self.find_min_max(filter(lambda x: x.has_lu_time(), self._results.values())) + return LUStats(attempted, completed, min_latency, max_latency) + def print_stats(self): - all_completed = self.all_completed() - min_value, max_value = self.find_min_max(filter(lambda x: x.has_lu_time(), self._results.values())) + stats = self.get_stats() + all_completed = stats.num_attempted == stats.num_completed self.log("Tests done", all_completed=all_completed, - min=min_value, max=max_value) + min=stats.min_latency, max=stats.max_latency) diff --git a/suites/nitb_netreg_mass/register_default_mass.py b/suites/nitb_netreg_mass/register_default_mass.py index ede2b74..860a244 100644 --- a/suites/nitb_netreg_mass/register_default_mass.py +++ b/suites/nitb_netreg_mass/register_default_mass.py @@ -1,16 +1,17 @@ #!/usr/bin/env python3 """ -Run a network registration with a 'massive' amount of MS +Runs a network registration with a 'massive' amount of MS using the ms_driver infrastructure. """ from osmo_gsm_tester.testenv import * +from datetime import timedelta -print('use resources...') +print('Claiming resources for the test') nitb = suite.nitb() bts = suite.bts() ms_driver = suite.ms_driver() -print('start nitb and bts...') +print('Launching a simple network') nitb.bts_add(bts) nitb.start() bts.start() @@ -23,5 +24,34 @@ # Run the base test. ms_driver.run_test() -# Print stats +# Print the stats of the run. ms_driver.print_stats() + +# Evaluate if this run was successful or not. Our initial acceptance criteria +# is quite basic but it should allow us to scale to a larger number of MS and +# reasons (e.g. have a full BCCH). +# +# 99% of LUs should complete +# 99% of successful LUs should complete within 10s. +stats = ms_driver.get_stats() +completion_ratio = stats.num_attempted / stats.num_completed + +# Verify that 99% of LUs completed. +if completion_ratio < 0.99: + raise Exception("Completion ratio of %f%% lower than threshold." % (completion_ratio * 100.0)) + +# Check how many results are below our threshold. +acceptable_delay = timedelta(seconds=20) +count = 0 +quick_enough = 0 +for result in ms_driver.get_result_values(): + count = count + 1 + if not result.has_lu_time(): + continue + if timedelta(seconds=result.lu_delay()) >= acceptable_delay: + continue + quick_enough = quick_enough + 1 + +latency_ratio = quick_enough / count +if latency_ratio < 0.99: + raise Exception("Latency ratio of %f%% lower than threshold." % (latency_ratio * 100.0)) -- To view, visit https://gerrit.osmocom.org/13029 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-MessageType: newchange Gerrit-Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86 Gerrit-Change-Number: 13029 Gerrit-PatchSet: 1 Gerrit-Owner: Holger Freyther <hol...@freyther.de>