Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Holger Freyther has submitted this change and it was merged. (
https://gerrit.osmocom.org/13130 )
Change subject: nitb_netreg_mass: Add code to declare a run successful/failure
..
nitb_netreg_mass: Add code to declare a run successful/failure
Change-Id: If26047f0635db8c7209af143fbd1da4c9b2312d1
---
M src/osmo_gsm_tester/ms_driver.py
M src/osmo_ms_driver/location_update_test.py
M suites/nitb_netreg_mass/register_default_mass.py
3 files changed, 71 insertions(+), 7 deletions(-)
Approvals:
Pau Espin Pedrol: Looks good to me, approved
Jenkins Builder: Verified
diff --git a/src/osmo_gsm_tester/ms_driver.py b/src/osmo_gsm_tester/ms_driver.py
index 8ee93b7..96b907a 100644
--- a/src/osmo_gsm_tester/ms_driver.py
+++ b/src/osmo_gsm_tester/ms_driver.py
@@ -118,6 +118,18 @@
"""
self._test_case.print_stats()
+def get_stats(self):
+"""
+Returns a statistical summary of the test.
+"""
+return self._test_case.get_stats()
+
+def get_result_values(self):
+"""
+Returns the raw result values of the test run in any order.
+"""
+return self._test_case.get_result_values()
+
def cleanup(self):
"""
Cleans up the driver (e.g. AF_UNIX files).
diff --git a/src/osmo_ms_driver/location_update_test.py
b/src/osmo_ms_driver/location_update_test.py
index 2e3a648..5ff2199 100644
--- a/src/osmo_ms_driver/location_update_test.py
+++ b/src/osmo_ms_driver/location_update_test.py
@@ -23,9 +23,11 @@
from datetime import timedelta
+import collections
import time
class LUResult(Results):
+"""Representation of a Location Updating Result."""
def __init__(self, name):
super().__init__(name)
@@ -44,6 +46,10 @@
def lu_delay(self):
return self.lu_time() - self.start_time()
+
+LUStats = collections.namedtuple("LUStats", ["num_attempted", "num_completed",
+ "min_latency", "max_latency"])
+
class MassUpdateLocationTest(log.Origin):
"""
A test to launch a configurable amount of MS and make them
@@ -219,9 +225,24 @@
max_value = result.lu_delay()
return min_value, max_value
+def get_result_values(self):
+"""
+Returns the raw result values of the test run in any order.
+"""
+return self._results.values()
+
+def get_stats(self):
+"""
+Returns a statistical summary of the test.
+"""
+attempted = len(self._subscribers)
+completed = attempted - self._outstanding
+min_latency, max_latency = self.find_min_max(filter(lambda x:
x.has_lu_time(), self._results.values()))
+return LUStats(attempted, completed, min_latency, max_latency)
+
def print_stats(self):
-all_completed = self.all_completed()
-min_value, max_value = self.find_min_max(filter(lambda x:
x.has_lu_time(), self._results.values()))
+stats = self.get_stats()
+all_completed = stats.num_attempted == stats.num_completed
self.log("Tests done", all_completed=all_completed,
-min=min_value, max=max_value)
+min=stats.min_latency, max=stats.max_latency)
diff --git a/suites/nitb_netreg_mass/register_default_mass.py
b/suites/nitb_netreg_mass/register_default_mass.py
index e7c91da..76c53f1 100644
--- a/suites/nitb_netreg_mass/register_default_mass.py
+++ b/suites/nitb_netreg_mass/register_default_mass.py
@@ -1,17 +1,18 @@
#!/usr/bin/env python3
"""
-Run a network registration with a 'massive' amount of MS
+Runs a network registration with a 'massive' amount of MS
using the ms_driver infrastructure.
"""
from osmo_gsm_tester.testenv import *
+from datetime import timedelta
-print('use resources...')
+print('Claiming resources for the test')
nitb = suite.nitb()
bts = suite.bts()
ms_driver = suite.ms_driver()
modems = suite.all_resources(suite.modem)
-print('start nitb and bts...')
+print('Launching a simple network')
nitb.bts_add(bts)
nitb.start()
bts.start()
@@ -25,5 +26,35 @@
# Run the base test.
ms_driver.run_test()
-# Print stats
+# Print the stats of the run.
ms_driver.print_stats()
+
+# Evaluate if this run was successful or not. Our initial acceptance criteria
+# is quite basic but it should allow us to scale to a larger number of MS and
+# reasons (e.g. have a full BCCH).
+#
+# 99% of LUs should complete
+# 99% of successful LUs should complete within 10s.
+stats = ms_driver.get_stats()
+if len(modems) > 0 and stats.num_completed < 1:
+raise Exception("No run completed.")
+completion_ratio = stats.num_attempted / stats.num_completed
+
+# Verify that 99% of LUs completed.
+if completion_ratio < 0.99:
+raise Exception("Completion ratio of %f%% lower than threshold." %
(completion_ratio * 100.0))
+
+# Check how many results are below our threshold.
+acceptable_delay = timedelta(seconds=30)
+results =
Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Pau Espin Pedrol has posted comments on this change. ( https://gerrit.osmocom.org/13130 ) Change subject: nitb_netreg_mass: Add code to declare a run successful/failure .. Patch Set 3: Code-Review+2 -- To view, visit https://gerrit.osmocom.org/13130 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: If26047f0635db8c7209af143fbd1da4c9b2312d1 Gerrit-Change-Number: 13130 Gerrit-PatchSet: 3 Gerrit-Owner: Holger Freyther Gerrit-Reviewer: Jenkins Builder (102) Gerrit-Reviewer: Pau Espin Pedrol Gerrit-Comment-Date: Thu, 07 Mar 2019 14:57:30 + Gerrit-HasComments: No Gerrit-HasLabels: Yes
Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Hello Pau Espin Pedrol, Jenkins Builder, I'd like you to reexamine a change. Please visit https://gerrit.osmocom.org/13130 to look at the new patch set (#3). Change subject: nitb_netreg_mass: Add code to declare a run successful/failure .. nitb_netreg_mass: Add code to declare a run successful/failure Change-Id: If26047f0635db8c7209af143fbd1da4c9b2312d1 --- M src/osmo_gsm_tester/ms_driver.py M src/osmo_ms_driver/location_update_test.py M suites/nitb_netreg_mass/register_default_mass.py 3 files changed, 71 insertions(+), 7 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-gsm-tester refs/changes/30/13130/3 -- To view, visit https://gerrit.osmocom.org/13130 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-MessageType: newpatchset Gerrit-Change-Id: If26047f0635db8c7209af143fbd1da4c9b2312d1 Gerrit-Change-Number: 13130 Gerrit-PatchSet: 3 Gerrit-Owner: Holger Freyther Gerrit-Reviewer: Jenkins Builder (102) Gerrit-Reviewer: Pau Espin Pedrol
Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Pau Espin Pedrol has posted comments on this change. ( https://gerrit.osmocom.org/13130 ) Change subject: nitb_netreg_mass: Add code to declare a run successful/failure .. Patch Set 1: Code-Review+2 -- To view, visit https://gerrit.osmocom.org/13130 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: If26047f0635db8c7209af143fbd1da4c9b2312d1 Gerrit-Change-Number: 13130 Gerrit-PatchSet: 1 Gerrit-Owner: Holger Freyther Gerrit-Reviewer: Jenkins Builder (102) Gerrit-Reviewer: Pau Espin Pedrol Gerrit-Comment-Date: Tue, 05 Mar 2019 15:34:31 + Gerrit-HasComments: No Gerrit-HasLabels: Yes
Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Holger Freyther has uploaded this change for review. (
https://gerrit.osmocom.org/13130
Change subject: nitb_netreg_mass: Add code to declare a run successful/failure
..
nitb_netreg_mass: Add code to declare a run successful/failure
Change-Id: If26047f0635db8c7209af143fbd1da4c9b2312d1
---
M src/osmo_gsm_tester/ms_driver.py
M src/osmo_ms_driver/location_update_test.py
M suites/nitb_netreg_mass/register_default_mass.py
3 files changed, 71 insertions(+), 7 deletions(-)
git pull ssh://gerrit.osmocom.org:29418/osmo-gsm-tester
refs/changes/30/13130/1
diff --git a/src/osmo_gsm_tester/ms_driver.py b/src/osmo_gsm_tester/ms_driver.py
index 8ee93b7..96b907a 100644
--- a/src/osmo_gsm_tester/ms_driver.py
+++ b/src/osmo_gsm_tester/ms_driver.py
@@ -118,6 +118,18 @@
"""
self._test_case.print_stats()
+def get_stats(self):
+"""
+Returns a statistical summary of the test.
+"""
+return self._test_case.get_stats()
+
+def get_result_values(self):
+"""
+Returns the raw result values of the test run in any order.
+"""
+return self._test_case.get_result_values()
+
def cleanup(self):
"""
Cleans up the driver (e.g. AF_UNIX files).
diff --git a/src/osmo_ms_driver/location_update_test.py
b/src/osmo_ms_driver/location_update_test.py
index 2e3a648..5ff2199 100644
--- a/src/osmo_ms_driver/location_update_test.py
+++ b/src/osmo_ms_driver/location_update_test.py
@@ -23,9 +23,11 @@
from datetime import timedelta
+import collections
import time
class LUResult(Results):
+"""Representation of a Location Updating Result."""
def __init__(self, name):
super().__init__(name)
@@ -44,6 +46,10 @@
def lu_delay(self):
return self.lu_time() - self.start_time()
+
+LUStats = collections.namedtuple("LUStats", ["num_attempted", "num_completed",
+ "min_latency", "max_latency"])
+
class MassUpdateLocationTest(log.Origin):
"""
A test to launch a configurable amount of MS and make them
@@ -219,9 +225,24 @@
max_value = result.lu_delay()
return min_value, max_value
+def get_result_values(self):
+"""
+Returns the raw result values of the test run in any order.
+"""
+return self._results.values()
+
+def get_stats(self):
+"""
+Returns a statistical summary of the test.
+"""
+attempted = len(self._subscribers)
+completed = attempted - self._outstanding
+min_latency, max_latency = self.find_min_max(filter(lambda x:
x.has_lu_time(), self._results.values()))
+return LUStats(attempted, completed, min_latency, max_latency)
+
def print_stats(self):
-all_completed = self.all_completed()
-min_value, max_value = self.find_min_max(filter(lambda x:
x.has_lu_time(), self._results.values()))
+stats = self.get_stats()
+all_completed = stats.num_attempted == stats.num_completed
self.log("Tests done", all_completed=all_completed,
-min=min_value, max=max_value)
+min=stats.min_latency, max=stats.max_latency)
diff --git a/suites/nitb_netreg_mass/register_default_mass.py
b/suites/nitb_netreg_mass/register_default_mass.py
index 1033608..028e7e5 100644
--- a/suites/nitb_netreg_mass/register_default_mass.py
+++ b/suites/nitb_netreg_mass/register_default_mass.py
@@ -1,17 +1,18 @@
#!/usr/bin/env python3
"""
-Run a network registration with a 'massive' amount of MS
+Runs a network registration with a 'massive' amount of MS
using the ms_driver infrastructure.
"""
from osmo_gsm_tester.testenv import *
+from datetime import timedelta
-print('use resources...')
+print('Claiming resources for the test')
nitb = suite.nitb()
bts = suite.bts()
ms_driver = suite.ms_driver()
modems = suite.all_modems()
-print('start nitb and bts...')
+print('Launching a simple network')
nitb.bts_add(bts)
nitb.start()
bts.start()
@@ -25,5 +26,35 @@
# Run the base test.
ms_driver.run_test()
-# Print stats
+# Print the stats of the run.
ms_driver.print_stats()
+
+# Evaluate if this run was successful or not. Our initial acceptance criteria
+# is quite basic but it should allow us to scale to a larger number of MS and
+# reasons (e.g. have a full BCCH).
+#
+# 99% of LUs should complete
+# 99% of successful LUs should complete within 10s.
+stats = ms_driver.get_stats()
+if len(modems) > 0 and stats.num_completed < 1:
+raise Exception("No run completed.")
+completion_ratio = stats.num_attempted / stats.num_completed
+
+# Verify that 99% of LUs completed.
+if completion_ratio < 0.99:
+raise Exception("Completion ratio of %f%% lower than threshold." %
(completion_ratio * 100.0))
+
+# Check how many results are below our threshold.
+acceptable_delay = timedelta(seconds=20)
+results = ms_driver.get_result_values()
Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Holger Freyther has created a revert of this change. ( https://gerrit.osmocom.org/13029 ) Change subject: nitb_netreg_mass: Add code to declare a run successful/failure .. -- To view, visit https://gerrit.osmocom.org/13029 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-MessageType: revert Gerrit-Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86 Gerrit-Change-Number: 13029 Gerrit-PatchSet: 5 Gerrit-Owner: Holger Freyther Gerrit-Reviewer: Holger Freyther Gerrit-Reviewer: Jenkins Builder (102) Gerrit-Reviewer: Pau Espin Pedrol
Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Holger Freyther has submitted this change and it was merged. (
https://gerrit.osmocom.org/13029 )
Change subject: nitb_netreg_mass: Add code to declare a run successful/failure
..
nitb_netreg_mass: Add code to declare a run successful/failure
Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86
---
M src/osmo_gsm_tester/ms_driver.py
M src/osmo_ms_driver/location_update_test.py
M suites/nitb_netreg_mass/register_default_mass.py
3 files changed, 71 insertions(+), 7 deletions(-)
Approvals:
Jenkins Builder: Verified
Pau Espin Pedrol: Looks good to me, approved
diff --git a/src/osmo_gsm_tester/ms_driver.py b/src/osmo_gsm_tester/ms_driver.py
index e8e543c..de2f967 100644
--- a/src/osmo_gsm_tester/ms_driver.py
+++ b/src/osmo_gsm_tester/ms_driver.py
@@ -146,6 +146,18 @@
"""
self._test_case.print_stats()
+def get_stats(self):
+"""
+Returns a statistical summary of the test.
+"""
+return self._test_case.get_stats()
+
+def get_result_values(self):
+"""
+Returns the raw result values of the test run in any order.
+"""
+return self._test_case.get_result_values()
+
def cleanup(self):
"""
Cleans up the driver (e.g. AF_UNIX files).
diff --git a/src/osmo_ms_driver/location_update_test.py
b/src/osmo_ms_driver/location_update_test.py
index 8500607..f14bd28 100644
--- a/src/osmo_ms_driver/location_update_test.py
+++ b/src/osmo_ms_driver/location_update_test.py
@@ -23,9 +23,11 @@
from datetime import timedelta
+import collections
import time
class LUResult(Results):
+"""Representation of a Location Updating Result."""
def __init__(self, name):
super().__init__(name)
@@ -44,6 +46,10 @@
def lu_delay(self):
return self.lu_time() - self.start_time()
+
+LUStats = collections.namedtuple("LUStats", ["num_attempted", "num_completed",
+ "min_latency", "max_latency"])
+
class MassUpdateLocationTest(log.Origin):
"""
A test to launch a configurable amount of MS and make them
@@ -207,9 +213,24 @@
max_value = result.lu_delay()
return min_value, max_value
+def get_result_values(self):
+"""
+Returns the raw result values of the test run in any order.
+"""
+return self._results.values()
+
+def get_stats(self):
+"""
+Returns a statistical summary of the test.
+"""
+attempted = self._number_of_ms
+completed = attempted - self._outstanding
+min_latency, max_latency = self.find_min_max(filter(lambda x:
x.has_lu_time(), self._results.values()))
+return LUStats(attempted, completed, min_latency, max_latency)
+
def print_stats(self):
-all_completed = self.all_completed()
-min_value, max_value = self.find_min_max(filter(lambda x:
x.has_lu_time(), self._results.values()))
+stats = self.get_stats()
+all_completed = stats.num_attempted == stats.num_completed
self.log("Tests done", all_completed=all_completed,
-min=min_value, max=max_value)
+min=stats.min_latency, max=stats.max_latency)
diff --git a/suites/nitb_netreg_mass/register_default_mass.py
b/suites/nitb_netreg_mass/register_default_mass.py
index ede2b74..8259359 100644
--- a/suites/nitb_netreg_mass/register_default_mass.py
+++ b/suites/nitb_netreg_mass/register_default_mass.py
@@ -1,16 +1,17 @@
#!/usr/bin/env python3
"""
-Run a network registration with a 'massive' amount of MS
+Runs a network registration with a 'massive' amount of MS
using the ms_driver infrastructure.
"""
from osmo_gsm_tester.testenv import *
+from datetime import timedelta
-print('use resources...')
+print('Claiming resources for the test')
nitb = suite.nitb()
bts = suite.bts()
ms_driver = suite.ms_driver()
-print('start nitb and bts...')
+print('Launching a simple network')
nitb.bts_add(bts)
nitb.start()
bts.start()
@@ -23,5 +24,35 @@
# Run the base test.
ms_driver.run_test()
-# Print stats
+# Print the stats of the run.
ms_driver.print_stats()
+
+# Evaluate if this run was successful or not. Our initial acceptance criteria
+# is quite basic but it should allow us to scale to a larger number of MS and
+# reasons (e.g. have a full BCCH).
+#
+# 99% of LUs should complete
+# 99% of successful LUs should complete within 10s.
+stats = ms_driver.get_stats()
+if len(mobiles) > 0 and stats.num_completed < 1:
+raise Exception("No run completed.")
+completion_ratio = stats.num_attempted / stats.num_completed
+
+# Verify that 99% of LUs completed.
+if completion_ratio < 0.99:
+raise Exception("Completion ratio of %f%% lower than threshold." %
(completion_ratio * 100.0))
+
+# Check how many results are below our threshold.
+acceptable_delay = timedelta(seconds=20)
+results = ms_driver.get_result_values()
+quick_enough = 0
+for
Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Holger Freyther has posted comments on this change. ( https://gerrit.osmocom.org/13029 ) Change subject: nitb_netreg_mass: Add code to declare a run successful/failure .. Patch Set 5: This change is ready for review. -- To view, visit https://gerrit.osmocom.org/13029 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86 Gerrit-Change-Number: 13029 Gerrit-PatchSet: 5 Gerrit-Owner: Holger Freyther Gerrit-Reviewer: Holger Freyther Gerrit-Reviewer: Jenkins Builder (102) Gerrit-Reviewer: Pau Espin Pedrol Gerrit-Comment-Date: Tue, 05 Mar 2019 03:08:38 + Gerrit-HasComments: No Gerrit-HasLabels: No
Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Hello Pau Espin Pedrol, Jenkins Builder, I'd like you to reexamine a change. Please visit https://gerrit.osmocom.org/13029 to look at the new patch set (#4). Change subject: nitb_netreg_mass: Add code to declare a run successful/failure .. nitb_netreg_mass: Add code to declare a run successful/failure Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86 --- M src/osmo_gsm_tester/ms_driver.py M src/osmo_ms_driver/location_update_test.py M suites/nitb_netreg_mass/register_default_mass.py 3 files changed, 71 insertions(+), 7 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-gsm-tester refs/changes/29/13029/4 -- To view, visit https://gerrit.osmocom.org/13029 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-MessageType: newpatchset Gerrit-Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86 Gerrit-Change-Number: 13029 Gerrit-PatchSet: 4 Gerrit-Owner: Holger Freyther Gerrit-Reviewer: Holger Freyther Gerrit-Reviewer: Jenkins Builder (102) Gerrit-Reviewer: Pau Espin Pedrol
Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Hello Pau Espin Pedrol, Jenkins Builder, I'd like you to reexamine a change. Please visit https://gerrit.osmocom.org/13029 to look at the new patch set (#3). Change subject: nitb_netreg_mass: Add code to declare a run successful/failure .. nitb_netreg_mass: Add code to declare a run successful/failure Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86 --- M src/osmo_gsm_tester/ms_driver.py M src/osmo_ms_driver/location_update_test.py M suites/nitb_netreg_mass/register_default_mass.py 3 files changed, 71 insertions(+), 7 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-gsm-tester refs/changes/29/13029/3 -- To view, visit https://gerrit.osmocom.org/13029 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-MessageType: newpatchset Gerrit-Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86 Gerrit-Change-Number: 13029 Gerrit-PatchSet: 3 Gerrit-Owner: Holger Freyther Gerrit-Reviewer: Holger Freyther Gerrit-Reviewer: Jenkins Builder (102) Gerrit-Reviewer: Pau Espin Pedrol
Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Holger Freyther has posted comments on this change. ( https://gerrit.osmocom.org/13029 ) Change subject: nitb_netreg_mass: Add code to declare a run successful/failure .. Patch Set 2: (1 comment) https://gerrit.osmocom.org/#/c/13029/1/suites/nitb_netreg_mass/register_default_mass.py File suites/nitb_netreg_mass/register_default_mass.py: https://gerrit.osmocom.org/#/c/13029/1/suites/nitb_netreg_mass/register_default_mass.py@48 PS1, Line 48: if not result.has_lu_time(): > do we really need "count"? It's just len(ms_driver. […] Sure. Good idea. I will store the result in a temp to avoid having to assume that two calls return the same data. -- To view, visit https://gerrit.osmocom.org/13029 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86 Gerrit-Change-Number: 13029 Gerrit-PatchSet: 2 Gerrit-Owner: Holger Freyther Gerrit-Reviewer: Holger Freyther Gerrit-Reviewer: Jenkins Builder (102) Gerrit-Reviewer: Pau Espin Pedrol Gerrit-Comment-Date: Mon, 25 Feb 2019 03:28:08 + Gerrit-HasComments: Yes Gerrit-HasLabels: No
Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Hello Pau Espin Pedrol, Jenkins Builder, I'd like you to reexamine a change. Please visit https://gerrit.osmocom.org/13029 to look at the new patch set (#2). Change subject: nitb_netreg_mass: Add code to declare a run successful/failure .. nitb_netreg_mass: Add code to declare a run successful/failure Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86 --- M src/osmo_gsm_tester/ms_driver.py M src/osmo_ms_driver/location_update_test.py M suites/nitb_netreg_mass/register_default_mass.py 3 files changed, 69 insertions(+), 7 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-gsm-tester refs/changes/29/13029/2 -- To view, visit https://gerrit.osmocom.org/13029 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-MessageType: newpatchset Gerrit-Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86 Gerrit-Change-Number: 13029 Gerrit-PatchSet: 2 Gerrit-Owner: Holger Freyther Gerrit-Reviewer: Jenkins Builder (102) Gerrit-Reviewer: Pau Espin Pedrol
Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Pau Espin Pedrol has posted comments on this change. ( https://gerrit.osmocom.org/13029 ) Change subject: nitb_netreg_mass: Add code to declare a run successful/failure .. Patch Set 1: Code-Review+1 (1 comment) https://gerrit.osmocom.org/#/c/13029/1/suites/nitb_netreg_mass/register_default_mass.py File suites/nitb_netreg_mass/register_default_mass.py: https://gerrit.osmocom.org/#/c/13029/1/suites/nitb_netreg_mass/register_default_mass.py@48 PS1, Line 48: count = count + 1 do we really need "count"? It's just len(ms_driver.get_result_values()) -- To view, visit https://gerrit.osmocom.org/13029 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86 Gerrit-Change-Number: 13029 Gerrit-PatchSet: 1 Gerrit-Owner: Holger Freyther Gerrit-Reviewer: Pau Espin Pedrol Gerrit-Comment-Date: Sat, 23 Feb 2019 22:44:24 + Gerrit-HasComments: Yes Gerrit-HasLabels: Yes
Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure
Holger Freyther has uploaded this change for review. (
https://gerrit.osmocom.org/13029
Change subject: nitb_netreg_mass: Add code to declare a run successful/failure
..
nitb_netreg_mass: Add code to declare a run successful/failure
Change-Id: I49a821e8e4656466259e519ef901726cbb76ef86
---
M src/osmo_gsm_tester/ms_driver.py
M src/osmo_ms_driver/location_update_test.py
M suites/nitb_netreg_mass/register_default_mass.py
3 files changed, 70 insertions(+), 7 deletions(-)
git pull ssh://gerrit.osmocom.org:29418/osmo-gsm-tester
refs/changes/29/13029/1
diff --git a/src/osmo_gsm_tester/ms_driver.py b/src/osmo_gsm_tester/ms_driver.py
index e8e543c..de2f967 100644
--- a/src/osmo_gsm_tester/ms_driver.py
+++ b/src/osmo_gsm_tester/ms_driver.py
@@ -146,6 +146,18 @@
"""
self._test_case.print_stats()
+def get_stats(self):
+"""
+Returns a statistical summary of the test.
+"""
+return self._test_case.get_stats()
+
+def get_result_values(self):
+"""
+Returns the raw result values of the test run in any order.
+"""
+return self._test_case.get_result_values()
+
def cleanup(self):
"""
Cleans up the driver (e.g. AF_UNIX files).
diff --git a/src/osmo_ms_driver/location_update_test.py
b/src/osmo_ms_driver/location_update_test.py
index 8500607..f14bd28 100644
--- a/src/osmo_ms_driver/location_update_test.py
+++ b/src/osmo_ms_driver/location_update_test.py
@@ -23,9 +23,11 @@
from datetime import timedelta
+import collections
import time
class LUResult(Results):
+"""Representation of a Location Updating Result."""
def __init__(self, name):
super().__init__(name)
@@ -44,6 +46,10 @@
def lu_delay(self):
return self.lu_time() - self.start_time()
+
+LUStats = collections.namedtuple("LUStats", ["num_attempted", "num_completed",
+ "min_latency", "max_latency"])
+
class MassUpdateLocationTest(log.Origin):
"""
A test to launch a configurable amount of MS and make them
@@ -207,9 +213,24 @@
max_value = result.lu_delay()
return min_value, max_value
+def get_result_values(self):
+"""
+Returns the raw result values of the test run in any order.
+"""
+return self._results.values()
+
+def get_stats(self):
+"""
+Returns a statistical summary of the test.
+"""
+attempted = self._number_of_ms
+completed = attempted - self._outstanding
+min_latency, max_latency = self.find_min_max(filter(lambda x:
x.has_lu_time(), self._results.values()))
+return LUStats(attempted, completed, min_latency, max_latency)
+
def print_stats(self):
-all_completed = self.all_completed()
-min_value, max_value = self.find_min_max(filter(lambda x:
x.has_lu_time(), self._results.values()))
+stats = self.get_stats()
+all_completed = stats.num_attempted == stats.num_completed
self.log("Tests done", all_completed=all_completed,
-min=min_value, max=max_value)
+min=stats.min_latency, max=stats.max_latency)
diff --git a/suites/nitb_netreg_mass/register_default_mass.py
b/suites/nitb_netreg_mass/register_default_mass.py
index ede2b74..860a244 100644
--- a/suites/nitb_netreg_mass/register_default_mass.py
+++ b/suites/nitb_netreg_mass/register_default_mass.py
@@ -1,16 +1,17 @@
#!/usr/bin/env python3
"""
-Run a network registration with a 'massive' amount of MS
+Runs a network registration with a 'massive' amount of MS
using the ms_driver infrastructure.
"""
from osmo_gsm_tester.testenv import *
+from datetime import timedelta
-print('use resources...')
+print('Claiming resources for the test')
nitb = suite.nitb()
bts = suite.bts()
ms_driver = suite.ms_driver()
-print('start nitb and bts...')
+print('Launching a simple network')
nitb.bts_add(bts)
nitb.start()
bts.start()
@@ -23,5 +24,34 @@
# Run the base test.
ms_driver.run_test()
-# Print stats
+# Print the stats of the run.
ms_driver.print_stats()
+
+# Evaluate if this run was successful or not. Our initial acceptance criteria
+# is quite basic but it should allow us to scale to a larger number of MS and
+# reasons (e.g. have a full BCCH).
+#
+# 99% of LUs should complete
+# 99% of successful LUs should complete within 10s.
+stats = ms_driver.get_stats()
+completion_ratio = stats.num_attempted / stats.num_completed
+
+# Verify that 99% of LUs completed.
+if completion_ratio < 0.99:
+raise Exception("Completion ratio of %f%% lower than threshold." %
(completion_ratio * 100.0))
+
+# Check how many results are below our threshold.
+acceptable_delay = timedelta(seconds=20)
+count = 0
+quick_enough = 0
+for result in ms_driver.get_result_values():
+count = count + 1
+if not result.has_lu_time():
+continue
+if
