This is an automated email from the ASF dual-hosted git repository. dcapwell pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra-dtest.git
The following commit(s) were added to refs/heads/trunk by this push: new f9ff078 CASSANDRA-16729 - user_types_test.py::TestUserTypes::test_type_keyspace_permission_isolation uses retries to avoid flakyness f9ff078 is described below commit f9ff078c3f54119ef8c7aa22987dd33f57d56de2 Author: David Capwell <dcapw...@gmail.com> AuthorDate: Thu Jun 10 16:35:22 2021 -0700 CASSANDRA-16729 - user_types_test.py::TestUserTypes::test_type_keyspace_permission_isolation uses retries to avoid flakyness patch by David Capwell; reviewed by Brandon Williams for CASSANDRA-16729 --- tools/flaky.py | 20 ++++++++++++++++++++ user_types_test.py | 26 +++++++++++++++++--------- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/tools/flaky.py b/tools/flaky.py index 667f8fe..7baedad 100644 --- a/tools/flaky.py +++ b/tools/flaky.py @@ -1,3 +1,8 @@ +import logging +import time + +logger = logging.getLogger(__name__) + class RerunTestException(Exception): """ This exception can be raised to signal a likely harmless test problem. If fixing a test is reasonable, that should be preferred. @@ -30,3 +35,18 @@ def requires_rerun(err, *args): """ # err[0] contains the type of the error that occurred return err[0] == RerunTestException + +def retry(fn, num_retries=10, allowed_error=None, sleep_seconds=1): + last_error = None + for x in range(0, num_retries): + try: + return fn() + except Exception as e: + last_error = e + if allowed_error and not allowed_error(e): + break + logger.info("Retrying as error '{}' was seen; sleeping for {} seconds".format(str(e), str(sleep_seconds))) + time.sleep(sleep_seconds) + raise last_error + + diff --git a/user_types_test.py b/user_types_test.py index dcbb833..2c162e9 100644 --- a/user_types_test.py +++ b/user_types_test.py @@ -5,11 +5,12 @@ import logging from flaky import flaky -from cassandra import ConsistencyLevel, Unauthorized +from cassandra import ConsistencyLevel, Unauthorized, InvalidRequest from cassandra.query import SimpleStatement from dtest import Tester, create_ks from tools.assertions import assert_invalid +from tools.flaky import retry from plugins.assert_tools import assert_regexp_matches since = pytest.mark.since @@ -529,17 +530,24 @@ class TestUserTypes(Tester): cluster.set_configuration_options(values=config) cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() - # need a bit of time for user to be created and propagate - time.sleep(5) # do setup that requires a super user superuser_session = self.patient_cql_connection(node1, user='cassandra', password='cassandra') - superuser_session.execute("create user ks1_user with password 'cassandra' nosuperuser;") - superuser_session.execute("create user ks2_user with password 'cassandra' nosuperuser;") - create_ks(superuser_session, 'ks1', 2) - create_ks(superuser_session, 'ks2', 2) - superuser_session.execute("grant all permissions on keyspace ks1 to ks1_user;") - superuser_session.execute("grant all permissions on keyspace ks2 to ks2_user;") + + # role setup is async and does not log when complete; the only way to know its done (on a single instance) is to get a successful return. + def allowed_error(error): + return isinstance(error, InvalidRequest) and "Cannot process role related query as the role manager isn't yet setup" in str(error) + def r(fn): + retry(fn, allowed_error=allowed_error, sleep_seconds=10) + + # role setup process is local to each instance, so can't rely on the first query passing to say this is complete; node1 may be complete but not node2 and node3 + # for this reason, need to be aggressive about retries + r(lambda: superuser_session.execute("create user ks1_user with password 'cassandra' nosuperuser;")) + r(lambda: superuser_session.execute("create user ks2_user with password 'cassandra' nosuperuser;")) + r(lambda: create_ks(superuser_session, 'ks1', 2)) + r(lambda: create_ks(superuser_session, 'ks2', 2)) + r(lambda: superuser_session.execute("grant all permissions on keyspace ks1 to ks1_user;")) + r(lambda: superuser_session.execute("grant all permissions on keyspace ks2 to ks2_user;")) user1_session = self.patient_cql_connection(node1, user='ks1_user', password='cassandra') user2_session = self.patient_cql_connection(node1, user='ks2_user', password='cassandra') --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org