Made all allocator tests allow for slave backoff. Most of the allocator tests were expecting a single invocation of 'addSlave' on the allocator when registering a slave. Due to the nature of our slave registration backoff, a single slave possibly sends multiple registration requests in short succession. Any event closely connected to a slave registration will therefor be possibly invoked multiple times over the lifetime of such a test.
We now allow multiple invocations of 'addSlave'. This patch also enhances the timing control by introducing a paused clock over the slave registration phase of these tests. Review: https://reviews.apache.org/r/65581/ Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/75dd7188 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/75dd7188 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/75dd7188 Branch: refs/heads/master Commit: 75dd718866675542be77696eecd7570c0a74a966 Parents: 143484e Author: Till Toenshoff <toensh...@me.com> Authored: Tue Feb 20 21:45:31 2018 +0100 Committer: Benjamin Bannier <bbann...@apache.org> Committed: Tue Feb 20 21:49:59 2018 +0100 ---------------------------------------------------------------------- src/tests/master_allocator_tests.cpp | 132 +++++++++++++++++++++++++----- 1 file changed, 113 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/75dd7188/src/tests/master_allocator_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/master_allocator_tests.cpp b/src/tests/master_allocator_tests.cpp index e4a6383..43f1c71 100644 --- a/src/tests/master_allocator_tests.cpp +++ b/src/tests/master_allocator_tests.cpp @@ -162,6 +162,8 @@ TYPED_TEST_CASE(MasterAllocatorTest, AllocatorTypes); // the slave's resources are offered to the framework. TYPED_TEST(MasterAllocatorTest, SingleFramework) { + Clock::pause(); + TestAllocator<TypeParam> allocator; EXPECT_CALL(allocator, initialize(_, _, _, _, _, _)); @@ -172,12 +174,17 @@ TYPED_TEST(MasterAllocatorTest, SingleFramework) slave::Flags flags = this->CreateSlaveFlags(); flags.resources = Some("cpus:2;mem:1024;disk:0"); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = this->StartSlave(detector.get(), flags); ASSERT_SOME(slave); + // Skip all the backoffs to make sure the sent a registration request. + Clock::advance(flags.authentication_backoff_factor); + Clock::advance(flags.registration_backoff_factor); + MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); @@ -319,6 +326,8 @@ TYPED_TEST(MasterAllocatorTest, ResourcesUnused) // recoverResources is called for an already removed framework. TYPED_TEST(MasterAllocatorTest, OutOfOrderDispatch) { + Clock::pause(); + TestAllocator<TypeParam> allocator; EXPECT_CALL(allocator, initialize(_, _, _, _, _, _)); @@ -329,7 +338,8 @@ TYPED_TEST(MasterAllocatorTest, OutOfOrderDispatch) slave::Flags slaveFlags = this->CreateSlaveFlags(); slaveFlags.resources = Some("cpus:2;mem:1024"); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); Owned<MasterDetector> detector = master.get()->createDetector(); @@ -337,6 +347,9 @@ TYPED_TEST(MasterAllocatorTest, OutOfOrderDispatch) this->StartSlave(detector.get(), slaveFlags); ASSERT_SOME(slave); + Clock::advance(slaveFlags.authentication_backoff_factor); + Clock::advance(slaveFlags.registration_backoff_factor); + FrameworkInfo frameworkInfo1 = DEFAULT_FRAMEWORK_INFO; frameworkInfo1.set_user("user1"); frameworkInfo1.set_name("framework1"); @@ -448,6 +461,8 @@ TYPED_TEST(MasterAllocatorTest, OutOfOrderDispatch) // is running. TYPED_TEST(MasterAllocatorTest, SchedulerFailover) { + Clock::pause(); + TestAllocator<TypeParam> allocator; EXPECT_CALL(allocator, initialize(_, _, _, _, _, _)); @@ -461,7 +476,8 @@ TYPED_TEST(MasterAllocatorTest, SchedulerFailover) slave::Flags flags = this->CreateSlaveFlags(); flags.resources = Some("cpus:3;mem:1024"); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); Owned<MasterDetector> detector = master.get()->createDetector(); @@ -469,6 +485,9 @@ TYPED_TEST(MasterAllocatorTest, SchedulerFailover) this->StartSlave(detector.get(), &containerizer, flags); ASSERT_SOME(slave); + Clock::advance(flags.authentication_backoff_factor); + Clock::advance(flags.registration_backoff_factor); + FrameworkInfo frameworkInfo1 = DEFAULT_FRAMEWORK_INFO; frameworkInfo1.set_name("framework1"); frameworkInfo1.set_user("user1"); @@ -739,6 +758,8 @@ TYPED_TEST(MasterAllocatorTest, FrameworkExited) // slave, never offered again. TYPED_TEST(MasterAllocatorTest, SlaveLost) { + Clock::pause(); + TestAllocator<TypeParam> allocator; EXPECT_CALL(allocator, initialize(_, _, _, _, _, _)); @@ -756,7 +777,8 @@ TYPED_TEST(MasterAllocatorTest, SlaveLost) flags1.executor_shutdown_grace_period = Milliseconds(50); flags1.resources = Some("cpus:2;mem:1024"); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); Owned<MasterDetector> detector = master.get()->createDetector(); @@ -764,6 +786,9 @@ TYPED_TEST(MasterAllocatorTest, SlaveLost) this->StartSlave(detector.get(), &containerizer, flags1); ASSERT_SOME(slave1); + Clock::advance(flags1.authentication_backoff_factor); + Clock::advance(flags1.registration_backoff_factor); + MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); @@ -827,7 +852,8 @@ TYPED_TEST(MasterAllocatorTest, SlaveLost) flags2.resources = string("cpus:3;gpus:0;mem:256;" "disk:1024;ports:[31000-32000]"); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); // Eventually after slave2 is launched, we should get // an offer that contains all of slave2's resources @@ -839,6 +865,9 @@ TYPED_TEST(MasterAllocatorTest, SlaveLost) Try<Owned<cluster::Slave>> slave2 = this->StartSlave(detector.get(), flags2); ASSERT_SOME(slave2); + Clock::advance(flags2.authentication_backoff_factor); + Clock::advance(flags2.registration_backoff_factor); + AWAIT_READY(resourceOffers); EXPECT_EQ(Resources(resourceOffers.get()[0].resources()), @@ -863,6 +892,8 @@ TYPED_TEST(MasterAllocatorTest, SlaveLost) // resources and offered appropriately. TYPED_TEST(MasterAllocatorTest, SlaveAdded) { + Clock::pause(); + TestAllocator<TypeParam> allocator; EXPECT_CALL(allocator, initialize(_, _, _, _, _, _)); @@ -878,7 +909,8 @@ TYPED_TEST(MasterAllocatorTest, SlaveAdded) slave::Flags flags1 = this->CreateSlaveFlags(); flags1.resources = Some("cpus:3;mem:1024"); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); Owned<MasterDetector> detector = master.get()->createDetector(); @@ -886,6 +918,9 @@ TYPED_TEST(MasterAllocatorTest, SlaveAdded) this->StartSlave(detector.get(), &containerizer, flags1); ASSERT_SOME(slave1); + Clock::advance(flags1.authentication_backoff_factor); + Clock::advance(flags1.registration_backoff_factor); + MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); @@ -930,7 +965,8 @@ TYPED_TEST(MasterAllocatorTest, SlaveAdded) slave::Flags flags2 = this->CreateSlaveFlags(); flags2.resources = Some("cpus:4;mem:2048"); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); // After slave2 launches, all of its resources are combined with the // resources on slave1 that the task isn't using. @@ -941,6 +977,9 @@ TYPED_TEST(MasterAllocatorTest, SlaveAdded) Try<Owned<cluster::Slave>> slave2 = this->StartSlave(detector.get(), flags2); ASSERT_SOME(slave2); + // TODO(tillt): Allow for a fully paused clock throughout this test. + Clock::resume(); + AWAIT_READY(resourceOffers); EXPECT_CALL(exec, shutdown(_)) @@ -959,6 +998,8 @@ TYPED_TEST(MasterAllocatorTest, SlaveAdded) // resources are recovered and reoffered correctly. TYPED_TEST(MasterAllocatorTest, TaskFinished) { + Clock::pause(); + TestAllocator<TypeParam> allocator; EXPECT_CALL(allocator, initialize(_, _, _, _, _, _)); @@ -975,7 +1016,8 @@ TYPED_TEST(MasterAllocatorTest, TaskFinished) slave::Flags flags = this->CreateSlaveFlags(); flags.resources = Some("cpus:3;mem:1024"); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); Owned<MasterDetector> detector = master.get()->createDetector(); @@ -983,6 +1025,9 @@ TYPED_TEST(MasterAllocatorTest, TaskFinished) this->StartSlave(detector.get(), &containerizer, flags); ASSERT_SOME(slave); + Clock::advance(flags.authentication_backoff_factor); + Clock::advance(flags.registration_backoff_factor); + MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); @@ -1045,6 +1090,9 @@ TYPED_TEST(MasterAllocatorTest, TaskFinished) execDriver->sendStatusUpdate(status); + // TODO(tillt): Allow for a fully paused clock throughout this test. + Clock::resume(); + AWAIT_READY(resourceOffers); EXPECT_CALL(exec, shutdown(_)) @@ -1063,6 +1111,8 @@ TYPED_TEST(MasterAllocatorTest, TaskFinished) // and tasks using only cpus are launched. TYPED_TEST(MasterAllocatorTest, CpusOnlyOfferedAndTaskLaunched) { + Clock::pause(); + TestAllocator<TypeParam> allocator; EXPECT_CALL(allocator, initialize(_, _, _, _, _, _)); @@ -1080,7 +1130,8 @@ TYPED_TEST(MasterAllocatorTest, CpusOnlyOfferedAndTaskLaunched) slave::Flags flags = this->CreateSlaveFlags(); flags.resources = Some("cpus:2;mem:0"); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); Owned<MasterDetector> detector = master.get()->createDetector(); @@ -1088,6 +1139,9 @@ TYPED_TEST(MasterAllocatorTest, CpusOnlyOfferedAndTaskLaunched) this->StartSlave(detector.get(), &containerizer, flags); ASSERT_SOME(slave); + Clock::advance(flags.authentication_backoff_factor); + Clock::advance(flags.registration_backoff_factor); + MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); @@ -1129,6 +1183,9 @@ TYPED_TEST(MasterAllocatorTest, CpusOnlyOfferedAndTaskLaunched) execDriver->sendStatusUpdate(status); + // TODO(tillt): Allow for a fully paused clock throughout this test. + Clock::resume(); + AWAIT_READY(resourceOffers); EXPECT_CALL(exec, shutdown(_)) @@ -1144,6 +1201,8 @@ TYPED_TEST(MasterAllocatorTest, CpusOnlyOfferedAndTaskLaunched) // and tasks using only memory are launched. TYPED_TEST(MasterAllocatorTest, MemoryOnlyOfferedAndTaskLaunched) { + Clock::pause(); + TestAllocator<TypeParam> allocator; EXPECT_CALL(allocator, initialize(_, _, _, _, _, _)); @@ -1161,7 +1220,8 @@ TYPED_TEST(MasterAllocatorTest, MemoryOnlyOfferedAndTaskLaunched) slave::Flags flags = this->CreateSlaveFlags(); flags.resources = Some("cpus:0;mem:200"); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); Owned<MasterDetector> detector = master.get()->createDetector(); @@ -1169,6 +1229,9 @@ TYPED_TEST(MasterAllocatorTest, MemoryOnlyOfferedAndTaskLaunched) this->StartSlave(detector.get(), &containerizer, flags); ASSERT_SOME(slave); + Clock::advance(flags.authentication_backoff_factor); + Clock::advance(flags.registration_backoff_factor); + MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); @@ -1210,6 +1273,9 @@ TYPED_TEST(MasterAllocatorTest, MemoryOnlyOfferedAndTaskLaunched) execDriver->sendStatusUpdate(status); + // TODO(tillt): Allow for a fully paused clock throughout this test. + Clock::resume(); + AWAIT_READY(resourceOffers); EXPECT_CALL(exec, shutdown(_)) @@ -1277,6 +1343,8 @@ TYPED_TEST(MasterAllocatorTest, Whitelist) // master's command line flags. TYPED_TEST(MasterAllocatorTest, RoleTest) { + Clock::pause(); + TestAllocator<TypeParam> allocator; EXPECT_CALL(allocator, initialize(_, _, _, _, _, _)); @@ -1356,6 +1424,8 @@ TYPED_TEST(MasterAllocatorTest, RoleTest) // accounted for correctly. TYPED_TEST(MasterAllocatorTest, FrameworkReregistersFirst) { + Clock::pause(); + // Objects that persist after the election of a new master. StandaloneMasterDetector slaveDetector; StandaloneMasterDetector schedulerDetector; @@ -1383,7 +1453,8 @@ TYPED_TEST(MasterAllocatorTest, FrameworkReregistersFirst) slaveDetector.appoint(master.get()->pid); schedulerDetector.appoint(master.get()->pid); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); slave::Flags flags = this->CreateSlaveFlags(); flags.resources = Some("cpus:2;mem:1024"); @@ -1391,6 +1462,9 @@ TYPED_TEST(MasterAllocatorTest, FrameworkReregistersFirst) slave = this->StartSlave(&slaveDetector, &containerizer, flags); ASSERT_SOME(slave); + Clock::advance(flags.authentication_backoff_factor); + Clock::advance(flags.registration_backoff_factor); + EXPECT_CALL(allocator, addFramework(_, _, _, _, _)); EXPECT_CALL(sched, registered(&driver, _, _)); @@ -1461,6 +1535,9 @@ TYPED_TEST(MasterAllocatorTest, FrameworkReregistersFirst) // Inform the slave about the new master. slaveDetector.appoint(master2.get()->pid); + // Trigger a batch allocation. + Clock::advance(masterFlags.allocation_interval); + AWAIT_READY(resourceOffers2); // Since the task is still running on the slave, the framework @@ -1484,6 +1561,8 @@ TYPED_TEST(MasterAllocatorTest, FrameworkReregistersFirst) // accounted for correctly. TYPED_TEST(MasterAllocatorTest, SlaveReregistersFirst) { + Clock::pause(); + // Objects that persist after the election of a new master. StandaloneMasterDetector slaveDetector; StandaloneMasterDetector schedulerDetector; @@ -1492,6 +1571,10 @@ TYPED_TEST(MasterAllocatorTest, SlaveReregistersFirst) TestContainerizer containerizer(&exec); TestingMesosSchedulerDriver driver(&sched, &schedulerDetector); Try<Owned<cluster::Slave>> slave = nullptr; + master::Flags masterFlags = this->CreateMasterFlags(); + + slave::Flags slaveFlags = this->CreateSlaveFlags(); + slaveFlags.resources = Some("cpus:2;mem:1024"); // Explicit scope is to ensure all object associated with the // leading master (e.g. allocator) are destroyed once the master @@ -1502,20 +1585,23 @@ TYPED_TEST(MasterAllocatorTest, SlaveReregistersFirst) EXPECT_CALL(allocator, initialize(_, _, _, _, _, _)); - Try<Owned<cluster::Master>> master = this->StartMaster(&allocator); + Try<Owned<cluster::Master>> master = + this->StartMaster(&allocator, masterFlags); + ASSERT_SOME(master); slaveDetector.appoint(master.get()->pid); schedulerDetector.appoint(master.get()->pid); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); - - slave::Flags flags = this->CreateSlaveFlags(); - flags.resources = Some("cpus:2;mem:1024"); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); - slave = this->StartSlave(&slaveDetector, &containerizer, flags); + slave = this->StartSlave(&slaveDetector, &containerizer, slaveFlags); ASSERT_SOME(slave); + Clock::advance(slaveFlags.authentication_backoff_factor); + Clock::advance(slaveFlags.registration_backoff_factor); + EXPECT_CALL(allocator, addFramework(_, _, _, _, _)); EXPECT_CALL(allocator, recoverResources(_, _, _, _)); @@ -1576,6 +1662,9 @@ TYPED_TEST(MasterAllocatorTest, SlaveReregistersFirst) // Inform the slave about the new master. slaveDetector.appoint(master2.get()->pid); + Clock::advance(slaveFlags.authentication_backoff_factor); + Clock::advance(slaveFlags.registration_backoff_factor); + AWAIT_READY(addSlave); AWAIT_READY(addFramework); @@ -1597,6 +1686,9 @@ TYPED_TEST(MasterAllocatorTest, SlaveReregistersFirst) // Inform the scheduler about the new master. schedulerDetector.appoint(master2.get()->pid); + // Trigger a batch allocation. + Clock::advance(masterFlags.allocation_interval); + AWAIT_READY(resourceOffers2); // Since the task is still running on the slave, the framework @@ -1823,7 +1915,8 @@ TYPED_TEST(MasterAllocatorTest, NestedRoles) this->StartMaster(&allocator, masterFlags); ASSERT_SOME(master); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); MockExecutor exec1(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer1(&exec1); @@ -1914,7 +2007,8 @@ TYPED_TEST(MasterAllocatorTest, NestedRoles) MockExecutor exec2(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer2(&exec2); - EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)); + EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _)) + .WillOnce(InvokeAddSlave(&allocator)); EXPECT_CALL(sched3, resourceOffers(&driver3, OfferEq(1, 512))) .WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 512, "b/x"));
