This is an automated email from the ASF dual-hosted git repository. bmahler pushed a commit to branch 1.5.x in repository https://gitbox.apache.org/repos/asf/mesos.git
commit f02add8696acaef19442e159cd6cca4a359f5be5 Author: Meng Zhu <m...@mesosphere.io> AuthorDate: Tue Aug 14 17:03:25 2018 -0700 Added two agent flags for min/max authentication retry interval. flags.authentication_timeout_min controls the minimum amount of time the agent waits before retrying authenticating with the master after a failed attempt. flags.authentication_timeout_max controls the maximum amount of time the agent waits before retrying authenticating with the master after a failed attempt. Review: https://reviews.apache.org/r/68324 --- docs/authentication.md | 14 ++++++++++++++ docs/configuration/agent.md | 37 ++++++++++++++++++++++++++++++------- src/slave/constants.hpp | 20 +++++++++----------- src/slave/flags.cpp | 24 ++++++++++++++++++++---- src/slave/flags.hpp | 2 ++ src/slave/slave.cpp | 8 ++++---- src/tests/authentication_tests.cpp | 2 +- 7 files changed, 80 insertions(+), 27 deletions(-) diff --git a/docs/authentication.md b/docs/authentication.md index f63b98f..7f2a4c3 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -115,6 +115,20 @@ Mesos master and agent processes. For more information, refer to the format) of accepted credentials. This may be optional depending on the authenticator being used. +* `--authentication_backoff_factor` - The agent will time out its authentication + with the master based on exponential backoff. The timeout will be randomly + chosen within the range `[min, min + factor*2^n]` where `n` is the number of + failed attempts. To tune these parameters, set the + `--authentication_timeout_[min|max|factor]` flags. (default: 1secs) + +* `--authentication_timeout_min` - The minimum amount of time the agent waits + before retrying authenticating with the master. See + `--authentication_backoff_factor` for more details. (default: 5secs) + +* `--authentication_timeout_max` - The maximum amount of time the agent waits + before retrying authenticating with the master. See + `--authentication_backoff_factor` for more details. (default: 1mins) + ### Scheduler Driver * `--authenticatee` - Analog to the master's `--authenticators` option to diff --git a/docs/configuration/agent.md b/docs/configuration/agent.md index bf2d7b9..3cccf89 100644 --- a/docs/configuration/agent.md +++ b/docs/configuration/agent.md @@ -225,13 +225,36 @@ load an alternate authenticatee module using <code>--modules</code>. (default: c --authentication_backoff_factor=VALUE </td> <td> -After a failed authentication the agent picks a random amount of time between -<code>[0, b]</code>, where <code>b = authentication_backoff_factor</code>, to -authenticate with a new master. Subsequent retries are exponentially backed -off based on this interval (e.g., 1st retry uses a random value between -<code>[0, b * 2^1]</code>, 2nd retry between <code>[0, b * 2^2]</code>, 3rd -retry between <code>[0, b * 2^3]</code>, etc up to a maximum of 1mins -(default: 1secs) +The agent will time out its authentication with the master based on +exponential backoff. The timeout will be randomly chosen within the +range <code>[min, min + factor*2^n]</code> where <code>n</code> is the number +of failed attempts. To tune these parameters, set the +<code>--authentication_timeout_[min|max|factor]</code> flags. (default: 1secs) + </td> +</tr> + +<tr id="authentication_timeout_min"> + <td> + --authentication_timeout_min=VALUE + </td> + <td> +The minimum amount of time the agent waits before retrying authenticating +with the master. See <code>--authentication_backoff_factor</code> for more +details. (default: 5secs) +<p/>NOTE that since authentication retry cancels the previous authentication +request, one should consider what is the normal authentication delay when +setting this flag to prevent premature retry.</p> + </td> +</tr> + +<tr id="authentication_timeout_max"> + <td> + --authentication_timeout_max=VALUE + </td> + <td> +The maximum amount of time the agent waits before retrying authenticating +with the master. See <code>--authentication_backoff_factor</code> for more +details. (default: 1mins) </td> </tr> <tr> diff --git a/src/slave/constants.hpp b/src/slave/constants.hpp index 96f734f..e2ebde5 100644 --- a/src/slave/constants.hpp +++ b/src/slave/constants.hpp @@ -65,19 +65,17 @@ constexpr Duration DEFAULT_REGISTRATION_BACKOFF_FACTOR = Seconds(1); // recovery and when it times out slave re-registration. constexpr Duration REGISTER_RETRY_INTERVAL_MAX = Minutes(1); -// The minimum timeout used when authenticating against the master. -// -// TODO(mzhu): Make this configurable. -constexpr Duration AUTHENTICATION_TIMEOUT_MIN = Seconds(5); +// Default value for `--authentication_backoff_factor`. The backoff interval +// factor affects the agent timeout interval after failed authentications. +constexpr Duration DEFAULT_AUTHENTICATION_BACKOFF_FACTOR = Seconds(1); -// The maximum timeout used when authenticating against the master. -// -// TODO(mzhu): Make this configurable. -constexpr Duration AUTHENTICATION_TIMEOUT_MAX = Minutes(1); +// Default value for `--authentication_timeout_min`. The minimum timeout +// interval the agent waits before retrying authentication. +constexpr Duration DEFAULT_AUTHENTICATION_TIMEOUT_MIN = Seconds(5); -// Default backoff interval factor used by the slave to wait after failed -// authentication. -constexpr Duration DEFAULT_AUTHENTICATION_BACKOFF_FACTOR = Seconds(1); +// Default value for `--authentication_timeout_max`. The maximum timeout +// interval the agent waits before retrying authentication. +constexpr Duration DEFAULT_AUTHENTICATION_TIMEOUT_MAX = Minutes(1); constexpr Duration GC_DELAY = Weeks(1); constexpr Duration DISK_WATCH_INTERVAL = Minutes(1); diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp index 653cffe..7129cea 100644 --- a/src/slave/flags.cpp +++ b/src/slave/flags.cpp @@ -344,12 +344,28 @@ mesos::internal::slave::Flags::Flags() "authentication_backoff_factor", "The agent will time out its authentication with the master based on\n" "exponential backoff. The timeout will be randomly chosen within the\n" - "range `[min, min + factor*2^n]` where `min` is " + - stringify(AUTHENTICATION_TIMEOUT_MIN) + ", `n` is the number of failed\n" - "attempts. The max timeout interval is capped at " + - stringify(AUTHENTICATION_TIMEOUT_MAX), + "range `[min, min + factor*2^n]` where `n` is the number of failed\n" + "attempts. To tune these parameters, set the\n" + "`--authentication_timeout_[min|max|factor]` flags.\n", DEFAULT_AUTHENTICATION_BACKOFF_FACTOR); + add(&Flags::authentication_timeout_min, + "authentication_timeout_min", + "The minimum amount of time the agent waits before retrying\n" + "authenticating with the master. See `authentication_backoff_factor`\n" + "for more details. NOTE that since authentication retry cancels the\n" + "previous authentication request, one should consider what is the\n" + "normal authentication delay when setting this flag to prevent\n" + "premature retry.", + DEFAULT_AUTHENTICATION_TIMEOUT_MIN); + + add(&Flags::authentication_timeout_max, + "authentication_timeout_max", + "The maximum amount of time the agent waits before retrying\n" + "authenticating with the master. See `authentication_backoff_factor`\n" + "for more details.", + DEFAULT_AUTHENTICATION_TIMEOUT_MAX); + add(&Flags::executor_environment_variables, "executor_environment_variables", "JSON object representing the environment variables that should be\n" diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp index 0c67bf2..5e62e54 100644 --- a/src/slave/flags.hpp +++ b/src/slave/flags.hpp @@ -80,6 +80,8 @@ public: std::string frameworks_home; // TODO(benh): Make an Option. Duration registration_backoff_factor; Duration authentication_backoff_factor; + Duration authentication_timeout_min; + Duration authentication_timeout_max; Option<JSON::Object> executor_environment_variables; Duration executor_registration_timeout; Duration executor_reregistration_timeout; diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp index 1eada93..2e64388 100644 --- a/src/slave/slave.cpp +++ b/src/slave/slave.cpp @@ -1302,15 +1302,15 @@ void Slave::detected(const Future<Option<MasterInfo>>& _master) // Authenticate with the master. // TODO(vinod): Consider adding an "AUTHENTICATED" state to the // slave instead of "authenticate" variable. - Duration maxTimeout = AUTHENTICATION_TIMEOUT_MIN + + Duration maxTimeout = flags.authentication_timeout_min + flags.authentication_backoff_factor * 2; delay( duration, self(), &Slave::authenticate, - AUTHENTICATION_TIMEOUT_MIN, - std::min(maxTimeout, AUTHENTICATION_TIMEOUT_MAX)); + flags.authentication_timeout_min, + std::min(maxTimeout, flags.authentication_timeout_max)); } else { // Proceed with registration without authentication. LOG(INFO) << "No credentials provided." @@ -1440,7 +1440,7 @@ void Slave::_authenticate( authenticate( currentMinTimeout, - std::min(maxTimeout, AUTHENTICATION_TIMEOUT_MAX)); + std::min(maxTimeout, flags.authentication_timeout_max)); return; } diff --git a/src/tests/authentication_tests.cpp b/src/tests/authentication_tests.cpp index eeb3752..c571de9 100644 --- a/src/tests/authentication_tests.cpp +++ b/src/tests/authentication_tests.cpp @@ -383,7 +383,7 @@ TEST_F(AuthenticationTest, RetrySlaveAuthentication) // Advance the clock for the slave to retry. Clock::pause(); - Clock::advance(slave::AUTHENTICATION_TIMEOUT_MAX); + Clock::advance(slave::DEFAULT_AUTHENTICATION_TIMEOUT_MAX); Clock::settle(); Clock::resume();