[mesos] 03/04: Enabled `--fetch_stall_timeout` in curl-based URI fetcher plugins.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch 1.4.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit c57471feb9135d3cea5bb8a05ced1d772be3e3fb
Author: Chun-Hung Hsiao
AuthorDate: Wed Mar 28 22:47:58 2018 -0700
Enabled `--fetch_stall_timeout` in curl-based URI fetcher plugins.
This patch passes the `--fetch_stall_timeout` agent flag into
`DockerFetcherPlugin` (through setting flag `docker_stall_timeout` in
the Docker store) and `CurlFetcherPlugin` (through setting flag
`curl_stall_timeout` in the Appc store).
Review: https://reviews.apache.org/r/65876/
---
.../containerizer/mesos/provisioner/appc/store.cpp | 5 +-
.../mesos/provisioner/docker/store.cpp | 1 +
src/uri/fetchers/curl.cpp | 23 +++-
src/uri/fetchers/curl.hpp | 12 +++-
src/uri/fetchers/docker.cpp| 64 --
src/uri/fetchers/docker.hpp| 1 +
6 files changed, 85 insertions(+), 21 deletions(-)
diff --git a/src/slave/containerizer/mesos/provisioner/appc/store.cpp
b/src/slave/containerizer/mesos/provisioner/appc/store.cpp
index 9e65990..c859518 100644
--- a/src/slave/containerizer/mesos/provisioner/appc/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/appc/store.cpp
@@ -129,7 +129,10 @@ Try> Store::create(
// TODO(jojy): Uri fetcher has 'shared' semantics for the
// provisioner. It's a shared pointer which needs to be injected
// from top level into the store (instead of being created here).
- Try> uriFetcher = uri::fetcher::create();
+ uri::fetcher::Flags _flags;
+ _flags.curl_stall_timeout = flags.fetcher_stall_timeout;
+
+ Try> uriFetcher = uri::fetcher::create(_flags);
if (uriFetcher.isError()) {
return Error("Failed to create uri fetcher: " + uriFetcher.error());
}
diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
index f357710..34d36a7 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
@@ -125,6 +125,7 @@ Try> Store::create(
// TODO(dpravat): Remove after resolving MESOS-5473.
#ifndef __WINDOWS__
_flags.docker_config = flags.docker_config;
+ _flags.docker_stall_timeout = flags.fetcher_stall_timeout;
#endif
Try> fetcher = uri::fetcher::create(_flags);
diff --git a/src/uri/fetchers/curl.cpp b/src/uri/fetchers/curl.cpp
index f34daf2..2f67a86 100644
--- a/src/uri/fetchers/curl.cpp
+++ b/src/uri/fetchers/curl.cpp
@@ -54,6 +54,16 @@ using process::Subprocess;
namespace mesos {
namespace uri {
+CurlFetcherPlugin::Flags::Flags()
+{
+ add(::curl_stall_timeout,
+ "curl_stall_timeout",
+ "Amount of time for the fetcher to wait before considering a download\n"
+ "being too slow and abort it when the download stalls (i.e., the speed\n"
+ "keeps below one byte per second).\n");
+}
+
+
const char CurlFetcherPlugin::NAME[] = "curl";
@@ -61,7 +71,7 @@ Try> CurlFetcherPlugin::create(const
Flags& flags)
{
// TODO(jieyu): Make sure curl is available.
- return Owned(new CurlFetcherPlugin());
+ return Owned(new CurlFetcherPlugin(flags));
}
@@ -98,7 +108,7 @@ Future CurlFetcherPlugin::fetch(
// TODO(jieyu): Allow user to specify the name of the output file.
const string output = path::join(directory, Path(uri.path()).basename());
- const vector argv = {
+ vector argv = {
"curl",
"-s", // Don't show progress meter or error messages.
"-S", // Makes curl show an error message if it fails.
@@ -108,6 +118,15 @@ Future CurlFetcherPlugin::fetch(
strings::trim(stringify(uri))
};
+ // Add a timeout for curl to abort when the download speed keeps low
+ // (1 byte per second by default) for the specified duration. See:
+ // https://curl.haxx.se/docs/manpage.html#-y
+ if (flags.curl_stall_timeout.isSome()) {
+argv.push_back("-y");
+argv.push_back(
+std::to_string(static_cast(flags.curl_stall_timeout->secs(;
+ }
+
Try s = subprocess(
"curl",
argv,
diff --git a/src/uri/fetchers/curl.hpp b/src/uri/fetchers/curl.hpp
index 909c2eb..e07c1e2 100644
--- a/src/uri/fetchers/curl.hpp
+++ b/src/uri/fetchers/curl.hpp
@@ -30,7 +30,13 @@ namespace uri {
class CurlFetcherPlugin : public Fetcher::Plugin
{
public:
- class Flags : public virtual flags::FlagsBase {};
+ class Flags : public virtual flags::FlagsBase
+ {
+ public:
+Flags();
+
+Option curl_stall_timeout;
+ };
static const char NAME[];
@@ -48,7 +54,9 @@ public:
const Option& data = None()) const;
private:
- CurlFetcherPlugin() {}
+ explicit CurlFetcherPlugin(const Flags& _flags) : flags(_flags) {}
+
+ const Flags flags;
};
} // namespace uri {
diff --git a/src/uri/fetchers/docker.cpp
[mesos] 03/04: Enabled `--fetch_stall_timeout` in curl-based URI fetcher plugins.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch 1.5.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 97f73a9e844f9f37d54a97c8993dbf05cffc9592
Author: Chun-Hung Hsiao
AuthorDate: Wed Mar 28 22:47:58 2018 -0700
Enabled `--fetch_stall_timeout` in curl-based URI fetcher plugins.
This patch passes the `--fetch_stall_timeout` agent flag into
`DockerFetcherPlugin` (through setting flag `docker_stall_timeout` in
the Docker store) and `CurlFetcherPlugin` (through setting flag
`curl_stall_timeout` in the Appc store).
Review: https://reviews.apache.org/r/65876/
---
.../containerizer/mesos/provisioner/appc/store.cpp | 5 +-
.../mesos/provisioner/docker/store.cpp | 1 +
src/uri/fetchers/curl.cpp | 23 +++-
src/uri/fetchers/curl.hpp | 12 +++-
src/uri/fetchers/docker.cpp| 64 --
src/uri/fetchers/docker.hpp| 1 +
6 files changed, 85 insertions(+), 21 deletions(-)
diff --git a/src/slave/containerizer/mesos/provisioner/appc/store.cpp
b/src/slave/containerizer/mesos/provisioner/appc/store.cpp
index c1f9661..f30c166 100644
--- a/src/slave/containerizer/mesos/provisioner/appc/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/appc/store.cpp
@@ -131,7 +131,10 @@ Try> Store::create(
// TODO(jojy): Uri fetcher has 'shared' semantics for the
// provisioner. It's a shared pointer which needs to be injected
// from top level into the store (instead of being created here).
- Try> uriFetcher = uri::fetcher::create();
+ uri::fetcher::Flags _flags;
+ _flags.curl_stall_timeout = flags.fetcher_stall_timeout;
+
+ Try> uriFetcher = uri::fetcher::create(_flags);
if (uriFetcher.isError()) {
return Error("Failed to create uri fetcher: " + uriFetcher.error());
}
diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
index d64e6eb..d277cc6 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
@@ -141,6 +141,7 @@ Try> Store::create(
// TODO(dpravat): Remove after resolving MESOS-5473.
#ifndef __WINDOWS__
_flags.docker_config = flags.docker_config;
+ _flags.docker_stall_timeout = flags.fetcher_stall_timeout;
#endif
Try> fetcher = uri::fetcher::create(_flags);
diff --git a/src/uri/fetchers/curl.cpp b/src/uri/fetchers/curl.cpp
index f34daf2..2f67a86 100644
--- a/src/uri/fetchers/curl.cpp
+++ b/src/uri/fetchers/curl.cpp
@@ -54,6 +54,16 @@ using process::Subprocess;
namespace mesos {
namespace uri {
+CurlFetcherPlugin::Flags::Flags()
+{
+ add(::curl_stall_timeout,
+ "curl_stall_timeout",
+ "Amount of time for the fetcher to wait before considering a download\n"
+ "being too slow and abort it when the download stalls (i.e., the speed\n"
+ "keeps below one byte per second).\n");
+}
+
+
const char CurlFetcherPlugin::NAME[] = "curl";
@@ -61,7 +71,7 @@ Try> CurlFetcherPlugin::create(const
Flags& flags)
{
// TODO(jieyu): Make sure curl is available.
- return Owned(new CurlFetcherPlugin());
+ return Owned(new CurlFetcherPlugin(flags));
}
@@ -98,7 +108,7 @@ Future CurlFetcherPlugin::fetch(
// TODO(jieyu): Allow user to specify the name of the output file.
const string output = path::join(directory, Path(uri.path()).basename());
- const vector argv = {
+ vector argv = {
"curl",
"-s", // Don't show progress meter or error messages.
"-S", // Makes curl show an error message if it fails.
@@ -108,6 +118,15 @@ Future CurlFetcherPlugin::fetch(
strings::trim(stringify(uri))
};
+ // Add a timeout for curl to abort when the download speed keeps low
+ // (1 byte per second by default) for the specified duration. See:
+ // https://curl.haxx.se/docs/manpage.html#-y
+ if (flags.curl_stall_timeout.isSome()) {
+argv.push_back("-y");
+argv.push_back(
+std::to_string(static_cast(flags.curl_stall_timeout->secs(;
+ }
+
Try s = subprocess(
"curl",
argv,
diff --git a/src/uri/fetchers/curl.hpp b/src/uri/fetchers/curl.hpp
index 909c2eb..e07c1e2 100644
--- a/src/uri/fetchers/curl.hpp
+++ b/src/uri/fetchers/curl.hpp
@@ -30,7 +30,13 @@ namespace uri {
class CurlFetcherPlugin : public Fetcher::Plugin
{
public:
- class Flags : public virtual flags::FlagsBase {};
+ class Flags : public virtual flags::FlagsBase
+ {
+ public:
+Flags();
+
+Option curl_stall_timeout;
+ };
static const char NAME[];
@@ -48,7 +54,9 @@ public:
const Option& data = None()) const;
private:
- CurlFetcherPlugin() {}
+ explicit CurlFetcherPlugin(const Flags& _flags) : flags(_flags) {}
+
+ const Flags flags;
};
} // namespace uri {
diff --git a/src/uri/fetchers/docker.cpp
