Removed CgroupsPerfEventIsolatorProcess. Review: https://reviews.apache.org/r/50751/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/edf824b4 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/edf824b4 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/edf824b4 Branch: refs/heads/master Commit: edf824b4d95d44c33c7c381718594b01d0357712 Parents: 34bcae4 Author: haosdent huang <haosd...@gmail.com> Authored: Mon Sep 12 16:34:51 2016 -0700 Committer: Jie Yu <yujie....@gmail.com> Committed: Tue Sep 13 10:52:01 2016 -0700 ---------------------------------------------------------------------- src/CMakeLists.txt | 1 - src/Makefile.am | 2 - src/slave/containerizer/mesos/containerizer.cpp | 1 - .../mesos/isolators/cgroups/perf_event.cpp | 415 ------------------- .../mesos/isolators/cgroups/perf_event.hpp | 118 ------ src/tests/containerizer/isolator_tests.cpp | 232 ----------- 6 files changed, 769 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/edf824b4/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7caa466..42c52b6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -159,7 +159,6 @@ set(LINUX_SRC slave/containerizer/mesos/linux_launcher.cpp slave/containerizer/mesos/isolators/appc/runtime.cpp slave/containerizer/mesos/isolators/cgroups/cgroups.cpp - slave/containerizer/mesos/isolators/cgroups/perf_event.cpp slave/containerizer/mesos/isolators/cgroups/subsystem.cpp slave/containerizer/mesos/isolators/cgroups/subsystems/cpu.cpp slave/containerizer/mesos/isolators/cgroups/subsystems/cpuacct.cpp http://git-wip-us.apache.org/repos/asf/mesos/blob/edf824b4/src/Makefile.am ---------------------------------------------------------------------- diff --git a/src/Makefile.am b/src/Makefile.am index d4b90f1..e789994 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1025,7 +1025,6 @@ MESOS_LINUX_FILES = \ slave/containerizer/mesos/linux_launcher.cpp \ slave/containerizer/mesos/isolators/appc/runtime.cpp \ slave/containerizer/mesos/isolators/cgroups/cgroups.cpp \ - slave/containerizer/mesos/isolators/cgroups/perf_event.cpp \ slave/containerizer/mesos/isolators/cgroups/subsystem.cpp \ slave/containerizer/mesos/isolators/cgroups/subsystems/cpu.cpp \ slave/containerizer/mesos/isolators/cgroups/subsystems/cpuacct.cpp \ @@ -1062,7 +1061,6 @@ MESOS_LINUX_FILES += \ slave/containerizer/mesos/isolators/appc/runtime.hpp \ slave/containerizer/mesos/isolators/cgroups/cgroups.hpp \ slave/containerizer/mesos/isolators/cgroups/constants.hpp \ - slave/containerizer/mesos/isolators/cgroups/perf_event.hpp \ slave/containerizer/mesos/isolators/cgroups/subsystem.hpp \ slave/containerizer/mesos/isolators/cgroups/subsystems/cpu.hpp \ slave/containerizer/mesos/isolators/cgroups/subsystems/cpuacct.hpp \ http://git-wip-us.apache.org/repos/asf/mesos/blob/edf824b4/src/slave/containerizer/mesos/containerizer.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/containerizer.cpp b/src/slave/containerizer/mesos/containerizer.cpp index bbf5d53..dc18e4e 100644 --- a/src/slave/containerizer/mesos/containerizer.cpp +++ b/src/slave/containerizer/mesos/containerizer.cpp @@ -73,7 +73,6 @@ #ifdef __linux__ #include "slave/containerizer/mesos/isolators/cgroups/cgroups.hpp" -#include "slave/containerizer/mesos/isolators/cgroups/perf_event.hpp" #endif // __linux__ #ifdef __linux__ http://git-wip-us.apache.org/repos/asf/mesos/blob/edf824b4/src/slave/containerizer/mesos/isolators/cgroups/perf_event.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/isolators/cgroups/perf_event.cpp b/src/slave/containerizer/mesos/isolators/cgroups/perf_event.cpp deleted file mode 100644 index 31f3538..0000000 --- a/src/slave/containerizer/mesos/isolators/cgroups/perf_event.cpp +++ /dev/null @@ -1,415 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include <stdint.h> - -#include <vector> - -#include <google/protobuf/descriptor.h> -#include <google/protobuf/message.h> - -#include <process/collect.hpp> -#include <process/defer.hpp> -#include <process/delay.hpp> -#include <process/io.hpp> -#include <process/pid.hpp> -#include <process/reap.hpp> -#include <process/subprocess.hpp> - -#include <stout/bytes.hpp> -#include <stout/check.hpp> -#include <stout/error.hpp> -#include <stout/foreach.hpp> -#include <stout/hashset.hpp> -#include <stout/lambda.hpp> -#include <stout/os.hpp> -#include <stout/path.hpp> -#include <stout/stringify.hpp> -#include <stout/try.hpp> - -#include "linux/cgroups.hpp" -#include "linux/perf.hpp" - -#include "slave/containerizer/mesos/isolators/cgroups/perf_event.hpp" - -using mesos::slave::ContainerConfig; -using mesos::slave::ContainerLaunchInfo; -using mesos::slave::ContainerLimitation; -using mesos::slave::ContainerState; -using mesos::slave::Isolator; - -using std::list; -using std::set; -using std::string; -using std::vector; - -using process::Clock; -using process::Failure; -using process::Future; -using process::PID; -using process::Time; - -namespace mesos { -namespace internal { -namespace slave { - -Try<Isolator*> CgroupsPerfEventIsolatorProcess::create(const Flags& flags) -{ - LOG(INFO) << "Creating PerfEvent isolator"; - - if (!perf::supported()) { - return Error("Perf is not supported"); - } - - if (flags.perf_duration > flags.perf_interval) { - return Error("Sampling perf for duration (" + - stringify(flags.perf_duration) + - ") > interval (" + - stringify(flags.perf_interval) + - ") is not supported."); - } - - if (!flags.perf_events.isSome()) { - return Error("No perf events specified"); - } - - set<string> events; - foreach (const string& event, - strings::tokenize(flags.perf_events.get(), ",")) { - events.insert(event); - } - - if (!perf::valid(events)) { - return Error("Failed to create PerfEvent isolator, invalid events: " + - stringify(events)); - } - - Try<string> hierarchy = cgroups::prepare( - flags.cgroups_hierarchy, - "perf_event", - flags.cgroups_root); - - if (hierarchy.isError()) { - return Error("Failed to create perf_event cgroup: " + hierarchy.error()); - } - - LOG(INFO) << "PerfEvent isolator will profile for " << flags.perf_duration - << " every " << flags.perf_interval - << " for events: " << stringify(events); - - process::Owned<MesosIsolatorProcess> process( - new CgroupsPerfEventIsolatorProcess(flags, hierarchy.get(), events)); - - return new MesosIsolator(process); -} - - -CgroupsPerfEventIsolatorProcess::~CgroupsPerfEventIsolatorProcess() {} - - -void CgroupsPerfEventIsolatorProcess::initialize() -{ - // Start sampling. - sample(); -} - - -Future<Nothing> CgroupsPerfEventIsolatorProcess::recover( - const list<ContainerState>& states, - const hashset<ContainerID>& orphans) -{ - foreach (const ContainerState& state, states) { - const ContainerID& containerId = state.container_id(); - const string cgroup = path::join(flags.cgroups_root, containerId.value()); - - Try<bool> exists = cgroups::exists(hierarchy, cgroup); - if (exists.isError()) { - foreachvalue (Info* info, infos) { - delete info; - } - - infos.clear(); - return Failure("Failed to check cgroup " + cgroup + - " for container '" + stringify(containerId) + "'"); - } - - if (!exists.get()) { - // This may occur if the executor is exiting and the isolator has - // destroyed the cgroup but the slave dies before noticing this. This - // will be detected when the containerizer tries to monitor the - // executor's pid. - // NOTE: This could also occur if this isolator is now enabled for a - // container that was started without this isolator. For this - // particular isolator it is acceptable to continue running this - // container without a perf_event cgroup because we don't ever - // query it and the destroy will succeed immediately. - VLOG(1) << "Couldn't find perf event cgroup for container " << containerId - << ", perf statistics will not be available"; - continue; - } - - infos[containerId] = new Info(containerId, cgroup); - } - - // Remove orphan cgroups. - Try<vector<string>> cgroups = cgroups::get(hierarchy, flags.cgroups_root); - if (cgroups.isError()) { - foreachvalue (Info* info, infos) { - delete info; - } - infos.clear(); - return Failure(cgroups.error()); - } - - foreach (const string& cgroup, cgroups.get()) { - // Ignore the slave cgroup (see the --agent_subsystems flag). - // TODO(idownes): Remove this when the cgroups layout is updated, - // see MESOS-1185. - if (cgroup == path::join(flags.cgroups_root, "slave")) { - continue; - } - - ContainerID containerId; - containerId.set_value(Path(cgroup).basename()); - - if (infos.contains(containerId)) { - continue; - } - - // Known orphan cgroups will be destroyed by the containerizer - // using the normal cleanup path. See details in MESOS-2367. - if (orphans.contains(containerId)) { - infos[containerId] = new Info(containerId, cgroup); - continue; - } - - LOG(INFO) << "Removing unknown orphaned cgroup '" << cgroup << "'"; - - // We don't wait on the destroy as we don't want to block recovery. - cgroups::destroy(hierarchy, cgroup, cgroups::DESTROY_TIMEOUT); - } - - return Nothing(); -} - - -Future<Option<ContainerLaunchInfo>> CgroupsPerfEventIsolatorProcess::prepare( - const ContainerID& containerId, - const ContainerConfig& containerConfig) -{ - if (infos.contains(containerId)) { - return Failure("Container has already been prepared"); - } - - LOG(INFO) << "Preparing perf event cgroup for " << containerId; - - Info* info = new Info( - containerId, - path::join(flags.cgroups_root, containerId.value())); - - infos[containerId] = CHECK_NOTNULL(info); - - // Create a cgroup for this container. - Try<bool> exists = cgroups::exists(hierarchy, info->cgroup); - - if (exists.isError()) { - return Failure("Failed to prepare isolator: " + exists.error()); - } - - if (exists.get()) { - return Failure("Failed to prepare isolator: cgroup already exists"); - } - - if (!exists.get()) { - Try<Nothing> create = cgroups::create(hierarchy, info->cgroup); - if (create.isError()) { - return Failure("Failed to prepare isolator: " + create.error()); - } - } - - // Chown the cgroup so the executor can create nested cgroups. Do - // not recurse so the control files are still owned by the slave - // user and thus cannot be changed by the executor. - if (containerConfig.has_user()) { - Try<Nothing> chown = os::chown( - containerConfig.user(), - path::join(hierarchy, info->cgroup), - false); - if (chown.isError()) { - return Failure("Failed to prepare isolator: " + chown.error()); - } - } - - return None(); -} - - -Future<Nothing> CgroupsPerfEventIsolatorProcess::isolate( - const ContainerID& containerId, - pid_t pid) -{ - if (!infos.contains(containerId)) { - return Failure("Unknown container"); - } - - Info* info = CHECK_NOTNULL(infos[containerId]); - - Try<Nothing> assign = cgroups::assign(hierarchy, info->cgroup, pid); - if (assign.isError()) { - return Failure("Failed to assign container '" + - stringify(info->containerId) + "' to its own cgroup '" + - path::join(hierarchy, info->cgroup) + - "' : " + assign.error()); - } - - return Nothing(); -} - - -Future<ResourceStatistics> CgroupsPerfEventIsolatorProcess::usage( - const ContainerID& containerId) -{ - if (!infos.contains(containerId)) { - // Return an empty ResourceStatistics, i.e., without - // PerfStatistics, if we don't know about this container. - return ResourceStatistics(); - } - - CHECK_NOTNULL(infos[containerId]); - - ResourceStatistics statistics; - statistics.mutable_perf()->CopyFrom(infos[containerId]->statistics); - - return statistics; -} - - -Future<Nothing> CgroupsPerfEventIsolatorProcess::cleanup( - const ContainerID& containerId) -{ - // Tolerate clean up attempts for unknown containers which may arise from - // repeated clean up attempts (during test cleanup). - if (!infos.contains(containerId)) { - VLOG(1) << "Ignoring cleanup request for unknown container: " - << containerId; - return Nothing(); - } - - Info* info = CHECK_NOTNULL(infos[containerId]); - - info->destroying = true; - - return cgroups::destroy(hierarchy, info->cgroup) - .then(defer(PID<CgroupsPerfEventIsolatorProcess>(this), - &CgroupsPerfEventIsolatorProcess::_cleanup, - containerId)); -} - - -Future<Nothing> CgroupsPerfEventIsolatorProcess::_cleanup( - const ContainerID& containerId) -{ - if (!infos.contains(containerId)) { - return Nothing(); - } - - delete infos[containerId]; - infos.erase(containerId); - - return Nothing(); -} - - -Future<hashmap<string, PerfStatistics>> discardSample( - Future<hashmap<string, PerfStatistics>> future, - const Duration& duration, - const Duration& timeout) -{ - LOG(ERROR) << "Perf sample of " << stringify(duration) - << " failed to complete within " << stringify(timeout) - << "; sampling will be halted"; - - future.discard(); - - return future; -} - - -void CgroupsPerfEventIsolatorProcess::sample() -{ - // Collect a perf sample for all cgroups that are not being - // destroyed. Since destroyal is asynchronous, 'perf stat' may - // fail if the cgroup is destroyed before running perf. - set<string> cgroups; - - foreachvalue (Info* info, infos) { - CHECK_NOTNULL(info); - - if (!info->destroying) { - cgroups.insert(info->cgroup); - } - } - - // The discard timeout includes an allowance of twice the - // reaper interval to ensure we see the perf process exit. - Duration timeout = flags.perf_duration + process::MAX_REAP_INTERVAL() * 2; - - perf::sample(events, cgroups, flags.perf_duration) - .after(timeout, - lambda::bind(&discardSample, - lambda::_1, - flags.perf_duration, - timeout)) - .onAny(defer(PID<CgroupsPerfEventIsolatorProcess>(this), - &CgroupsPerfEventIsolatorProcess::_sample, - Clock::now() + flags.perf_interval, - lambda::_1)); -} - - -void CgroupsPerfEventIsolatorProcess::_sample( - const Time& next, - const Future<hashmap<string, PerfStatistics>>& statistics) -{ - if (!statistics.isReady()) { - // In case the failure is transient or this is due to a timeout, - // we continue sampling. Note that since sampling is done on an - // interval, it should be ok if this is a non-transient failure. - LOG(ERROR) << "Failed to get perf sample: " - << (statistics.isFailed() - ? statistics.failure() - : "discarded due to timeout"); - } else { - // Store the latest statistics, note that cgroups added in the - // interim will be picked up by the next sample. - foreachvalue (Info* info, infos) { - CHECK_NOTNULL(info); - - if (statistics->contains(info->cgroup)) { - info->statistics = statistics->get(info->cgroup).get(); - } - } - } - - // Schedule sample for the next time. - delay(next - Clock::now(), - PID<CgroupsPerfEventIsolatorProcess>(this), - &CgroupsPerfEventIsolatorProcess::sample); -} - -} // namespace slave { -} // namespace internal { -} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/edf824b4/src/slave/containerizer/mesos/isolators/cgroups/perf_event.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/isolators/cgroups/perf_event.hpp b/src/slave/containerizer/mesos/isolators/cgroups/perf_event.hpp deleted file mode 100644 index 4abde12..0000000 --- a/src/slave/containerizer/mesos/isolators/cgroups/perf_event.hpp +++ /dev/null @@ -1,118 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __PERF_EVENT_ISOLATOR_HPP__ -#define __PERF_EVENT_ISOLATOR_HPP__ - -#include <set> - -#include <process/id.hpp> -#include <process/time.hpp> - -#include <stout/hashmap.hpp> -#include <stout/nothing.hpp> - -#include "slave/flags.hpp" - -#include "slave/containerizer/mesos/isolator.hpp" - -namespace mesos { -namespace internal { -namespace slave { - -class CgroupsPerfEventIsolatorProcess : public MesosIsolatorProcess -{ -public: - static Try<mesos::slave::Isolator*> create(const Flags& flags); - - virtual ~CgroupsPerfEventIsolatorProcess(); - - virtual process::Future<Nothing> recover( - const std::list<mesos::slave::ContainerState>& states, - const hashset<ContainerID>& orphans); - - virtual process::Future<Option<mesos::slave::ContainerLaunchInfo>> prepare( - const ContainerID& containerId, - const mesos::slave::ContainerConfig& containerConfig); - - virtual process::Future<Nothing> isolate( - const ContainerID& containerId, - pid_t pid); - - virtual process::Future<ResourceStatistics> usage( - const ContainerID& containerId); - - virtual process::Future<Nothing> cleanup( - const ContainerID& containerId); - -protected: - virtual void initialize(); - -private: - CgroupsPerfEventIsolatorProcess( - const Flags& _flags, - const std::string& _hierarchy, - const std::set<std::string>& _events) - : ProcessBase(process::ID::generate("cgroups-perf-event-isolator")), - flags(_flags), - hierarchy(_hierarchy), - events(_events) {} - - void sample(); - - void _sample( - const process::Time& next, - const process::Future<hashmap<std::string, PerfStatistics>>& statistics); - - virtual process::Future<Nothing> _cleanup(const ContainerID& containerId); - - struct Info - { - Info(const ContainerID& _containerId, const std::string& _cgroup) - : containerId(_containerId), cgroup(_cgroup), destroying(false) - { - // Ensure the initial statistics include the required fields. - // Note the duration is set to zero to indicate no sampling has - // taken place. This empty sample will be returned from usage() - // until the first true sample is obtained. - statistics.set_timestamp(process::Clock::now().secs()); - statistics.set_duration(Seconds(0).secs()); - } - - const ContainerID containerId; - const std::string cgroup; - PerfStatistics statistics; - // Mark a container when we start destruction so we stop sampling it. - bool destroying; - }; - - const Flags flags; - - // The path to the cgroups subsystem hierarchy root. - const std::string hierarchy; - - // Set of events to sample. - std::set<std::string> events; - - // TODO(jieyu): Use Owned<Info>. - hashmap<ContainerID, Info*> infos; -}; - -} // namespace slave { -} // namespace internal { -} // namespace mesos { - -#endif // __PERF_EVENT_ISOLATOR_HPP__ http://git-wip-us.apache.org/repos/asf/mesos/blob/edf824b4/src/tests/containerizer/isolator_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/isolator_tests.cpp b/src/tests/containerizer/isolator_tests.cpp index 85a13d7..9bb1e69 100644 --- a/src/tests/containerizer/isolator_tests.cpp +++ b/src/tests/containerizer/isolator_tests.cpp @@ -52,7 +52,6 @@ #include "slave/slave.hpp" #ifdef __linux__ -#include "slave/containerizer/mesos/isolators/cgroups/perf_event.hpp" #include "slave/containerizer/mesos/isolators/filesystem/shared.hpp" #endif // __linux__ #include "slave/containerizer/mesos/isolators/posix.hpp" @@ -74,7 +73,6 @@ using namespace process; using mesos::internal::master::Master; #ifdef __linux__ -using mesos::internal::slave::CgroupsPerfEventIsolatorProcess; using mesos::internal::slave::Fetcher; using mesos::internal::slave::LinuxLauncher; using mesos::internal::slave::SharedFilesystemIsolatorProcess; @@ -101,85 +99,6 @@ namespace internal { namespace tests { #ifdef __linux__ -class PerfEventIsolatorTest : public MesosTest {}; - - -TEST_F(PerfEventIsolatorTest, ROOT_CGROUPS_PERF_Sample) -{ - slave::Flags flags; - - flags.perf_events = "cycles,task-clock"; - flags.perf_duration = Milliseconds(250); - flags.perf_interval = Milliseconds(500); - - Try<Isolator*> _isolator = CgroupsPerfEventIsolatorProcess::create(flags); - ASSERT_SOME(_isolator); - Owned<Isolator> isolator(_isolator.get()); - - ExecutorInfo executorInfo; - - ContainerID containerId; - containerId.set_value(UUID::random().toString()); - - // Use a relative temporary directory so it gets cleaned up - // automatically with the test. - Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX")); - ASSERT_SOME(dir); - - ContainerConfig containerConfig; - containerConfig.mutable_executor_info()->CopyFrom(executorInfo); - containerConfig.set_directory(dir.get()); - - AWAIT_READY(isolator->prepare( - containerId, - containerConfig)); - - // This first sample is likely to be empty because perf hasn't - // completed yet but we should still have the required fields. - Future<ResourceStatistics> statistics1 = isolator->usage(containerId); - AWAIT_READY(statistics1); - ASSERT_TRUE(statistics1.get().has_perf()); - EXPECT_TRUE(statistics1.get().perf().has_timestamp()); - EXPECT_TRUE(statistics1.get().perf().has_duration()); - - // Wait until we get the next sample. We use a generous timeout of - // two seconds because we currently have a one second reap interval; - // when running perf with perf_duration of 250ms we won't notice the - // exit for up to one second. - ResourceStatistics statistics2; - Duration waited = Duration::zero(); - do { - Future<ResourceStatistics> statistics = isolator->usage(containerId); - AWAIT_READY(statistics); - - statistics2 = statistics.get(); - - ASSERT_TRUE(statistics2.has_perf()); - - if (statistics1.get().perf().timestamp() != - statistics2.perf().timestamp()) { - break; - } - - os::sleep(Milliseconds(250)); - waited += Milliseconds(250); - } while (waited < Seconds(2)); - - sleep(2); - - EXPECT_NE(statistics1.get().perf().timestamp(), - statistics2.perf().timestamp()); - - EXPECT_TRUE(statistics2.perf().has_cycles()); - EXPECT_LE(0u, statistics2.perf().cycles()); - - EXPECT_TRUE(statistics2.perf().has_task_clock()); - EXPECT_LE(0.0, statistics2.perf().task_clock()); - - AWAIT_READY(isolator->cleanup(containerId)); -} - - class SharedFilesystemIsolatorTest : public MesosTest {}; @@ -444,157 +363,6 @@ TEST_F(NamespacesPidIsolatorTest, ROOT_PidNamespace) EXPECT_EQ("sh", strings::trim(init.get())); } - - -// Username for the unprivileged user that will be created to test -// unprivileged cgroup creation. It will be removed after the tests. -// It is presumed this user does not normally exist. -const string UNPRIVILEGED_USERNAME = "mesos.test.unprivileged.user"; - - -template <typename T> -class UserCgroupIsolatorTest - : public ContainerizerTest<slave::MesosContainerizer> -{ -public: - static void SetUpTestCase() - { - ContainerizerTest<slave::MesosContainerizer>::SetUpTestCase(); - - // Remove the user in case it wasn't cleaned up from a previous - // test. - os::system("userdel -r " + UNPRIVILEGED_USERNAME + " > /dev/null"); - - ASSERT_EQ(0, os::system("useradd " + UNPRIVILEGED_USERNAME)); - } - - - static void TearDownTestCase() - { - ContainerizerTest<slave::MesosContainerizer>::TearDownTestCase(); - - ASSERT_EQ(0, os::system("userdel -r " + UNPRIVILEGED_USERNAME)); - } -}; - - -// Test all isolators that use cgroups. -typedef ::testing::Types<CgroupsPerfEventIsolatorProcess> CgroupsIsolatorTypes; - - -TYPED_TEST_CASE(UserCgroupIsolatorTest, CgroupsIsolatorTypes); - - -TYPED_TEST(UserCgroupIsolatorTest, ROOT_CGROUPS_PERF_UserCgroup) -{ - slave::Flags flags = UserCgroupIsolatorTest<TypeParam>::CreateSlaveFlags(); - flags.perf_events = "cpu-cycles"; // Needed for CgroupsPerfEventIsolator. - - Try<Isolator*> _isolator = TypeParam::create(flags); - ASSERT_SOME(_isolator); - Owned<Isolator> isolator(_isolator.get()); - - ExecutorInfo executorInfo; - executorInfo.mutable_resources()->CopyFrom( - Resources::parse("mem:1024;cpus:1").get()); // For cpu/mem isolators. - - ContainerID containerId; - containerId.set_value(UUID::random().toString()); - - ContainerConfig containerConfig; - containerConfig.mutable_executor_info()->CopyFrom(executorInfo); - containerConfig.set_directory(os::getcwd()); - containerConfig.set_user(UNPRIVILEGED_USERNAME); - - AWAIT_READY(isolator->prepare( - containerId, - containerConfig)); - - // Isolators don't provide a way to determine the cgroups they use - // so we'll inspect the cgroups for an isolated dummy process. - pid_t pid = fork(); - if (pid == 0) { - // Child just sleeps. - ::sleep(100); - - ABORT("Child process should not reach here"); - } - ASSERT_GT(pid, 0); - - AWAIT_READY(isolator->isolate(containerId, pid)); - - // Get the container's cgroups from /proc/$PID/cgroup. We're only - // interested in the cgroups that this isolator has created which we - // can do explicitly by selecting those that have the path that - // corresponds to the 'cgroups_root' slave flag. For example: - // - // $ cat /proc/pid/cgroup - // 6:blkio:/ - // 5:perf_event:/ - // 4:memory:/mesos/b7410ed8-c85b-445e-b50e-3a1698d0e18c - // 3:freezer:/ - // 2:cpuacct:/ - // 1:cpu:/ - // - // Our 'grep' will only select the 'memory' line and then 'awk' will - // output 'memory/mesos/b7410ed8-c85b-445e-b50e-3a1698d0e18c'. - Try<string> grepOut = os::shell( - "grep '" + path::join("/", flags.cgroups_root) + "' /proc/" + - stringify(pid) + "/cgroup | awk -F ':' '{print $2$3}'"); - - ASSERT_SOME(grepOut); - - // Kill the dummy child process. - ::kill(pid, SIGKILL); - int exitStatus; - EXPECT_NE(-1, ::waitpid(pid, &exitStatus, 0)); - - vector<string> cgroups = strings::tokenize(grepOut.get(), "\n"); - ASSERT_FALSE(cgroups.empty()); - - foreach (string cgroup, cgroups) { - if (!os::exists(path::join(flags.cgroups_hierarchy, cgroup)) && - strings::startsWith(cgroup, "cpuacct,cpu")) { - // An existing bug in CentOS 7.x causes 'cpuacct,cpu' cgroup to - // be under 'cpu,cpuacct'. Actively detect this here to - // work around this problem. - vector<string> parts = strings::split(cgroup, "/"); - parts[0] = "cpu,cpuacct"; - cgroup = strings::join("/", parts); - } - - // Check the user cannot manipulate the container's cgroup control - // files. - EXPECT_NE(0, os::system( - "su - " + UNPRIVILEGED_USERNAME + - " -c 'echo $$ >" + - path::join(flags.cgroups_hierarchy, cgroup, "cgroup.procs") + - "'")); - - // Check the user can create a cgroup under the container's - // cgroup. - string userCgroup = path::join(cgroup, "user"); - - EXPECT_EQ(0, os::system( - "su - " + - UNPRIVILEGED_USERNAME + - " -c 'mkdir " + - path::join(flags.cgroups_hierarchy, userCgroup) + - "'")); - - // Check the user can manipulate control files in the created - // cgroup. - EXPECT_EQ(0, os::system( - "su - " + - UNPRIVILEGED_USERNAME + - " -c 'echo $$ >" + - path::join(flags.cgroups_hierarchy, userCgroup, "cgroup.procs") + - "'")); - } - - // Clean up the container. This will also remove the nested cgroups. - AWAIT_READY(isolator->cleanup(containerId)); -} #endif // __linux__ } // namespace tests {
