Repository: hadoop Updated Branches: refs/heads/trunk e5c2fdddd -> aa461f909
YARN-7516. Add security check for trusted docker images. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa461f90 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa461f90 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa461f90 Branch: refs/heads/trunk Commit: aa461f909144fe4312c456905d9b8c37e858456f Parents: e5c2fdd Author: Billie Rinaldi <bil...@apache.org> Authored: Wed Feb 7 07:02:58 2018 -0800 Committer: Billie Rinaldi <bil...@apache.org> Committed: Wed Feb 7 07:09:45 2018 -0800 ---------------------------------------------------------------------- .../container-executor/impl/utils/docker-util.c | 92 ++++++- .../container-executor/impl/utils/docker-util.h | 3 +- .../test/utils/test_docker_util.cc | 256 +++++++++++++------ .../src/site/markdown/DockerContainers.md | 16 ++ .../src/site/markdown/yarn-service/Examples.md | 4 + 5 files changed, 293 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa461f90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index 597af35..7159374 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -74,6 +74,48 @@ static int add_param_to_command(const struct configuration *command_config, cons return ret; } +int check_trusted_image(const struct configuration *command_config, const struct configuration *conf) { + int found = 0; + int i = 0; + int ret = 0; + char *image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, command_config); + char **privileged_registry = get_configuration_values_delimiter("docker.privileged-containers.registries", CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ","); + char *registry_ptr = NULL; + if (image_name == NULL) { + ret = INVALID_DOCKER_IMAGE_NAME; + goto free_and_exit; + } + if (privileged_registry != NULL) { + for (i = 0; privileged_registry[i] != NULL; i++) { + int len = strlen(privileged_registry[i]); + if (privileged_registry[i][len - 1] != '/') { + registry_ptr = (char *) alloc_and_clear_memory(len + 2, sizeof(char)); + strncpy(registry_ptr, privileged_registry[i], len); + registry_ptr[len] = '/'; + registry_ptr[len + 1] = '\0'; + } else { + registry_ptr = strdup(privileged_registry[i]); + } + if (strncmp(image_name, registry_ptr, strlen(registry_ptr))==0) { + fprintf(ERRORFILE, "image: %s is trusted in %s registry.\n", image_name, privileged_registry[i]); + found=1; + free(registry_ptr); + break; + } + free(registry_ptr); + } + } + if (found==0) { + fprintf(ERRORFILE, "image: %s is not trusted.\n", image_name); + ret = INVALID_DOCKER_IMAGE_TRUST; + } + free(image_name); + + free_and_exit: + free(privileged_registry); + return ret; +} + static int add_param_to_command_if_allowed(const struct configuration *command_config, const struct configuration *executor_cfg, const char *key, const char *allowed_key, const char *param, @@ -100,6 +142,14 @@ static int add_param_to_command_if_allowed(const struct configuration *command_c } if (values != NULL) { + // Disable capabilities, devices if image is not trusted. + if (strcmp(key, "net") != 0) { + if (check_trusted_image(command_config, executor_cfg) != 0) { + fprintf(ERRORFILE, "Disable %s for untrusted image\n", key); + return INVALID_DOCKER_IMAGE_TRUST; + } + } + if (permitted_values != NULL) { for (i = 0; values[i] != NULL; ++i) { memset(tmp_buffer, 0, tmp_buffer_size); @@ -222,6 +272,8 @@ const char *get_docker_error_message(const int error_code) { return "Host pid namespace is disabled"; case INVALID_PID_NAMESPACE: return "Invalid pid namespace"; + case INVALID_DOCKER_IMAGE_TRUST: + return "Docker image is not trusted"; default: return "Unknown error"; } @@ -840,14 +892,22 @@ static int set_capabilities(const struct configuration *command_config, if (ret != 0) { return BUFFER_TOO_SMALL; } + ret = add_param_to_command_if_allowed(command_config, conf, "cap-add", "docker.allowed.capabilities", "--cap-add=", 1, 0, out, outlen); - if (ret != 0) { - fprintf(ERRORFILE, "Invalid docker capability requested\n"); - ret = INVALID_DOCKER_CAPABILITY; - memset(out, 0, outlen); + switch (ret) { + case 0: + break; + case INVALID_DOCKER_IMAGE_TRUST: + fprintf(ERRORFILE, "Docker capability disabled for untrusted image\n"); + ret = 0; + break; + default: + fprintf(ERRORFILE, "Invalid docker capability requested\n"); + ret = INVALID_DOCKER_CAPABILITY; + memset(out, 0, outlen); } return ret; @@ -999,6 +1059,19 @@ static int add_mounts(const struct configuration *command_config, const struct c } if (values != NULL) { + // Disable mount volumes if image is not trusted. + if (check_trusted_image(command_config, conf) != 0) { + fprintf(ERRORFILE, "Disable mount volume for untrusted image\n"); + // YARN will implicitly bind node manager local directory to + // docker image. This can create file system security holes, + // if docker container has binary to escalate privileges. + // For untrusted image, we drop mounting without reporting + // INVALID_DOCKER_MOUNT messages to allow running untrusted + // image in a sandbox. + ret = 0; + goto free_and_exit; + } + ret = normalize_mounts(permitted_ro_mounts); ret |= normalize_mounts(permitted_rw_mounts); if (ret != 0) { @@ -1100,6 +1173,12 @@ static int set_privileged(const struct configuration *command_config, const stru if (privileged_container_enabled != NULL) { if (strcmp(privileged_container_enabled, "1") == 0 || strcasecmp(privileged_container_enabled, "True") == 0) { + // Disable set privileged if image is not trusted. + if (check_trusted_image(command_config, conf) != 0) { + fprintf(ERRORFILE, "Privileged containers are disabled from untrusted source\n"); + ret = PRIVILEGED_CONTAINERS_DISABLED; + goto free_and_exit; + } ret = add_to_buffer(out, outlen, "--privileged "); if (ret != 0) { ret = BUFFER_TOO_SMALL; @@ -1251,6 +1330,11 @@ int get_docker_run_command(const char *command_file, const struct configuration launch_command = get_configuration_values_delimiter("launch-command", DOCKER_COMMAND_FILE_SECTION, &command_config, ","); + + if (check_trusted_image(&command_config, conf) != 0) { + launch_command = NULL; + } + if (launch_command != NULL) { for (i = 0; launch_command[i] != NULL; ++i) { memset(tmp_buffer, 0, tmp_buffer_size); http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa461f90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h index a14928d..8299acd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h @@ -57,7 +57,8 @@ enum docker_error_codes { INVALID_DOCKER_VOLUME_NAME, INVALID_DOCKER_VOLUME_COMMAND, PID_HOST_DISABLED, - INVALID_PID_NAMESPACE + INVALID_PID_NAMESPACE, + INVALID_DOCKER_IMAGE_TRUST }; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa461f90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc index 3651fe0..7617d2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc @@ -590,16 +590,20 @@ namespace ContainerExecutor { const int buff_len = 1024; char buff[buff_len]; int ret = 0; - std::string container_executor_cfg_contents[] = {"[docker]\n docker.privileged-containers.enabled=1", - "[docker]\n docker.privileged-containers.enabled=true", - "[docker]\n docker.privileged-containers.enabled=True", + std::string container_executor_cfg_contents[] = {"[docker]\n docker.privileged-containers.enabled=1\n docker.privileged-containers.registries=hadoop", + "[docker]\n docker.privileged-containers.enabled=true\n docker.privileged-containers.registries=hadoop", + "[docker]\n docker.privileged-containers.enabled=True\n docker.privileged-containers.registries=hadoop", "[docker]\n docker.privileged-containers.enabled=0", "[docker]\n docker.privileged-containers.enabled=false", "[docker]\n"}; std::vector<std::pair<std::string, std::string> > file_cmd_vec; std::vector<std::pair<std::string, std::string> >::const_iterator itr; file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n privileged=true", "--privileged ")); + "[docker-command-execution]\n docker-command=run\n privileged=true\n image=hadoop/image", "--privileged ")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n docker-command=run\n privileged=false\n image=hadoop/image", "")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n docker-command=run\n privileged=false\n image=nothadoop/image", "")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( "[docker-command-execution]\n docker-command=run\n privileged=false", "")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( @@ -622,6 +626,14 @@ namespace ContainerExecutor { ASSERT_EQ(0, ret); ASSERT_STREQ(itr->second.c_str(), buff); } + write_command_file("[docker-command-execution]\n docker-command=run\n privileged=true\n image=nothadoop/image"); + ret = read_config(docker_command_file.c_str(), &cmd_cfg); + if (ret != 0) { + FAIL(); + } + ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len); + ASSERT_EQ(PRIVILEGED_CONTAINERS_DISABLED, ret); + ASSERT_EQ(0, strlen(buff)); } @@ -664,15 +676,25 @@ namespace ContainerExecutor { const int buff_len = 1024; char buff[buff_len]; int ret = 0; - std::string container_executor_cfg_contents = "[docker]\n docker.allowed.capabilities=CHROOT,MKNOD"; + std::string container_executor_cfg_contents = "[docker]\n" + " docker.allowed.capabilities=CHROOT,MKNOD\n" + " docker.privileged-containers.registries=hadoop\n"; std::vector<std::pair<std::string, std::string> > file_cmd_vec; file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n cap-add=CHROOT,MKNOD", + "[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n cap-add=CHROOT,MKNOD", "--cap-drop='ALL' --cap-add='CHROOT' --cap-add='MKNOD' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n cap-add=CHROOT", "--cap-drop='ALL' --cap-add='CHROOT' ")); + "[docker-command-execution]\n docker-command=run\n image=nothadoop/docker-image\n cap-add=CHROOT,MKNOD", + "--cap-drop='ALL' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run", "--cap-drop='ALL' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n cap-add=CHROOT", + "--cap-drop='ALL' --cap-add='CHROOT' ")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n", + "--cap-drop='ALL' ")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n docker-command=run\n image=nothadoop/docker-image\n", + "--cap-drop='ALL' ")); write_container_executor_cfg(container_executor_cfg_contents); ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); @@ -691,7 +713,7 @@ namespace ContainerExecutor { ASSERT_EQ(0, ret); ASSERT_STREQ(itr->second.c_str(), buff); } - write_command_file("[docker-command-execution]\n docker-command=run\n cap-add=SETGID"); + write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n cap-add=SETGID"); ret = read_config(docker_command_file.c_str(), &cmd_cfg); if (ret != 0) { FAIL(); @@ -701,7 +723,7 @@ namespace ContainerExecutor { ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret); ASSERT_EQ(0, strlen(buff)); - container_executor_cfg_contents = "[docker]\n"; + container_executor_cfg_contents = "[docker]\n docker.privileged-containers.registries=hadoop\n"; write_container_executor_cfg(container_executor_cfg_contents); ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); if (ret != 0) { @@ -718,20 +740,22 @@ namespace ContainerExecutor { const int buff_len = 1024; char buff[buff_len]; int ret = 0; - std::string container_executor_cfg_contents = "[docker]\n docker.allowed.devices=/dev/test-device,/dev/device2"; + std::string container_executor_cfg_contents = "[docker]\n" + " docker.privileged-containers.registries=hadoop\n" + " docker.allowed.devices=/dev/test-device,/dev/device2"; std::vector<std::pair<std::string, std::string> > file_cmd_vec; file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n devices=/dev/test-device:/dev/test-device", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/test-device:/dev/test-device", "--device='/dev/test-device:/dev/test-device' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n devices=/dev/device2:/dev/device2", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/device2:/dev/device2", "--device='/dev/device2:/dev/device2' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n " - "devices=/dev/test-device:/dev/test-device,/dev/device2:/dev/device2", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n" + " devices=/dev/test-device:/dev/test-device,/dev/device2:/dev/device2", "--device='/dev/test-device:/dev/test-device' --device='/dev/device2:/dev/device2' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n", "")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image", "")); write_container_executor_cfg(container_executor_cfg_contents); ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); @@ -750,7 +774,7 @@ namespace ContainerExecutor { ASSERT_EQ(0, ret); ASSERT_STREQ(itr->second.c_str(), buff); } - write_command_file("[docker-command-execution]\n docker-command=run\n devices=/dev/device3:/dev/device3"); + write_command_file("[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n devices=/dev/test-device:/dev/test-device"); ret = read_config(docker_command_file.c_str(), &cmd_cfg); if (ret != 0) { FAIL(); @@ -760,7 +784,17 @@ namespace ContainerExecutor { ASSERT_EQ(INVALID_DOCKER_DEVICE, ret); ASSERT_EQ(0, strlen(buff)); - write_command_file("[docker-command-execution]\n docker-command=run\n devices=/dev/device1"); + write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/device3:/dev/device3"); + ret = read_config(docker_command_file.c_str(), &cmd_cfg); + if (ret != 0) { + FAIL(); + } + strcpy(buff, "test string"); + ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len); + ASSERT_EQ(INVALID_DOCKER_DEVICE, ret); + ASSERT_EQ(0, strlen(buff)); + + write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/device1"); ret = read_config(docker_command_file.c_str(), &cmd_cfg); if (ret != 0) { FAIL(); @@ -788,21 +822,32 @@ namespace ContainerExecutor { const int buff_len = 1024; char buff[buff_len]; int ret = 0; - std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/opt,/var,/usr/bin/cut,..\n " + std::string container_executor_cfg_contents = "[docker]\n docker.privileged-containers.registries=hadoop\n " + "docker.allowed.rw-mounts=/opt,/var,/usr/bin/cut,..\n " "docker.allowed.ro-mounts=/etc/passwd"; std::vector<std::pair<std::string, std::string> > file_cmd_vec; file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n rw-mounts=/var:/var", "-v '/var:/var' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/var:/var", "-v '/var:/var' ")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n rw-mounts=/var:/var", "")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/var/:/var/", "-v '/var/:/var/' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n rw-mounts=/var/:/var/", "-v '/var/:/var/' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/usr/bin/cut:/usr/bin/cut", + "-v '/usr/bin/cut:/usr/bin/cut' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr/bin/cut:/usr/bin/cut", - "-v '/usr/bin/cut:/usr/bin/cut' ")); + "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n rw-mounts=/lib:/lib", + "")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n rw-mounts=/opt:/mydisk1,/var/log/:/mydisk2", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/opt:/mydisk1,/var/log/:/mydisk2", "-v '/opt:/mydisk1' -v '/var/log/:/mydisk2' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n", "")); + "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n rw-mounts=/opt:/mydisk1,/var/log/:/mydisk2", + "")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n", "")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n", "")); write_container_executor_cfg(container_executor_cfg_contents); ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); if (ret != 0) { @@ -831,13 +876,13 @@ namespace ContainerExecutor { std::vector<std::pair<std::string, int> > bad_file_cmds_vec; bad_file_cmds_vec.push_back(std::make_pair<std::string, int>( - "[docker-command-execution]\n docker-command=run\n rw-mounts=/lib:/lib", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/lib:/lib", static_cast<int>(INVALID_DOCKER_RW_MOUNT))); bad_file_cmds_vec.push_back(std::make_pair<std::string, int>( - "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr/bin/:/usr/bin", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/usr/bin/:/usr/bin", static_cast<int>(INVALID_DOCKER_RW_MOUNT))); bad_file_cmds_vec.push_back(std::make_pair<std::string, int>( - "[docker-command-execution]\n docker-command=run\n rw-mounts=/blah:/blah", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/blah:/blah", static_cast<int>(INVALID_DOCKER_MOUNT))); std::vector<std::pair<std::string, int> >::const_iterator itr2; @@ -858,7 +903,7 @@ namespace ContainerExecutor { // verify that you can't mount any directory in the container-executor.cfg path char *ce_path = realpath("../etc/hadoop/container-executor.cfg", NULL); while (strlen(ce_path) != 0) { - std::string cmd_file_contents = "[docker-command-execution]\n docker-command=run\n rw-mounts="; + std::string cmd_file_contents = "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts="; cmd_file_contents.append(ce_path).append(":").append("/etc/hadoop"); memset(buff, 0, buff_len); write_command_file(cmd_file_contents); @@ -877,6 +922,8 @@ namespace ContainerExecutor { } free(ce_path); + // For untrusted image, container add_rw_mounts will pass through + // without mounting or report error code. container_executor_cfg_contents = "[docker]\n"; write_container_executor_cfg(container_executor_cfg_contents); ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); @@ -885,8 +932,8 @@ namespace ContainerExecutor { } strcpy(buff, "test string"); ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len); - ASSERT_EQ(INVALID_DOCKER_RW_MOUNT, ret); - ASSERT_EQ(0, strlen(buff)); + ASSERT_EQ(0, ret); + ASSERT_EQ(11, strlen(buff)); } TEST_F(TestDockerUtil, test_add_ro_mounts) { @@ -895,31 +942,36 @@ namespace ContainerExecutor { char buff[buff_len]; int ret = 0; - std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/home/,/var,/usr/bin/cut,..\n " + std::string container_executor_cfg_contents = "[docker]\n docker.privileged-containers.registries=hadoop\n " + "docker.allowed.rw-mounts=/home/,/var,/usr/bin/cut,..\n " "docker.allowed.ro-mounts=/etc/passwd,/etc/group"; std::vector<std::pair<std::string, std::string> > file_cmd_vec; file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/var:/var", "-v '/var:/var:ro' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/var:/var", "-v '/var:/var:ro' ")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n ro-mounts=/var:/var", "")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/var/:/var/", "-v '/var/:/var/:ro' ")); + "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n ro-mounts=/etc:/etc", "")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/home:/home", "-v '/home:/home:ro' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/var/:/var/", "-v '/var/:/var/:ro' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/home/:/home", "-v '/home/:/home:ro' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home:/home", "-v '/home:/home:ro' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr/bin/cut:/usr/bin/cut", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home/:/home", "-v '/home/:/home:ro' ")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/usr/bin/cut:/usr/bin/cut", "-v '/usr/bin/cut:/usr/bin/cut:ro' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/etc/group:/etc/group", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/etc/group:/etc/group", "-v '/etc/group:/etc/group:ro' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/etc/passwd:/etc/passwd", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/etc/passwd:/etc/passwd", "-v '/etc/passwd:/etc/passwd:ro' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/var/log:/mydisk1,/etc/passwd:/etc/passwd", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/var/log:/mydisk1,/etc/passwd:/etc/passwd", "-v '/var/log:/mydisk1:ro' -v '/etc/passwd:/etc/passwd:ro' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n", "")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n", "")); write_container_executor_cfg(container_executor_cfg_contents); ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); if (ret != 0) { @@ -948,10 +1000,10 @@ namespace ContainerExecutor { std::vector<std::pair<std::string, int> > bad_file_cmds_vec; bad_file_cmds_vec.push_back(std::make_pair<std::string, int>( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/etc:/etc", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/etc:/etc", static_cast<int>(INVALID_DOCKER_RO_MOUNT))); bad_file_cmds_vec.push_back(std::make_pair<std::string, int>( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/blah:/blah", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/blah:/blah", static_cast<int>(INVALID_DOCKER_MOUNT))); std::vector<std::pair<std::string, int> >::const_iterator itr2; @@ -969,13 +1021,13 @@ namespace ContainerExecutor { ASSERT_STREQ("", buff); } - container_executor_cfg_contents = "[docker]\n"; + container_executor_cfg_contents = "[docker]\n docker.privileged-containers.registries=hadoop\n"; write_container_executor_cfg(container_executor_cfg_contents); ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); if (ret != 0) { FAIL(); } - write_command_file("[docker-command-execution]\n docker-command=run\n ro-mounts=/home:/home"); + write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home:/home"); strcpy(buff, "test string"); ret = add_ro_mounts(&cmd_cfg, &container_cfg, buff, buff_len); ASSERT_EQ(INVALID_DOCKER_RO_MOUNT, ret); @@ -987,7 +1039,7 @@ namespace ContainerExecutor { std::string container_executor_contents = "[docker]\n docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n" " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n " " docker.privileged-containers.enabled=1\n docker.allowed.capabilities=CHOWN,SETUID\n" - " docker.allowed.devices=/dev/test"; + " docker.allowed.devices=/dev/test\n docker.privileged-containers.registries=hadoop\n"; write_file(container_executor_cfg_file, container_executor_contents); int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg); if (ret != 0) { @@ -1001,53 +1053,74 @@ namespace ContainerExecutor { std::vector<std::pair<std::string, std::string> > file_cmd_vec; file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test", - "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' ")); + "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test", + "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'hadoop/docker-image' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( - "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n" + "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=test", + "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'nothadoop/docker-image' ")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n" " launch-command=bash,test_script.sh,arg1,arg2", - "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); + "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'hadoop/docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" " launch-command=bash,test_script.sh,arg1,arg2", "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'" - " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' " + " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash' " "'test_script.sh' 'arg1' 'arg2' ")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=test\n hostname=host-id\n" + " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" + " network=bridge\n" + " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" + " launch-command=bash,test_script.sh,arg1,arg2", + "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm" + " --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --hostname='host-id' 'nothadoop/docker-image' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" " launch-command=bash,test_script.sh,arg1,arg2", "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' " - "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'" + "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash'" " 'test_script.sh' 'arg1' 'arg2' ")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=test\n hostname=host-id\n" + " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" + " network=bridge\n net=bridge\n" + " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" + " launch-command=bash,test_script.sh,arg1,arg2", + "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge'" + " --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --hostname='host-id' 'nothadoop/docker-image' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" " launch-command=bash,test_script.sh,arg1,arg2", "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' " - "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' " + "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' " "'bash' 'test_script.sh' 'arg1' 'arg2' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n" @@ -1055,25 +1128,42 @@ namespace ContainerExecutor { "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' " "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --group-add '1000' --group-add '1001' " - "--device='/dev/test:/dev/test' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); + "--device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " network=bridge\n net=bridge\n" + " detach=true\n rm=true\n group-add=1000,1001\n" + " launch-command=bash,test_script.sh,arg1,arg2", + "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' --cap-drop='ALL' " + "--hostname='host-id' --group-add '1000' --group-add '1001' " + "'docker-image' ")); std::vector<std::pair<std::string, int> > bad_file_cmd_vec; bad_file_cmd_vec.push_back(std::make_pair<std::string, int>( - "[docker-command-execution]\n docker-command=run\n image=docker-image\n user=test", + "[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n user=test", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME))); bad_file_cmd_vec.push_back(std::make_pair<std::string, int>( "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n user=test\n", static_cast<int>(INVALID_DOCKER_IMAGE_NAME))); bad_file_cmd_vec.push_back(std::make_pair<std::string, int>( - "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n", + "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n", static_cast<int>(INVALID_DOCKER_USER_NAME))); + bad_file_cmd_vec.push_back(std::make_pair<std::string, int>( + "[docker-command-execution]\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=test\n hostname=host-id\n" + " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" + " network=bridge\n net=bridge\n privileged=true\n" + " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n" + " launch-command=bash,test_script.sh,arg1,arg2", + PRIVILEGED_CONTAINERS_DISABLED)); // invalid rw mount bad_file_cmd_vec.push_back(std::make_pair<std::string, int>( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/var/log:/var/log\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -1083,7 +1173,7 @@ namespace ContainerExecutor { // invalid ro mount bad_file_cmd_vec.push_back(std::make_pair<std::string, int>( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/bin:/bin,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -1093,7 +1183,7 @@ namespace ContainerExecutor { // invalid capability bad_file_cmd_vec.push_back(std::make_pair<std::string, int>( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID,SETGID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -1103,7 +1193,7 @@ namespace ContainerExecutor { // invalid device bad_file_cmd_vec.push_back(std::make_pair<std::string, int>( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/dev1:/dev/dev1\n privileged=true\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -1113,7 +1203,7 @@ namespace ContainerExecutor { // invalid network bad_file_cmd_vec.push_back(std::make_pair<std::string, int>( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n privileged=true\n net=host\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -1126,11 +1216,13 @@ namespace ContainerExecutor { TEST_F(TestDockerUtil, test_docker_run_no_privileged) { std::string container_executor_contents[] = {"[docker]\n docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n" - " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n " + " docker.privileged-containers.registries=hadoop\n" + " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n" " docker.allowed.capabilities=CHOWN,SETUID\n" " docker.allowed.devices=/dev/test", "[docker]\n docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n" - " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n " + " docker.privileged-containers.registries=hadoop\n" + " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n" " docker.allowed.capabilities=CHOWN,SETUID\n" " privileged=0\n" " docker.allowed.devices=/dev/test"}; @@ -1153,36 +1245,54 @@ namespace ContainerExecutor { file_cmd_vec.push_back(std::make_pair<std::string, std::string>( "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n" " user=test\n launch-command=bash,test_script.sh,arg1,arg2", - "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); + "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" " launch-command=bash,test_script.sh,arg1,arg2", "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'" - " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' " + " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash' " "'test_script.sh' 'arg1' 'arg2' ")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=test\n hostname=host-id\n" + " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" + " network=bridge\n" + " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" + " launch-command=bash,test_script.sh,arg1,arg2", + "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm" + " --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --hostname='host-id' 'nothadoop/docker-image' ")); file_cmd_vec.push_back(std::make_pair<std::string, std::string>( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" " launch-command=bash,test_script.sh,arg1,arg2", "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' " - "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'" + "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash'" " 'test_script.sh' 'arg1' 'arg2' ")); + file_cmd_vec.push_back(std::make_pair<std::string, std::string>( + "[docker-command-execution]\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=test\n hostname=host-id\n" + " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" + " network=bridge\n net=bridge\n" + " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" + " launch-command=bash,test_script.sh,arg1,arg2", + "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge'" + " --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --hostname='host-id' 'nothadoop/docker-image' ")); std::vector<std::pair<std::string, int> > bad_file_cmd_vec; bad_file_cmd_vec.push_back(std::make_pair<std::string, int>( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa461f90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index 09e094f..442ce09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -206,6 +206,7 @@ are allowed. It contains the following properties: | `docker.allowed.rw-mounts` | Comma separated directories that containers are allowed to mount in read-write mode. By default, no directories are allowed to mounted. | | `docker.host-pid-namespace.enabled` | Set to "true" or "false" to enable or disable using the host's PID namespace. Default value is "false". | | `docker.privileged-containers.enabled` | Set to "true" or "false" to enable or disable launching privileged containers. Default value is "false". | +| `docker.privileged-containers.registries` | Comma separated list of trusted docker registries for running trusted privileged docker containers. By default, no registries are defined. | Please note that if you wish to run Docker containers that require access to the YARN local directories, you must add them to the docker.allowed.rw-mounts list. @@ -226,6 +227,8 @@ Part of a container-executor.cfg which allows Docker containers to be launched i yarn.nodemanager.linux-container-executor.group=yarn [docker] module.enabled=true + docker.privileged-containers.enabled=true + docker.privileged-containers.registries=centos docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE docker.allowed.networks=bridge,host,none docker.allowed.ro-mounts=/sys/fs/cgroup @@ -361,6 +364,19 @@ the environment variable would be set to "/sys/fs/cgroup:/sys/fs/cgroup:ro". The destination path is not restricted, "/sys/fs/cgroup:/cgroup:ro" would also be valid given the example admin whitelist. +Privileged Container Security Consideration +------------------------------------------- + +Privileged docker container can interact with host system devices. This can cause harm to host operating system without proper care. In order to mitigate risk of allowing privileged container to run on Hadoop cluster, we implemented a controlled process to sandbox unauthorized privileged docker images. + +The default behavior is disallow any privileged docker containers. When `docker.privileged-containers.enabled` is set to enabled, docker image can run with root privileges in the docker container, but access to host level devices are disabled. This allows developer and tester to run docker images from internet without causing harm to host operating system. + +When docker images have been certified by developers and testers to be trustworthy. The trusted image can be promoted to trusted docker registry. System administrator can define `docker.privileged-containers.registries`, and setup private docker registry server to promote trusted images. + +Trusted images are allowed to mount external devices such as HDFS via NFS gateway, or host level Hadoop configuration. If system administrators allow writing to external volumes using `docker.allow.rw-mounts directive`, privileged docker container can have full control of host level files in the predefined volumes. + +For [YARN Service HTTPD example](./yarn-service/Examples.html), container-executor.cfg must define centos docker registry to be trusted for the example to run. + Connecting to a Secure Docker Repository ---------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa461f90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md index 9c5d176..4163635 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md @@ -19,6 +19,10 @@ This document describes some example service definitions (`Yarnfile`). <!-- MACRO{toc|fromDepth=0|toDepth=3} --> ## Apache web server - httpd (with registry DNS) + +For this example to work, centos/httpd-24-centos7 image must be included in `docker.privileged-containers.registries`. +For server side configuration, please refer to [Running Applications in Docker Containers](../DockerContainers.html) document. + Below is the `Yarnfile` for a service called `httpd-service` with two `httpd` instances. There is also an httpd proxy instance (httpd-proxy-0) that proxies between the other two httpd instances (httpd-0 and httpd-1). --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
