YARN-6033. Add support for sections in container-executor configuration file. (Varun Vasudev via wandga)
Change-Id: Ibc6d2a959debe5d8ff2b51504149742449d1f1da Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec694145 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec694145 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec694145 Branch: refs/heads/YARN-1011 Commit: ec694145cf9c0ade7606813871ca2a4a371def8e Parents: 63cfcb9 Author: Wangda Tan <wan...@apache.org> Authored: Wed Aug 9 10:51:29 2017 -0700 Committer: Wangda Tan <wan...@apache.org> Committed: Wed Aug 9 10:51:29 2017 -0700 ---------------------------------------------------------------------- .../hadoop-yarn-server-nodemanager/pom.xml | 38 ++ .../src/CMakeLists.txt | 22 + .../container-executor/impl/configuration.c | 672 +++++++++++++------ .../container-executor/impl/configuration.h | 182 +++-- .../impl/container-executor.c | 39 +- .../impl/container-executor.h | 52 +- .../container-executor/impl/get_executable.c | 1 + .../main/native/container-executor/impl/main.c | 17 +- .../main/native/container-executor/impl/util.c | 134 ++++ .../main/native/container-executor/impl/util.h | 115 ++++ .../test-configurations/configuration-1.cfg | 31 + .../test-configurations/configuration-2.cfg | 28 + .../test/test-configurations/old-config.cfg | 25 + .../test/test-container-executor.c | 15 +- .../test/test_configuration.cc | 432 ++++++++++++ .../native/container-executor/test/test_main.cc | 29 + .../native/container-executor/test/test_util.cc | 138 ++++ 17 files changed, 1649 insertions(+), 321 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 28ee0d9..a50a769 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -215,6 +215,44 @@ <results>${project.build.directory}/native-results</results> </configuration> </execution> + <execution> + <id>cetest</id> + <goals><goal>cmake-test</goal></goals> + <phase>test</phase> + <configuration> + <!-- this should match the xml name without the TEST-part down below --> + <testName>cetest</testName> + <workingDirectory>${project.build.directory}/native/test</workingDirectory> + <source>${basedir}/src</source> + <binary>${project.build.directory}/native/test/cetest</binary> + <args> + <arg>--gtest_filter=-Perf.</arg> + <arg>--gtest_output=xml:${project.build.directory}/surefire-reports/TEST-cetest.xml</arg> + </args> + <results>${project.build.directory}/surefire-reports</results> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>make</id> + <phase>compile</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <copy todir="${project.build.directory}/native/test/" + overwrite="true"> + <fileset dir="${basedir}/src/main/native/container-executor/resources/test" /> + </copy> + </target> + </configuration> + </execution> </executions> </plugin> </plugins> http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt index 5b52536..100d7ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt @@ -19,6 +19,9 @@ cmake_minimum_required(VERSION 2.6 FATAL_ERROR) list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../../../hadoop-common-project/hadoop-common) include(HadoopCommon) +# Set gtest path +set(GTEST_SRC_DIR ${CMAKE_SOURCE_DIR}/../../../../../hadoop-common-project/hadoop-common/src/main/native/gtest) + # determine if container-executor.conf.dir is an absolute # path in case the OS we're compiling on doesn't have # a hook in get_executable. We'll use this define @@ -80,12 +83,20 @@ endfunction() include_directories( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_BINARY_DIR} + ${GTEST_SRC_DIR}/include main/native/container-executor main/native/container-executor/impl ) +# add gtest as system library to suppress gcc warnings +include_directories(SYSTEM ${GTEST_SRC_DIR}/include) + configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h) +add_library(gtest ${GTEST_SRC_DIR}/gtest-all.cc) +set_target_properties(gtest PROPERTIES COMPILE_FLAGS "-w") + add_library(container + main/native/container-executor/impl/util.c main/native/container-executor/impl/configuration.c main/native/container-executor/impl/container-executor.c main/native/container-executor/impl/get_executable.c @@ -95,9 +106,11 @@ add_library(container add_executable(container-executor main/native/container-executor/impl/main.c ) + target_link_libraries(container-executor container ) + output_directory(container-executor target/usr/local/bin) add_executable(test-container-executor @@ -107,3 +120,12 @@ target_link_libraries(test-container-executor container ${EXTRA_LIBS} ) output_directory(test-container-executor target/usr/local/bin) + +# unit tests for container executor +add_executable(cetest + main/native/container-executor/impl/util.c + main/native/container-executor/test/test_configuration.cc + main/native/container-executor/test/test_main.cc + main/native/container-executor/test/test_util.cc) +target_link_libraries(cetest gtest) +output_directory(cetest test) http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c index a6d7a9c..12dbc4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c @@ -20,35 +20,55 @@ #include <libgen.h> #include "configuration.h" -#include "container-executor.h" +#include "util.h" +#define __STDC_FORMAT_MACROS #include <inttypes.h> #include <errno.h> #include <unistd.h> -#include <stdio.h> #include <stdlib.h> #include <string.h> #include <sys/stat.h> -#include <sys/types.h> -#include <limits.h> -#include <ctype.h> #define MAX_SIZE 10 +static const char COMMENT_BEGIN_CHAR = '#'; +static const char SECTION_LINE_BEGIN_CHAR = '['; +static const char SECTION_LINE_END_CHAR = ']'; + +//clean up method for freeing section +void free_section(struct section *section) { + int i = 0; + for (i = 0; i < section->size; i++) { + if (section->kv_pairs[i]->key != NULL) { + free((void *) section->kv_pairs[i]->key); + } + if (section->kv_pairs[i]->value != NULL) { + free((void *) section->kv_pairs[i]->value); + } + free(section->kv_pairs[i]); + } + if (section->kv_pairs) { + free(section->kv_pairs); + section->kv_pairs = NULL; + } + if (section->name) { + free(section->name); + section->name = NULL; + } + section->size = 0; +} + //clean up method for freeing configuration -void free_configurations(struct configuration *cfg) { +void free_configuration(struct configuration *cfg) { int i = 0; for (i = 0; i < cfg->size; i++) { - if (cfg->confdetails[i]->key != NULL) { - free((void *)cfg->confdetails[i]->key); + if (cfg->sections[i] != NULL) { + free_section(cfg->sections[i]); } - if (cfg->confdetails[i]->value != NULL) { - free((void *)cfg->confdetails[i]->value); - } - free(cfg->confdetails[i]); } - if (cfg->size > 0) { - free(cfg->confdetails); + if (cfg->sections) { + free(cfg->sections); } cfg->size = 0; } @@ -65,13 +85,13 @@ static int is_only_root_writable(const char *file) { } if (file_stat.st_uid != 0) { fprintf(ERRORFILE, "File %s must be owned by root, but is owned by %" PRId64 "\n", - file, (int64_t)file_stat.st_uid); + file, (int64_t) file_stat.st_uid); return 0; } if ((file_stat.st_mode & (S_IWGRP | S_IWOTH)) != 0) { fprintf(ERRORFILE, - "File %s must not be world or group writable, but is %03lo\n", - file, (unsigned long)file_stat.st_mode & (~S_IFMT)); + "File %s must not be world or group writable, but is %03lo\n", + file, (unsigned long) file_stat.st_mode & (~S_IFMT)); return 0; } return 1; @@ -82,9 +102,9 @@ static int is_only_root_writable(const char *file) { * * NOTE: relative path names are resolved relative to the second argument not getwd(3) */ -char *resolve_config_path(const char* file_name, const char *root) { +char *resolve_config_path(const char *file_name, const char *root) { const char *real_fname = NULL; - char buffer[EXECUTOR_PATH_MAX*2 + 1]; + char buffer[EXECUTOR_PATH_MAX * 2 + 1]; if (file_name[0] == '/') { real_fname = file_name; @@ -96,7 +116,7 @@ char *resolve_config_path(const char* file_name, const char *root) { #ifdef HAVE_CANONICALIZE_FILE_NAME char * ret = (real_fname == NULL) ? NULL : canonicalize_file_name(real_fname); #else - char * ret = (real_fname == NULL) ? NULL : realpath(real_fname, NULL); + char *ret = (real_fname == NULL) ? NULL : realpath(real_fname, NULL); #endif #ifdef DEBUG fprintf(stderr,"ret = %s\n", ret); @@ -112,10 +132,19 @@ char *resolve_config_path(const char* file_name, const char *root) { * configuration and potentially cause damage. * returns 0 if permissions are ok */ -int check_configuration_permissions(const char* file_name) { +int check_configuration_permissions(const char *file_name) { + if (!file_name) { + return -1; + } + // copy the input so that we can modify it with dirname - char* dir = strdup(file_name); - char* buffer = dir; + char *dir = strdup(file_name); + if (!dir) { + fprintf(stderr, "Failed to make a copy of filename in %s.\n", __func__); + return -1; + } + + char *buffer = dir; do { if (!is_only_root_writable(dir)) { free(buffer); @@ -128,167 +157,396 @@ int check_configuration_permissions(const char* file_name) { } /** - * Trim whitespace from beginning and end. -*/ -char* trim(char* input) -{ - char *val_begin; - char *val_end; - char *ret; - - if (input == NULL) { - return NULL; + * Read a line from the the config file and return it without the newline. + * The caller must free the memory allocated. + */ +static char *read_config_line(FILE *conf_file) { + char *line = NULL; + size_t linesize = 100000; + ssize_t size_read = 0; + size_t eol = 0; + + line = (char *) malloc(linesize); + if (line == NULL) { + fprintf(ERRORFILE, "malloc failed while reading configuration file.\n"); + exit(OUT_OF_MEMORY); + } + size_read = getline(&line, &linesize, conf_file); + + //feof returns true only after we read past EOF. + //so a file with no new line, at last can reach this place + //if size_read returns negative check for eof condition + if (size_read == -1) { + free(line); + line = NULL; + if (!feof(conf_file)) { + fprintf(ERRORFILE, "Line read returned -1 without eof\n"); + exit(INVALID_CONFIG_FILE); + } + } else { + eol = strlen(line) - 1; + if (line[eol] == '\n') { + //trim the ending new line + line[eol] = '\0'; } + } + return line; +} - val_begin = input; - val_end = input + strlen(input); +/** + * Return if the given line is a comment line. + * + * @param line the line to check + * + * @return 1 if the line is a comment line, 0 otherwise + */ +static int is_comment_line(const char *line) { + if (line != NULL) { + return (line[0] == COMMENT_BEGIN_CHAR); + } + return 0; +} - while (val_begin < val_end && isspace(*val_begin)) - val_begin++; - while (val_end > val_begin && isspace(*(val_end - 1))) - val_end--; +/** + * Return if the given line is a section start line. + * + * @param line the line to check + * + * @return 1 if the line is a section start line, 0 otherwise + */ +static int is_section_start_line(const char *line) { + size_t len = 0; + if (line != NULL) { + len = strlen(line) - 1; + return (line[0] == SECTION_LINE_BEGIN_CHAR + && line[len] == SECTION_LINE_END_CHAR); + } + return 0; +} - ret = (char *) malloc( - sizeof(char) * (val_end - val_begin + 1)); - if (ret == NULL) { - fprintf(ERRORFILE, "Allocation error\n"); +/** + * Return the name of the section from the given section start line. The + * caller must free the memory used. + * + * @param line the line to extract the section name from + * + * @return string with the name of the section, NULL otherwise + */ +static char *get_section_name(const char *line) { + char *name = NULL; + size_t len; + + if (is_section_start_line(line)) { + // length of the name is the line - 2(to account for '[' and ']') + len = strlen(line) - 2; + name = (char *) malloc(len + 1); + if (name == NULL) { + fprintf(ERRORFILE, "malloc failed while reading section name.\n"); exit(OUT_OF_MEMORY); } - - strncpy(ret, val_begin, val_end - val_begin); - ret[val_end - val_begin] = '\0'; - return ret; + strncpy(name, line + sizeof(char), len); + name[len] = '\0'; + } + return name; } -void read_config(const char* file_name, struct configuration *cfg) { - FILE *conf_file; - char *line; +/** + * Read an entry for the section from the line. Function returns 0 if an entry + * was found, non-zero otherwise. Return values less than 0 indicate an error + * with the config file. + * + * @param line the line to read the entry from + * @param section the struct to read the entry into + * + * @return 0 if an entry was found + * <0 for config file errors + * >0 for issues such as empty line + * + */ +static int read_section_entry(const char *line, struct section *section) { char *equaltok; char *temp_equaltok; - size_t linesize = 1000; - int size_read = 0; - - if (file_name == NULL) { - fprintf(ERRORFILE, "Null configuration filename passed in\n"); - exit(INVALID_CONFIG_FILE); + const char *splitter = "="; + char *buffer; + size_t len = 0; + if (line == NULL || section == NULL) { + fprintf(ERRORFILE, "NULL params passed to read_section_entry"); + return -1; + } + len = strlen(line); + if (len == 0) { + return 1; + } + if ((section->size) % MAX_SIZE == 0) { + section->kv_pairs = (struct kv_pair **) realloc( + section->kv_pairs, + sizeof(struct kv_pair *) * (MAX_SIZE + section->size)); + if (section->kv_pairs == NULL) { + fprintf(ERRORFILE, + "Failed re-allocating memory for configuration items\n"); + exit(OUT_OF_MEMORY); + } } - #ifdef DEBUG - fprintf(LOGFILE, "read_config :Conf file name is : %s \n", file_name); - #endif + buffer = strdup(line); + if (!buffer) { + fprintf(ERRORFILE, "Failed to allocating memory for line, %s\n", __func__); + exit(OUT_OF_MEMORY); + } - //allocate space for ten configuration items. - cfg->confdetails = (struct confentry **) malloc(sizeof(struct confentry *) - * MAX_SIZE); - cfg->size = 0; - conf_file = fopen(file_name, "r"); - if (conf_file == NULL) { - fprintf(ERRORFILE, "Invalid conf file provided : %s \n", file_name); + //tokenize first to get key and list of values. + //if no equals is found ignore this line, can be an empty line also + equaltok = strtok_r(buffer, splitter, &temp_equaltok); + if (equaltok == NULL) { + fprintf(ERRORFILE, "Error with line '%s', no '=' found\n", buffer); exit(INVALID_CONFIG_FILE); } - while(!feof(conf_file)) { - line = (char *) malloc(linesize); - if(line == NULL) { - fprintf(ERRORFILE, "malloc failed while reading configuration file.\n"); - exit(OUT_OF_MEMORY); + section->kv_pairs[section->size] = (struct kv_pair *) malloc( + sizeof(struct kv_pair)); + if (section->kv_pairs[section->size] == NULL) { + fprintf(ERRORFILE, "Failed allocating memory for single section item\n"); + exit(OUT_OF_MEMORY); + } + memset(section->kv_pairs[section->size], 0, + sizeof(struct kv_pair)); + section->kv_pairs[section->size]->key = trim(equaltok); + + equaltok = strtok_r(NULL, splitter, &temp_equaltok); + if (equaltok == NULL) { + // this can happen because no value was set + // e.g. banned.users=#this is a comment + int has_values = 1; + if (strstr(line, splitter) == NULL) { + fprintf(ERRORFILE, "configuration tokenization failed, error with line %s\n", line); + has_values = 0; } - size_read = getline(&line,&linesize,conf_file); - //feof returns true only after we read past EOF. - //so a file with no new line, at last can reach this place - //if size_read returns negative check for eof condition - if (size_read == -1) { - free(line); - if(!feof(conf_file)){ - exit(INVALID_CONFIG_FILE); - } else { - break; - } - } - int eol = strlen(line) - 1; - if(line[eol] == '\n') { - //trim the ending new line - line[eol] = '\0'; + // It is not a valid line, free memory. + free((void *) section->kv_pairs[section->size]->key); + free((void *) section->kv_pairs[section->size]); + section->kv_pairs[section->size] = NULL; + free(buffer); + + // Return -1 when no values + if (!has_values) { + return -1; } - //comment line - if(line[0] == '#') { - free(line); - continue; + + // Return 2 for comments + return 2; + } + +#ifdef DEBUG + fprintf(LOGFILE, "read_config : Adding conf value : %s \n", equaltok); +#endif + + section->kv_pairs[section->size]->value = trim(equaltok); + section->size++; + free(buffer); + return 0; +} + +/** + * Remove any trailing comment from the supplied line. Function modifies the + * argument provided. + * + * @param line the line from which to remove the comment + */ +static void trim_comment(char *line) { + char *begin_comment = NULL; + if (line != NULL) { + begin_comment = strchr(line, COMMENT_BEGIN_CHAR); + if (begin_comment != NULL) { + *begin_comment = '\0'; } - //tokenize first to get key and list of values. - //if no equals is found ignore this line, can be an empty line also - equaltok = strtok_r(line, "=", &temp_equaltok); - if(equaltok == NULL) { + } +} + +/** + * Allocate a section struct and initialize it. The memory must be freed by + * the caller. Function calls exit if any error occurs. + * + * @return pointer to the allocated section struct + * + */ +static struct section *allocate_section() { + struct section *section = (struct section *) malloc(sizeof(struct section)); + if (section == NULL) { + fprintf(ERRORFILE, "malloc failed while allocating section.\n"); + exit(OUT_OF_MEMORY); + } + section->name = NULL; + section->kv_pairs = NULL; + section->size = 0; + return section; +} + +/** + * Populate the given section struct with fields from the config file. + * + * @param conf_file the file to read from + * @param section pointer to the section struct to populate + * + */ +static void populate_section_fields(FILE *conf_file, struct section *section) { + char *line; + long int offset = 0; + while (!feof(conf_file)) { + offset = ftell(conf_file); + line = read_config_line(conf_file); + if (line != NULL) { + if (!is_comment_line(line)) { + trim_comment(line); + if (!is_section_start_line(line)) { + if (section->name != NULL) { + if (read_section_entry(line, section) < 0) { + fprintf(ERRORFILE, "Error parsing line %s", line); + exit(INVALID_CONFIG_FILE); + } + } else { + fprintf(ERRORFILE, "Line '%s' doesn't belong to a section\n", + line); + exit(INVALID_CONFIG_FILE); + } + } else { + if (section->name == NULL) { + section->name = get_section_name(line); + if (strlen(section->name) == 0) { + fprintf(ERRORFILE, "Empty section name"); + exit(INVALID_CONFIG_FILE); + } + } else { + // we've reached the next section + fseek(conf_file, offset, SEEK_SET); + free(line); + return; + } + } + } free(line); - continue; - } - cfg->confdetails[cfg->size] = (struct confentry *) malloc( - sizeof(struct confentry)); - if(cfg->confdetails[cfg->size] == NULL) { - fprintf(LOGFILE, - "Failed allocating memory for single configuration item\n"); - goto cleanup; } + } +} - #ifdef DEBUG - fprintf(LOGFILE, "read_config : Adding conf key : %s \n", equaltok); - #endif +/** + * Read the section current section from the conf file. Section start is + * marked by lines of the form '[section-name]' and continue till the next + * section. + */ +static struct section *read_section(FILE *conf_file) { + struct section *section = allocate_section(); + populate_section_fields(conf_file, section); + if (section->name == NULL) { + free_section(section); + section = NULL; + } + return section; +} + +/** + * Merge two sections and free the second one after the merge, if desired. + * @param section1 the first section + * @param section2 the second section + * @param free_second_section free the second section if set + */ +static void merge_sections(struct section *section1, struct section *section2, const int free_second_section) { + int i = 0; + section1->kv_pairs = (struct kv_pair **) realloc( + section1->kv_pairs, + sizeof(struct kv_pair *) * (section1->size + section2->size)); + if (section1->kv_pairs == NULL) { + fprintf(ERRORFILE, + "Failed re-allocating memory for configuration items\n"); + exit(OUT_OF_MEMORY); + } + for (i = 0; i < section2->size; ++i) { + section1->kv_pairs[section1->size + i] = section2->kv_pairs[i]; + } + section1->size += section2->size; + if (free_second_section) { + free(section2->name); + memset(section2, 0, sizeof(*section2)); + free(section2); + } +} - memset(cfg->confdetails[cfg->size], 0, sizeof(struct confentry)); - cfg->confdetails[cfg->size]->key = trim(equaltok); +int read_config(const char *file_path, struct configuration *cfg) { + FILE *conf_file; - equaltok = strtok_r(NULL, "=", &temp_equaltok); - if (equaltok == NULL) { - fprintf(LOGFILE, "configuration tokenization failed \n"); - goto cleanup; - } - //means value is commented so don't store the key - if(equaltok[0] == '#') { - free(line); - free((void *)cfg->confdetails[cfg->size]->key); - free(cfg->confdetails[cfg->size]); - continue; + if (file_path == NULL) { + fprintf(ERRORFILE, "Null configuration filename passed in\n"); + return INVALID_CONFIG_FILE; + } + +#ifdef DEBUG + fprintf(LOGFILE, "read_config :Conf file name is : %s \n", file_path); +#endif + + cfg->size = 0; + conf_file = fopen(file_path, "r"); + if (conf_file == NULL) { + fprintf(ERRORFILE, "Invalid conf file provided, unable to open file" + " : %s \n", file_path); + return (INVALID_CONFIG_FILE); + } + + cfg->sections = (struct section **) malloc( + sizeof(struct section *) * MAX_SIZE); + if (!cfg->sections) { + fprintf(ERRORFILE, + "Failed to allocate memory for configuration sections\n"); + exit(OUT_OF_MEMORY); + } + + // populate any entries in the older format(no sections) + cfg->sections[cfg->size] = allocate_section(); + cfg->sections[cfg->size]->name = strdup(""); + populate_section_fields(conf_file, cfg->sections[cfg->size]); + if (cfg->sections[cfg->size]) { + if (cfg->sections[cfg->size]->size) { + cfg->size++; + } else { + free_section(cfg->sections[cfg->size]); } + } - #ifdef DEBUG - fprintf(LOGFILE, "read_config : Adding conf value : %s \n", equaltok); - #endif - - cfg->confdetails[cfg->size]->value = trim(equaltok); - if((cfg->size + 1) % MAX_SIZE == 0) { - cfg->confdetails = (struct confentry **) realloc(cfg->confdetails, - sizeof(struct confentry **) * (MAX_SIZE + cfg->size)); - if (cfg->confdetails == NULL) { - fprintf(LOGFILE, - "Failed re-allocating memory for configuration items\n"); - goto cleanup; + // populate entries in the sections format + while (!feof(conf_file)) { + cfg->sections[cfg->size] = NULL; + struct section *new_section = read_section(conf_file); + if (new_section != NULL) { + struct section *existing_section = + get_configuration_section(new_section->name, cfg); + if (existing_section != NULL) { + merge_sections((struct section *) existing_section, new_section, 1); + } else { + cfg->sections[cfg->size] = new_section; } } - if(cfg->confdetails[cfg->size]) { - cfg->size++; - } - free(line); + // Check if we need to expand memory for sections. + if (cfg->sections[cfg->size]) { + if ((cfg->size + 1) % MAX_SIZE == 0) { + cfg->sections = (struct section **) realloc(cfg->sections, + sizeof(struct sections *) * (MAX_SIZE + cfg->size)); + if (cfg->sections == NULL) { + fprintf(ERRORFILE, + "Failed re-allocating memory for configuration items\n"); + exit(OUT_OF_MEMORY); + } + } + cfg->size++; + } } - //close the file fclose(conf_file); if (cfg->size == 0) { - fprintf(ERRORFILE, "Invalid configuration provided in %s\n", file_name); - exit(INVALID_CONFIG_FILE); - } - - //clean up allocated file name - return; - //free spaces alloced. - cleanup: - if (line != NULL) { - free(line); + free_configuration(cfg); + fprintf(ERRORFILE, "Invalid configuration provided in %s\n", file_path); + return INVALID_CONFIG_FILE; } - fclose(conf_file); - free_configurations(cfg); - return; + return 0; } /* @@ -297,11 +555,14 @@ void read_config(const char* file_name, struct configuration *cfg) { * array, next time onwards used the populated array. * */ -char * get_value(const char* key, struct configuration *cfg) { +char *get_section_value(const char *key, const struct section *section) { int count; - for (count = 0; count < cfg->size; count++) { - if (strcmp(cfg->confdetails[count]->key, key) == 0) { - return strdup(cfg->confdetails[count]->value); + if (key == NULL || section == NULL) { + return NULL; + } + for (count = 0; count < section->size; count++) { + if (strcmp(section->kv_pairs[count]->key, key) == 0) { + return strdup(section->kv_pairs[count]->value); } } return NULL; @@ -311,61 +572,80 @@ char * get_value(const char* key, struct configuration *cfg) { * Function to return an array of values for a key. * Value delimiter is assumed to be a ','. */ -char ** get_values(const char * key, struct configuration *cfg) { - char *value = get_value(key, cfg); - return extract_values_delim(value, ","); +char **get_section_values(const char *key, const struct section *cfg) { + return get_section_values_delimiter(key, cfg, ","); } /** * Function to return an array of values for a key, using the specified delimiter. */ -char ** get_values_delim(const char * key, struct configuration *cfg, - const char *delim) { - char *value = get_value(key, cfg); - return extract_values_delim(value, delim); +char **get_section_values_delimiter(const char *key, const struct section *cfg, + const char *delim) { + if (key == NULL || cfg == NULL || delim == NULL) { + return NULL; + } + char *value = get_section_value(key, cfg); + char **split_values = split_delimiter(value, delim); + + if (value) { + free(value); + } + + return split_values; } -char ** extract_values_delim(char *value, const char *delim) { - char ** toPass = NULL; - char *tempTok = NULL; - char *tempstr = NULL; - int size = 0; - int toPassSize = MAX_SIZE; - //first allocate any array of 10 - if(value != NULL) { - toPass = (char **) malloc(sizeof(char *) * toPassSize); - tempTok = strtok_r((char *)value, delim, &tempstr); - while (tempTok != NULL) { - toPass[size++] = tempTok; - if(size == toPassSize) { - toPassSize += MAX_SIZE; - toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize)); - } - tempTok = strtok_r(NULL, delim, &tempstr); - } +char *get_configuration_value(const char *key, const char *section, + const struct configuration *cfg) { + const struct section *section_ptr; + if (key == NULL || section == NULL || cfg == NULL) { + return NULL; } - if (toPass != NULL) { - toPass[size] = NULL; + section_ptr = get_configuration_section(section, cfg); + if (section_ptr != NULL) { + return get_section_value(key, section_ptr); } - return toPass; + return NULL; } -/** - * Extracts array of values from the '%' separated list of values. - */ -char ** extract_values(char *value) { - return extract_values_delim(value, "%"); +char **get_configuration_values(const char *key, const char *section, + const struct configuration *cfg) { + const struct section *section_ptr; + if (key == NULL || section == NULL || cfg == NULL) { + return NULL; + } + section_ptr = get_configuration_section(section, cfg); + if (section_ptr != NULL) { + return get_section_values(key, section_ptr); + } + return NULL; +} + +char **get_configuration_values_delimiter(const char *key, const char *section, + const struct configuration *cfg, const char *delim) { + const struct section *section_ptr; + if (key == NULL || section == NULL || cfg == NULL || delim == NULL) { + return NULL; + } + section_ptr = get_configuration_section(section, cfg); + if (section_ptr != NULL) { + return get_section_values_delimiter(key, section_ptr, delim); + } + return NULL; } -// free an entry set of values -void free_values(char** values) { - if (*values != NULL) { - free(*values); +struct section *get_configuration_section(const char *section, + const struct configuration *cfg) { + int i = 0; + if (cfg == NULL || section == NULL) { + return NULL; } - if (values != NULL) { - free(values); + for (i = 0; i < cfg->size; ++i) { + if (strcmp(cfg->sections[i]->name, section) == 0) { + return cfg->sections[i]; + } } + return NULL; } /** @@ -376,12 +656,12 @@ int get_kv_key(const char *input, char *out, size_t out_len) { if (input == NULL) return -EINVAL; - char *split = strchr(input, '='); + const char *split = strchr(input, '='); if (split == NULL) return -EINVAL; - int key_len = split - input; + unsigned long key_len = split - input; if (out_len < (key_len + 1) || out == NULL) return -ENAMETOOLONG; @@ -400,13 +680,13 @@ int get_kv_value(const char *input, char *out, size_t out_len) { if (input == NULL) return -EINVAL; - char *split = strchr(input, '='); + const char *split = strchr(input, '='); if (split == NULL) return -EINVAL; split++; // advance past '=' to the value - int val_len = (input + strlen(input)) - split; + unsigned long val_len = (input + strlen(input)) - split; if (out_len < (val_len + 1) || out == NULL) return -ENAMETOOLONG; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h index 2d14867..1ea5561 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h @@ -16,6 +16,9 @@ * limitations under the License. */ +#ifndef __YARN_CONTAINER_EXECUTOR_CONFIG_H__ +#define __YARN_CONTAINER_EXECUTOR_CONFIG_H__ + #ifdef __FreeBSD__ #define _WITH_GETLINE #endif @@ -23,62 +26,160 @@ #include <stddef.h> /** Define a platform-independent constant instead of using PATH_MAX */ - #define EXECUTOR_PATH_MAX 4096 -/** - * Ensure that the configuration file and all of the containing directories - * are only writable by root. Otherwise, an attacker can change the - * configuration and potentially cause damage. - * returns 0 if permissions are ok - */ -int check_configuration_permissions(const char* file_name); - -/** - * Return a string with the configuration file path name resolved via realpath(3) - * - * NOTE: relative path names are resolved relative to the second argument not getwd(3) - */ -char *resolve_config_path(const char* file_name, const char *root); - -// Config data structures. -struct confentry { +// Configuration data structures. +struct kv_pair { const char *key; const char *value; }; +struct section { + int size; + char *name; + struct kv_pair **kv_pairs; +}; + struct configuration { int size; - struct confentry **confdetails; + struct section **sections; }; -// read the given configuration file into the specified config struct. -void read_config(const char* config_file, struct configuration *cfg); +/** + * Function to ensure that the configuration file and all of the containing + * directories are only writable by root. Otherwise, an attacker can change + * the configuration and potentially cause damage. + * + * @param file_name name of the config file + * + * @returns 0 if permissions are correct, non-zero on error + */ +int check_configuration_permissions(const char *file_name); + +/** + * Return a string with the configuration file path name resolved via + * realpath(3). Relative path names are resolved relative to the second + * argument and not getwd(3). It's up to the caller to free the returned + * value. + * + * @param file_name name of the config file + * @param root the path against which relative path names are to be resolved + * + * @returns the resolved configuration file path + */ +char* resolve_config_path(const char *file_name, const char *root); -//method exposed to get the configurations -char *get_value(const char* key, struct configuration *cfg); +/** + * Read the given configuration file into the specified configuration struct. + * It's the responsibility of the caller to call free_configurations to free + * the allocated memory. The function will check to ensure that the + * configuration file has the appropriate owner and permissions. + * + * @param file_path name of the configuration file to be read + * @param cfg the configuration structure to be filled. + * + * @return 0 on success, non-zero if there was an error + */ +int read_config(const char *file_path, struct configuration *cfg); + +/** + * Get the value for a key in the specified section. It's up to the caller to + * free the memory used for storing the return value. + * + * @param key key the name of the key + * @param section the section to be looked up + * + * @return pointer to the value if the key was found, null otherwise + */ +char* get_section_value(const char *key, const struct section *section); -//function to return array of values pointing to the key. Values are -//comma seperated strings. -char ** get_values(const char* key, struct configuration *cfg); +/** + * Function to get the values for a given key in the specified section. + * The value is split by ",". It's up to the caller to free the memory used + * for storing the return values. + * + * @param key the key to be looked up + * @param section the section to be looked up + * + * @return array of values, null if the key was not found + */ +char** get_section_values(const char *key, const struct section *section); /** - * Function to return an array of values for a key, using the specified - delimiter. + * Function to get the values for a given key in the specified section. + * The value is split by the specified delimiter. It's up to the caller to + * free the memory used for storing the return values. + * + * @param key the key to be looked up + * @param section the section to be looked up + * @param delimiter the delimiter to be used to split the value + * + * @return array of values, null if the key was not found */ -char ** get_values_delim(const char * key, struct configuration *cfg, +char** get_section_values_delimiter(const char *key, const struct section *section, const char *delim); -// Extracts array of values from the comma separated list of values. -char ** extract_values(char *value); +/** + * Get the value for a key in the specified section in the specified + * configuration. It's up to the caller to free the memory used for storing + * the return value. + * + * @param key key the name of the key + * @param section the name section to be looked up + * @param cfg the configuration to be used + * + * @return pointer to the value if the key was found, null otherwise + */ +char* get_configuration_value(const char *key, const char* section, + const struct configuration *cfg); + +/** + * Function to get the values for a given key in the specified section in the + * specified configuration. The value is split by ",". It's up to the caller to + * free the memory used for storing the return values. + * + * @param key the key to be looked up + * @param section the name of the section to be looked up + * @param cfg the configuration to be looked up + * + * @return array of values, null if the key was not found + */ +char** get_configuration_values(const char *key, const char* section, + const struct configuration *cfg); -char ** extract_values_delim(char *value, const char *delim); +/** + * Function to get the values for a given key in the specified section in the + * specified configuration. The value is split by the specified delimiter. + * It's up to the caller to free the memory used for storing the return values. + * + * @param key the key to be looked up + * @param section the name of the section to be looked up + * @param cfg the section to be looked up + * @param delimiter the delimiter to be used to split the value + * + * @return array of values, null if the key was not found + */ +char** get_configuration_values_delimiter(const char *key, const char* section, + const struct configuration *cfg, const char *delimiter); -// free the memory returned by get_values -void free_values(char** values); +/** + * Function to retrieve the specified section from the configuration. + * + * @param section the name of the section to retrieve + * @param cfg the configuration structure to use + * + * @return pointer to section struct containing details of the section + * null on error + */ +struct section* get_configuration_section(const char *section, + const struct configuration *cfg); -//method to free allocated configuration -void free_configurations(struct configuration *cfg); +/** + * Method to free an allocated config struct. + * + * @param cfg pointer to the structure to free + */ +void free_configuration(struct configuration *cfg); /** * If str is a string of the form key=val, find 'key' @@ -106,11 +207,4 @@ int get_kv_key(const char *input, char *out, size_t out_len); */ int get_kv_value(const char *input, char *out, size_t out_len); -/** - * Trim whitespace from beginning and end. - * - * @param input Input string that needs to be trimmed - * - * @return the trimmed string allocated with malloc. I has to be freed by the caller -*/ -char* trim(char* input); +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index def628e..9f754c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -19,6 +19,8 @@ #include "configuration.h" #include "container-executor.h" #include "utils/string-utils.h" +#include "util.h" +#include "config.h" #include <inttypes.h> #include <libgen.h> @@ -43,8 +45,6 @@ #include <getopt.h> #include <regex.h> -#include "config.h" - #ifndef HAVE_FCHMODAT #include "compat/fchmodat.h" #endif @@ -92,7 +92,8 @@ FILE* ERRORFILE = NULL; static uid_t nm_uid = -1; static gid_t nm_gid = -1; -struct configuration executor_cfg = {.size=0, .confdetails=NULL}; +struct configuration CFG = {.size=0, .sections=NULL}; +struct section executor_cfg = {.size=0, .kv_pairs=NULL}; char *concatenate(char *concat_pattern, char *return_path_name, int numArgs, ...); @@ -103,18 +104,25 @@ void set_nm_uid(uid_t user, gid_t group) { } //function used to load the configurations present in the secure config -void read_executor_config(const char* file_name) { - read_config(file_name, &executor_cfg); +void read_executor_config(const char *file_name) { + const struct section *tmp = NULL; + int ret = read_config(file_name, &CFG); + if (ret == 0) { + tmp = get_configuration_section("", &CFG); + if (tmp != NULL) { + executor_cfg = *tmp; + } + } } //function used to free executor configuration data void free_executor_configurations() { - free_configurations(&executor_cfg); + free_configuration(&CFG); } //Lookup nodemanager group from container executor configuration. char *get_nodemanager_group() { - return get_value(NM_GROUP_KEY, &executor_cfg); + return get_section_value(NM_GROUP_KEY, &executor_cfg); } int check_executor_permissions(char *executable_file) { @@ -431,8 +439,8 @@ int change_user(uid_t user, gid_t group) { } int is_feature_enabled(const char* feature_key, int default_value, - struct configuration *cfg) { - char *enabled_str = get_value(feature_key, cfg); + struct section *cfg) { + char *enabled_str = get_section_value(feature_key, cfg); int enabled = default_value; if (enabled_str != NULL) { @@ -753,7 +761,7 @@ static struct passwd* get_user_info(const char* user) { } int is_whitelisted(const char *user) { - char **whitelist = get_values(ALLOWED_SYSTEM_USERS_KEY, &executor_cfg); + char **whitelist = get_section_values(ALLOWED_SYSTEM_USERS_KEY, &executor_cfg); char **users = whitelist; if (whitelist != NULL) { for(; *users; ++users) { @@ -781,7 +789,7 @@ struct passwd* check_user(const char *user) { fflush(LOGFILE); return NULL; } - char *min_uid_str = get_value(MIN_USERID_KEY, &executor_cfg); + char *min_uid_str = get_section_value(MIN_USERID_KEY, &executor_cfg); int min_uid = DEFAULT_MIN_USERID; if (min_uid_str != NULL) { char *end_ptr = NULL; @@ -808,7 +816,7 @@ struct passwd* check_user(const char *user) { free(user_info); return NULL; } - char **banned_users = get_values(BANNED_USERS_KEY, &executor_cfg); + char **banned_users = get_section_values(BANNED_USERS_KEY, &executor_cfg); banned_users = banned_users == NULL ? (char**) DEFAULT_BANNED_USERS : banned_users; char **banned_user = banned_users; @@ -1194,7 +1202,6 @@ char** tokenize_docker_command(const char *input, int *split_counter) { char *line = (char *)calloc(strlen(input) + 1, sizeof(char)); char **linesplit = (char **) malloc(sizeof(char *)); char *p = NULL; - int c = 0; *split_counter = 0; strncpy(line, input, strlen(input)); @@ -1408,12 +1415,12 @@ char* parse_docker_command_file(const char* command_file) { int run_docker(const char *command_file) { char* docker_command = parse_docker_command_file(command_file); - char* docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg); + char* docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg); docker_binary = check_docker_binary(docker_binary); char* docker_command_with_binary = calloc(sizeof(char), EXECUTOR_PATH_MAX); snprintf(docker_command_with_binary, EXECUTOR_PATH_MAX, "%s %s", docker_binary, docker_command); - char **args = extract_values_delim(docker_command_with_binary, " "); + char **args = split_delimiter(docker_command_with_binary, " "); int exit_code = -1; if (execvp(docker_binary, args) != 0) { @@ -1574,7 +1581,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id, uid_t prev_uid = geteuid(); char *docker_command = parse_docker_command_file(command_file); - char *docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg); + char *docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg); docker_binary = check_docker_binary(docker_binary); fprintf(LOGFILE, "Creating script paths...\n"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 1dc0491..ea8b5e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -35,51 +35,6 @@ enum command { LIST_AS_USER = 5 }; -enum errorcodes { - INVALID_ARGUMENT_NUMBER = 1, - //INVALID_USER_NAME 2 - INVALID_COMMAND_PROVIDED = 3, - // SUPER_USER_NOT_ALLOWED_TO_RUN_TASKS (NOT USED) 4 - INVALID_NM_ROOT_DIRS = 5, - SETUID_OPER_FAILED, //6 - UNABLE_TO_EXECUTE_CONTAINER_SCRIPT, //7 - UNABLE_TO_SIGNAL_CONTAINER, //8 - INVALID_CONTAINER_PID, //9 - // ERROR_RESOLVING_FILE_PATH (NOT_USED) 10 - // RELATIVE_PATH_COMPONENTS_IN_FILE_PATH (NOT USED) 11 - // UNABLE_TO_STAT_FILE (NOT USED) 12 - // FILE_NOT_OWNED_BY_ROOT (NOT USED) 13 - // PREPARE_CONTAINER_DIRECTORIES_FAILED (NOT USED) 14 - // INITIALIZE_CONTAINER_FAILED (NOT USED) 15 - // PREPARE_CONTAINER_LOGS_FAILED (NOT USED) 16 - // INVALID_LOG_DIR (NOT USED) 17 - OUT_OF_MEMORY = 18, - // INITIALIZE_DISTCACHEFILE_FAILED (NOT USED) 19 - INITIALIZE_USER_FAILED = 20, - PATH_TO_DELETE_IS_NULL, //21 - INVALID_CONTAINER_EXEC_PERMISSIONS, //22 - // PREPARE_JOB_LOGS_FAILED (NOT USED) 23 - INVALID_CONFIG_FILE = 24, - SETSID_OPER_FAILED = 25, - WRITE_PIDFILE_FAILED = 26, - WRITE_CGROUP_FAILED = 27, - TRAFFIC_CONTROL_EXECUTION_FAILED = 28, - DOCKER_RUN_FAILED = 29, - ERROR_OPENING_DOCKER_FILE = 30, - ERROR_READING_DOCKER_FILE = 31, - FEATURE_DISABLED = 32, - COULD_NOT_CREATE_SCRIPT_COPY = 33, - COULD_NOT_CREATE_CREDENTIALS_FILE = 34, - COULD_NOT_CREATE_WORK_DIRECTORIES = 35, - COULD_NOT_CREATE_APP_LOG_DIRECTORIES = 36, - COULD_NOT_CREATE_TMP_DIRECTORIES = 37, - ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38, - ERROR_SANITIZING_DOCKER_COMMAND = 39, - DOCKER_IMAGE_INVALID = 40, - DOCKER_CONTAINER_NAME_INVALID = 41, - ERROR_COMPILING_REGEX = 42 -}; - enum operations { CHECK_SETUP = 1, MOUNT_CGROUPS = 2, @@ -111,11 +66,6 @@ enum operations { extern struct passwd *user_detail; -// the log file for messages -extern FILE *LOGFILE; -// the log file for error messages -extern FILE *ERRORFILE; - // get the executable's filename char* get_executable(char *argv0); @@ -276,7 +226,7 @@ int create_validate_dir(const char* npath, mode_t perm, const char* path, /** Check if a feature is enabled in the specified configuration. */ int is_feature_enabled(const char* feature_key, int default_value, - struct configuration *cfg); + struct section *cfg); /** Check if tc (traffic control) support is enabled in configuration. */ int is_tc_support_enabled(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c index ce46b77..55973a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c @@ -31,6 +31,7 @@ #include "config.h" #include "configuration.h" #include "container-executor.h" +#include "util.h" #include <errno.h> #include <stdio.h> http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index fdc0496..b2187c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -19,6 +19,7 @@ #include "config.h" #include "configuration.h" #include "container-executor.h" +#include "util.h" #include <errno.h> #include <grp.h> @@ -420,7 +421,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) cmd_input.resources_key = resources_key; cmd_input.resources_value = resources_value; - cmd_input.resources_values = extract_values(resources_value); + cmd_input.resources_values = split(resources_value); *operation = RUN_AS_USER_LAUNCH_DOCKER_CONTAINER; return 0; } else { @@ -471,7 +472,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) cmd_input.resources_key = resources_key; cmd_input.resources_value = resources_value; - cmd_input.resources_values = extract_values(resources_value); + cmd_input.resources_values = split(resources_value); *operation = RUN_AS_USER_LAUNCH_CONTAINER; return 0; @@ -565,8 +566,8 @@ int main(int argc, char **argv) { exit_code = initialize_app(cmd_input.yarn_user_name, cmd_input.app_id, cmd_input.cred_file, - extract_values(cmd_input.local_dirs), - extract_values(cmd_input.log_dirs), + split(cmd_input.local_dirs), + split(cmd_input.log_dirs), argv + optind); break; case RUN_AS_USER_LAUNCH_DOCKER_CONTAINER: @@ -591,8 +592,8 @@ int main(int argc, char **argv) { cmd_input.script_file, cmd_input.cred_file, cmd_input.pid_file, - extract_values(cmd_input.local_dirs), - extract_values(cmd_input.log_dirs), + split(cmd_input.local_dirs), + split(cmd_input.log_dirs), cmd_input.docker_command_file, cmd_input.resources_key, cmd_input.resources_values); @@ -619,8 +620,8 @@ int main(int argc, char **argv) { cmd_input.script_file, cmd_input.cred_file, cmd_input.pid_file, - extract_values(cmd_input.local_dirs), - extract_values(cmd_input.log_dirs), + split(cmd_input.local_dirs), + split(cmd_input.log_dirs), cmd_input.resources_key, cmd_input.resources_values); free(cmd_input.resources_key); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c new file mode 100644 index 0000000..8e39ca8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "util.h" +#include <stddef.h> +#include <stdlib.h> +#include <string.h> +#include <ctype.h> + +char** split_delimiter(char *value, const char *delim) { + char **return_values = NULL; + char *temp_tok = NULL; + char *tempstr = NULL; + int size = 0; + int per_alloc_size = 10; + int return_values_size = per_alloc_size; + int failed = 0; + + //first allocate any array of 10 + if(value != NULL) { + return_values = (char **) malloc(sizeof(char *) * return_values_size); + if (!return_values) { + fprintf(ERRORFILE, "Allocation error for return_values in %s.\n", + __func__); + failed = 1; + goto cleanup; + } + memset(return_values, 0, sizeof(char *) * return_values_size); + + temp_tok = strtok_r(value, delim, &tempstr); + while (temp_tok != NULL) { + temp_tok = strdup(temp_tok); + if (NULL == temp_tok) { + fprintf(ERRORFILE, "Allocation error in %s.\n", __func__); + failed = 1; + goto cleanup; + } + + return_values[size++] = temp_tok; + + // Make sure returned values has enough space for the trailing NULL. + if (size >= return_values_size - 1) { + return_values_size += per_alloc_size; + return_values = (char **) realloc(return_values,(sizeof(char *) * + return_values_size)); + + // Make sure new added memory are filled with NULL + for (int i = size; i < return_values_size; i++) { + return_values[i] = NULL; + } + } + temp_tok = strtok_r(NULL, delim, &tempstr); + } + } + + // Put trailing NULL to indicate values terminates. + if (return_values != NULL) { + return_values[size] = NULL; + } + +cleanup: + if (failed) { + free_values(return_values); + return NULL; + } + + return return_values; +} + +/** + * Extracts array of values from the '%' separated list of values. + */ +char** split(char *value) { + return split_delimiter(value, "%"); +} + +// free an entry set of values +void free_values(char** values) { + if (values != NULL) { + int idx = 0; + while (values[idx]) { + free(values[idx]); + idx++; + } + free(values); + } +} + +/** + * Trim whitespace from beginning and end. +*/ +char* trim(const char* input) { + const char *val_begin; + const char *val_end; + char *ret; + + if (input == NULL) { + return NULL; + } + + val_begin = input; + val_end = input + strlen(input); + + while (val_begin < val_end && isspace(*val_begin)) + val_begin++; + while (val_end > val_begin && isspace(*(val_end - 1))) + val_end--; + + ret = (char *) malloc( + sizeof(char) * (val_end - val_begin + 1)); + if (ret == NULL) { + fprintf(ERRORFILE, "Allocation error\n"); + exit(OUT_OF_MEMORY); + } + + strncpy(ret, val_begin, val_end - val_begin); + ret[val_end - val_begin] = '\0'; + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h new file mode 100644 index 0000000..a8a12a9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__ +#define __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__ + +#include <stdio.h> + +enum errorcodes { + INVALID_ARGUMENT_NUMBER = 1, + //INVALID_USER_NAME 2 + INVALID_COMMAND_PROVIDED = 3, + // SUPER_USER_NOT_ALLOWED_TO_RUN_TASKS (NOT USED) 4 + INVALID_NM_ROOT_DIRS = 5, + SETUID_OPER_FAILED, //6 + UNABLE_TO_EXECUTE_CONTAINER_SCRIPT, //7 + UNABLE_TO_SIGNAL_CONTAINER, //8 + INVALID_CONTAINER_PID, //9 + // ERROR_RESOLVING_FILE_PATH (NOT_USED) 10 + // RELATIVE_PATH_COMPONENTS_IN_FILE_PATH (NOT USED) 11 + // UNABLE_TO_STAT_FILE (NOT USED) 12 + // FILE_NOT_OWNED_BY_ROOT (NOT USED) 13 + // PREPARE_CONTAINER_DIRECTORIES_FAILED (NOT USED) 14 + // INITIALIZE_CONTAINER_FAILED (NOT USED) 15 + // PREPARE_CONTAINER_LOGS_FAILED (NOT USED) 16 + // INVALID_LOG_DIR (NOT USED) 17 + OUT_OF_MEMORY = 18, + // INITIALIZE_DISTCACHEFILE_FAILED (NOT USED) 19 + INITIALIZE_USER_FAILED = 20, + PATH_TO_DELETE_IS_NULL, //21 + INVALID_CONTAINER_EXEC_PERMISSIONS, //22 + // PREPARE_JOB_LOGS_FAILED (NOT USED) 23 + INVALID_CONFIG_FILE = 24, + SETSID_OPER_FAILED = 25, + WRITE_PIDFILE_FAILED = 26, + WRITE_CGROUP_FAILED = 27, + TRAFFIC_CONTROL_EXECUTION_FAILED = 28, + DOCKER_RUN_FAILED = 29, + ERROR_OPENING_DOCKER_FILE = 30, + ERROR_READING_DOCKER_FILE = 31, + FEATURE_DISABLED = 32, + COULD_NOT_CREATE_SCRIPT_COPY = 33, + COULD_NOT_CREATE_CREDENTIALS_FILE = 34, + COULD_NOT_CREATE_WORK_DIRECTORIES = 35, + COULD_NOT_CREATE_APP_LOG_DIRECTORIES = 36, + COULD_NOT_CREATE_TMP_DIRECTORIES = 37, + ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38, + ERROR_SANITIZING_DOCKER_COMMAND = 39, + DOCKER_IMAGE_INVALID = 40, + DOCKER_CONTAINER_NAME_INVALID = 41, + ERROR_COMPILING_REGEX = 42 +}; + + +// the log file for messages +extern FILE *LOGFILE; +// the log file for error messages +extern FILE *ERRORFILE; +/** + * Function to split the given string using '%' as the separator. It's + * up to the caller to free the memory for the returned array. Use the + * free_values function to free the allocated memory. + * + * @param str the string to split + * + * @return an array of strings + */ +char** split(char *str); + +/** + * Function to split the given string using the delimiter specified. It's + * up to the caller to free the memory for the returned array. Use the + * free_values function to free the allocated memory. + * + * @param str the string to split + * @param delimiter the delimiter to use + * + * @return an array of strings + */ +char** split_delimiter(char *value, const char *delimiter); + +/** + * Function to free an array of strings. + * + * @param values the array to free + * + */ +void free_values(char **values); + +/** + * Trim whitespace from beginning and end. The returned string has to be freed + * by the caller. + * + * @param input Input string that needs to be trimmed + * + * @return the trimmed string allocated with malloc +*/ +char* trim(const char *input); + +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-1.cfg ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-1.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-1.cfg new file mode 100644 index 0000000..4d0b90d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-1.cfg @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +[section-1] +key1=value1 +split-key=val1,val2,val3 +perc-key=perc-val1%perc-val2 +# some comment + +[split-section] +key3=value3 + +[section-2] +key1=value2 + +key2=value2 + +[split-section] +key4=value4 http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-2.cfg ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-2.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-2.cfg new file mode 100644 index 0000000..aa02db8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-2.cfg @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test mixed mode config file +# Initial few lines are in the key=value format +# and then the sections start + +key1=value1 +key2=value2 + + +[section-1] +key3=value3 +key1=value4 + http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/old-config.cfg ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/old-config.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/old-config.cfg new file mode 100644 index 0000000..947a3fa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/old-config.cfg @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +yarn.nodemanager.linux-container-executor.group=yarn +banned.users=root,testuser1,testuser2#comma separated list of users who can not run applications +min.user.id=1000 +allowed.system.users=nobody,daemon +feature.docker.enabled=1 +feature.tc.enabled=0 +docker.binary=/usr/bin/docker +yarn.local.dirs=/var/run/yarn%/tmp/mydir +test.key=#no value for this key +# test.key2=0 http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index 3202652..3cfefa0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -18,6 +18,7 @@ #include "configuration.h" #include "container-executor.h" #include "utils/string-utils.h" +#include "util.h" #include <inttypes.h> #include <errno.h> @@ -404,7 +405,7 @@ void test_delete_app() { } void validate_feature_enabled_value(int expected_value, const char* key, - int default_value, struct configuration *cfg) { + int default_value, struct section *cfg) { int value = is_feature_enabled(key, default_value, cfg); if (value != expected_value) { @@ -419,7 +420,8 @@ void test_is_feature_enabled() { FILE *file = fopen(filename, "w"); int disabled = 0; int enabled = 1; - struct configuration cfg = {.size=0, .confdetails=NULL}; + struct configuration exec_cfg = {.size=0, .sections=NULL}; + struct section cfg = {.size=0, .kv_pairs=NULL}; if (file == NULL) { printf("FAIL: Could not open configuration file: %s\n", filename); @@ -433,7 +435,8 @@ void test_is_feature_enabled() { fprintf(file, "feature.name5.enabled=-1\n"); fprintf(file, "feature.name6.enabled=2\n"); fclose(file); - read_config(filename, &cfg); + read_config(filename, &exec_cfg); + cfg = *(get_configuration_section("", &exec_cfg)); validate_feature_enabled_value(disabled, "feature.name1.enabled", disabled, &cfg); @@ -449,7 +452,7 @@ void test_is_feature_enabled() { disabled, &cfg); - free_configurations(&cfg); + free_configuration(&exec_cfg); } void test_delete_user() { @@ -1345,8 +1348,8 @@ int main(int argc, char **argv) { read_executor_config(TEST_ROOT "/test.cfg"); - local_dirs = extract_values(strdup(NM_LOCAL_DIRS)); - log_dirs = extract_values(strdup(NM_LOG_DIRS)); + local_dirs = split(strdup(NM_LOCAL_DIRS)); + log_dirs = split(strdup(NM_LOG_DIRS)); create_nm_roots(local_dirs); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
