Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-1107_AM_config_generation 0732af6d6 -> 40c451e29
SLIDER-1107 remove unneeded changes from this branch Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/40c451e2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/40c451e2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/40c451e2 Branch: refs/heads/feature/SLIDER-1107_AM_config_generation Commit: 40c451e29688ce9c21abcf48e0ba226a4a02df44 Parents: 0732af6 Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Thu Jun 2 07:50:52 2016 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Thu Jun 2 07:50:52 2016 -0700 ---------------------------------------------------------------------- .../src/main/python/agent/YarnDockerManager.py | 37 +++++++++++++++++++- .../python/resource_management/core/shell.py | 20 +++++++---- .../docstore/PublishedConfiguration.java | 1 + .../funtest/misc/AMConfigPublishingIT.groovy | 4 --- 4 files changed, 51 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/40c451e2/slider-agent/src/main/python/agent/YarnDockerManager.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/YarnDockerManager.py b/slider-agent/src/main/python/agent/YarnDockerManager.py index 4ae19ab..abdb2fb 100644 --- a/slider-agent/src/main/python/agent/YarnDockerManager.py +++ b/slider-agent/src/main/python/agent/YarnDockerManager.py @@ -55,7 +55,8 @@ class YarnDockerManager(Script): self.container_id = command['hostLevelParams']['container_id'] if command['roleCommand'] == 'INSTALL': - # do nothing + with Environment(self.workroot) as env: + self.install_container(command, env) returncode = 0; out = ''; err = '' if command['roleCommand'] == 'START': returncode, out, err = self.start_container(command) @@ -129,6 +130,40 @@ class YarnDockerManager(Script): return command['configurations']['global'][keyName] return None + def install_container(self, command, env): + try: + configFiles = self.extract_config_files_from_command(command) + for configFile in configFiles: + properties = self.extract_config_file_properties_from_command(command, configFile) + fileName = configFile['fileName'] + dir = self.get_config_file_global(command, configFile, 'destDir') + if dir is None: + dir = self.workroot + logger.info("creating config file " + str(configFile) + " in directory "+str(dir)) + Directory(dir, recursive = True) + if configFile['type'] == 'properties': + PropertiesFile(fileName, + dir=dir, + properties=properties) + elif configFile['type'] == 'env': + content = self.get_config_file_global(command, configFile, 'content', useEnv=False) + if content is not None: + File(os.path.join(dir, fileName), + content=InlineTemplate(content, **properties)) + elif configFile['type'] == 'template': + templateFile = self.get_config_file_global(command, configFile, 'templateFile') + if templateFile is not None: + with open(templateFile,"r") as fp: + fileContent = fp.read() + File(os.path.join(dir, fileName), + content=InlineTemplate(fileContent, **properties)) + elif configFile['type'] == 'xml': + XmlConfig(fileName, + conf_dir=dir, + configurations=properties) + except: + traceback.print_exc() + def start_container(self, command): #extracting param needed by docker run from the command passed from AM startCommand = self.extract_config_from_command(command, 'docker.startCommand') http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/40c451e2/slider-agent/src/main/python/resource_management/core/shell.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/resource_management/core/shell.py b/slider-agent/src/main/python/resource_management/core/shell.py index e717ae4..f21dbbf 100644 --- a/slider-agent/src/main/python/resource_management/core/shell.py +++ b/slider-agent/src/main/python/resource_management/core/shell.py @@ -31,8 +31,10 @@ from exceptions import ExecuteTimeoutException from resource_management.core.logger import Logger import time -APPLICATION_STD_OUTPUT_LOG_FILE = 'application.log' -APPLICATION_STD_ERROR_LOG_FILE = 'application.err' +APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application-' +APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = '.log' +APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application-' +APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = '.err' def checked_call(command, logoutput=False, cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file=None, poll_after=None): @@ -67,9 +69,15 @@ def _call(command, logoutput=False, throw_on_failure=True, """ command = ["/bin/bash","--login","-c", command] #adding redirecting stdout stderr to file - stdoutFile = open(APPLICATION_STD_OUTPUT_LOG_FILE, 'w+') - stderrFile = open(APPLICATION_STD_ERROR_LOG_FILE, 'w+') - + outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \ + str(pid_file_name) + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE + + errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \ + str(pid_file_name) + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE + + stdoutFile = open(outfilename, 'w') + stderrFile = open(errfilename, 'w') + proc = subprocess.Popen(command, stdout = stdoutFile, stderr = stderrFile, universal_newlines = True, cwd=cwd, env=env, shell=False, preexec_fn=preexec_fn) @@ -134,4 +142,4 @@ def quote_bash_args(command): for char in command: if char not in valid: return "'" + command.replace("'", "'\"'\"'") + "'" - return command + return command \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/40c451e2/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java index 477f7d3..50b522f 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java @@ -85,6 +85,7 @@ public class PublishedConfiguration { putValues(ConfigHelper.resolveConfiguration(keysource, valuesource)); } + /** * Is the configuration empty. This means either that it has not * been given any values, or it is stripped down copy set down over the http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/40c451e2/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/AMConfigPublishingIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/AMConfigPublishingIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/AMConfigPublishingIT.groovy index de3ea3e..e84ada6 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/AMConfigPublishingIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/AMConfigPublishingIT.groovy @@ -20,11 +20,7 @@ package org.apache.slider.funtest.misc import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import org.apache.hadoop.fs.Path -import org.apache.slider.api.ClusterDescription -import org.apache.slider.client.SliderClient import org.apache.slider.common.SliderExitCodes -import org.apache.slider.common.SliderKeys import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions import org.apache.slider.common.tools.SliderUtils
