Repository: ambari Updated Branches: refs/heads/branch-1.7.0 d8d6bf4bb -> e58eda61e refs/heads/trunk a320e033c -> c72888df0
AMBARI-8187. Download client configs gives incorrect content for certain components (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c72888df Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c72888df Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c72888df Branch: refs/heads/trunk Commit: c72888df0f6cccbd67a4106b76a26dea12631a96 Parents: a320e03 Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Thu Nov 6 20:39:26 2014 +0200 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Thu Nov 6 20:39:26 2014 +0200 ---------------------------------------------------------------------- .../resource_management/libraries/script/script.py | 14 ++++++++++++++ .../internal/ClientConfigResourceProvider.java | 6 +++++- .../stacks/HDP/2.0.6/services/HBASE/metainfo.xml | 5 +++++ .../stacks/HDP/2.1/services/FALCON/metainfo.xml | 4 ++-- .../test/python/stacks/1.3.2/HDFS/test_hdfs_client.py | 6 ++++++ .../stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py | 6 ++++++ .../src/test/python/stacks/1.3.2/configs/default.json | 1 + .../test/python/stacks/2.0.6/HDFS/test_hdfs_client.py | 6 ++++++ .../src/test/python/stacks/2.0.6/configs/default.json | 1 + 9 files changed, 46 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c72888df/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 4caaa78..001922d 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -30,6 +30,7 @@ from contextlib import closing from resource_management.libraries.resources import XmlConfig +from resource_management.libraries.resources import PropertiesFile from resource_management.core.resources import File, Directory from resource_management.core.source import InlineTemplate @@ -250,6 +251,10 @@ class Script(object): import params return {'configurations':params.config['configurations'][dict], 'configuration_attributes':params.config['configuration_attributes'][dict]} + + def generate_configs_get_xml_file_dict(self, filename, dict): + import params + return params.config['configurations'][dict] def generate_configs(self, env): """ @@ -260,6 +265,8 @@ class Script(object): env.set_params(params) xml_configs_list = params.config['commandParams']['xml_configs_list'] env_configs_list = params.config['commandParams']['env_configs_list'] + properties_configs_list = params.config['commandParams']['properties_configs_list'] + conf_tmp_dir = tempfile.mkdtemp() output_filename = os.path.join(self.get_tmp_dir(),params.config['commandParams']['output_file']) @@ -274,6 +281,13 @@ class Script(object): for filename,dicts in file_dict.iteritems(): File(os.path.join(conf_tmp_dir, filename), content=InlineTemplate(self.generate_configs_get_template_file_content(filename, dicts))) + + for file_dict in properties_configs_list: + for filename, dict in file_dict.iteritems(): + PropertiesFile(os.path.join(conf_tmp_dir, filename), + properties=self.generate_configs_get_xml_file_dict(filename, dict) + ) + with closing(tarfile.open(output_filename, "w:gz")) as tar: tar.add(conf_tmp_dir, arcname=os.path.basename(".")) tar.close() http://git-wip-us.apache.org/repos/asf/ambari/blob/c72888df/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java index 94c0d58..5aca65b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java @@ -267,6 +267,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv Map<String, Object> commandParams = new HashMap<String, Object>(); List<Map<String, String>> xmlConfigs = new LinkedList<Map<String, String>>(); List<Map<String, String>> envConfigs = new LinkedList<Map<String, String>>(); + List<Map<String, String>> propertiesConfigs = new LinkedList<Map<String, String>>(); //Fill file-dictionary configs from metainfo for (ClientConfigFileDefinition clientConfigFile : clientConfigFiles) { @@ -274,13 +275,16 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv fileDict.put(clientConfigFile.getFileName(), clientConfigFile.getDictionaryName()); if (clientConfigFile.getType().equals("xml")) { xmlConfigs.add(fileDict); - } else { + } else if (clientConfigFile.getType().equals("env")) { envConfigs.add(fileDict); + } else if (clientConfigFile.getType().equals("properties")) { + propertiesConfigs.add(fileDict); } } commandParams.put("xml_configs_list", xmlConfigs); commandParams.put("env_configs_list", envConfigs); + commandParams.put("properties_configs_list", propertiesConfigs); commandParams.put("output_file", componentName + "-configs.tar.gz"); Map<String, Object> jsonContent = new TreeMap<String, Object>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/c72888df/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml index 0816591..fd290df 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml @@ -97,6 +97,11 @@ <dictionaryName>hbase-env</dictionaryName> </configFile> <configFile> + <type>xml</type> + <fileName>hbase-policy.xml</fileName> + <dictionaryName>hbase-policy</dictionaryName> + </configFile> + <configFile> <type>env</type> <fileName>log4j.properties</fileName> <dictionaryName>hbase-log4j</dictionaryName> http://git-wip-us.apache.org/repos/asf/ambari/blob/c72888df/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml index b4e6883..78336e6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml @@ -41,12 +41,12 @@ <dictionaryName>falcon-env</dictionaryName> </configFile> <configFile> - <type>xml</type> + <type>properties</type> <fileName>runtime.properties</fileName> <dictionaryName>falcon-runtime.properties</dictionaryName> </configFile> <configFile> - <type>xml</type> + <type>properties</type> <fileName>startup.properties</fileName> <dictionaryName>falcon-startup.properties</dictionaryName> </configFile> http://git-wip-us.apache.org/repos/asf/ambari/blob/c72888df/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py index 80da9b6..f670ac4 100644 --- a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py +++ b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py @@ -52,6 +52,12 @@ class Test(RMFTestCase): self.assertResourceCalled('File', '/tmp/123/log4j.properties', content = InlineTemplate("log4jproperties\nline2log4jproperties\nline2\nambari.jobhistory.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nambari.jobhistory.driver=org.postgresql.Driver\nambari.jobhistory.user=mapred\nambari.jobhistory.password=mapred\nambari.jobhistory.logger=${hadoop.root.logger}\n\nlog4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender\nlog4j.appender.JHA.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nlog4j.appender.JHA.driver=org.postgresql.Driver\nlog4j.appender.JHA.user=mapred\nlog4j.appender.JHA.password=mapred\n\nlog4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=DEBUG,JHA\nlog4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true\n\n"), ) + self.assertResourceCalled('PropertiesFile', '/tmp/123/runtime.properties', + properties = UnknownConfigurationMock(), + ) + self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties', + properties = UnknownConfigurationMock(), + ) self.assertResourceCalled('Directory', '/tmp/123', action = ['delete'], ) http://git-wip-us.apache.org/repos/asf/ambari/blob/c72888df/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py b/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py index 2081c36..1621b42 100644 --- a/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py +++ b/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py @@ -196,6 +196,12 @@ class TestMapreduceClient(RMFTestCase): self.assertResourceCalled('File', '/tmp/123/log4j.properties', content = InlineTemplate("log4jproperties\nline2log4jproperties\nline2\nambari.jobhistory.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nambari.jobhistory.driver=org.postgresql.Driver\nambari.jobhistory.user=mapred\nambari.jobhistory.password=mapred\nambari.jobhistory.logger=${hadoop.root.logger}\n\nlog4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender\nlog4j.appender.JHA.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nlog4j.appender.JHA.driver=org.postgresql.Driver\nlog4j.appender.JHA.user=mapred\nlog4j.appender.JHA.password=mapred\n\nlog4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=DEBUG,JHA\nlog4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true\n\n"), ) + self.assertResourceCalled('PropertiesFile', '/tmp/123/runtime.properties', + properties = UnknownConfigurationMock(), + ) + self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties', + properties = UnknownConfigurationMock(), + ) self.assertResourceCalled('Directory', '/tmp/123', action = ['delete'], ) http://git-wip-us.apache.org/repos/asf/ambari/blob/c72888df/ambari-server/src/test/python/stacks/1.3.2/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json index 0256444..a39d51d 100644 --- a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json +++ b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json @@ -35,6 +35,7 @@ "update_exclude_file_only" : "false", "xml_configs_list":[{"core-site.xml":"core-site"},{"mapred-site.xml":"mapred-site"}], "env_configs_list":[{"log4j.properties":"hdfs-log4j,mapreduce-log4j"}], + "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], "output_file":"MAPREDUCE_CLIENT-configs.tar.gz" }, "taskId": 18, http://git-wip-us.apache.org/repos/asf/ambari/blob/c72888df/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py index 0a4bfb5..7c9e42b 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py @@ -52,6 +52,12 @@ class Test(RMFTestCase): content = InlineTemplate(self.getConfig()['configurations']['hdfs-log4j']['content']+ self.getConfig()['configurations']['yarn-log4j']['content']), ) + self.assertResourceCalled('PropertiesFile', '/tmp/123/runtime.properties', + properties = UnknownConfigurationMock(), + ) + self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties', + properties = UnknownConfigurationMock(), + ) self.assertResourceCalled('Directory', '/tmp/123', action = ['delete'], ) http://git-wip-us.apache.org/repos/asf/ambari/blob/c72888df/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index 2dc4a8d..6b68e0b 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -32,6 +32,7 @@ "update_exclude_file_only" : "false", "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}], "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}], + "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}], "output_file":"HDFS_CLIENT-configs.tar.gz" }, "taskId": 152,
