AMBARI-16288. Separate number of shards and replication factor configurations for audit logs and service logs (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3332dad4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3332dad4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3332dad4 Branch: refs/heads/trunk Commit: 3332dad4d6da83defc61dba6db3a147db63deecb Parents: 45be225 Author: oleewere <oleew...@gmail.com> Authored: Tue May 10 13:39:51 2016 +0200 Committer: oleewere <oleew...@gmail.com> Committed: Tue May 10 14:59:36 2016 +0200 ---------------------------------------------------------------------- .../configuration/logsearch-properties.xml | 40 ++++++++++++++++--- .../LOGSEARCH/0.5.0/package/scripts/params.py | 17 +++++--- .../package/templates/logsearch.properties.j2 | 10 ++--- .../package/templates/output.config.json.j2 | 4 +- .../LOGSEARCH/0.5.0/themes/theme.json | 42 +++++++++++++++++--- .../stacks/HDP/2.3/services/stack_advisor.py | 17 +++++--- .../stacks/2.3/common/test_stack_advisor.py | 18 ++++++--- .../test/python/stacks/2.4/configs/default.json | 10 +++-- ambari-web/app/data/HDP2/site_properties.js | 26 +++--------- 9 files changed, 126 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/3332dad4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml index 172ed95..85f0e4f 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml @@ -22,10 +22,10 @@ <configuration> <property> - <name>logsearch.collection.numshards</name> + <name>logsearch.collection.service.logs.numshards</name> <value>5</value> - <display-name>Logsearch Solr Collection Shards</display-name> - <description>Number of shards for Solr collections</description> + <display-name>Logsearch Solr Service Logs Shards</display-name> + <description>Number of shards for Service Logs collection</description> <value-attributes> <type>int</type> <minimum>1</minimum> @@ -36,10 +36,38 @@ </property> <property> - <name>logsearch.collection.replication.factor</name> + <name>logsearch.collection.service.logs.replication.factor</name> <value>1</value> - <display-name>Logsearch Solr Collection Replication Factor</display-name> - <description>Replication factor for Solr collections</description> + <display-name>Logsearch Solr Service Logs Replication Factor</display-name> + <description>Replication factor for Service Logs Solr collection</description> + <value-attributes> + <type>int</type> + <minimum>1</minimum> + <maximum>10</maximum> + <unit>int</unit> + <increment-step>1</increment-step> + </value-attributes> + </property> + + <property> + <name>logsearch.collection.audit.logs.numshards</name> + <value>5</value> + <display-name>Logsearch Solr Audit Logs Shards</display-name> + <description>Number of shards for Audit Logs collection</description> + <value-attributes> + <type>int</type> + <minimum>1</minimum> + <maximum>20</maximum> + <unit>int</unit> + <increment-step>1</increment-step> + </value-attributes> + </property> + + <property> + <name>logsearch.collection.audit.logs.replication.factor</name> + <value>1</value> + <display-name>Logsearch Solr Audit Logs Replication Factor</display-name> + <description>Replication factor for Audit Logs Solr collection</description> <value-attributes> <type>int</type> <minimum>1</minimum> http://git-wip-us.apache.org/repos/asf/ambari/blob/3332dad4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index 4b2d769..6c26afc 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -138,14 +138,21 @@ solr_client_log4j_content = config['configurations']['logsearch-solr-client-log4 ##################################### logsearch_dir = '/usr/lib/ambari-logsearch-portal' -logsearch_numshards_config = config['configurations']['logsearch-properties']['logsearch.collection.numshards'] +logsearch_service_numshards_config = config['configurations']['logsearch-properties']['logsearch.collection.service.logs.numshards'] +logsearch_audit_numshards_config = config['configurations']['logsearch-properties']['logsearch.collection.audit.logs.numshards'] -if logsearch_numshards_config > 0: - logsearch_numshards = str(logsearch_numshards_config) +if logsearch_service_numshards_config > 0: + logsearch_service_logs_numshards = str(logsearch_service_numshards_config) else: - logsearch_numshards = format(str(logsearch_solr_instance_count)) + logsearch_service_logs_numshards = format(str(logsearch_solr_instance_count)) -logsearch_repfactor = str(config['configurations']['logsearch-properties']['logsearch.collection.replication.factor']) +if logsearch_audit_numshards_config > 0: + logsearch_audit_logs_numshards = str(logsearch_audit_numshards_config) +else: + logsearch_audit_logs_numshards = format(str(logsearch_audit_numshards_config)) + +logsearch_service_logs_repfactor = str(config['configurations']['logsearch-properties']['logsearch.collection.service.logs.replication.factor']) +logsearch_audit_logs_repfactor = str(config['configurations']['logsearch-properties']['logsearch.collection.audit.logs.replication.factor']) logsearch_solr_collection_service_logs = default('/configurations/logsearch-properties/logsearch.solr.collection.service.logs', 'hadoop_logs') logsearch_solr_collection_audit_logs = default('/configurations/logsearch-properties/logsearch.solr.collection.audit.logs','audit_logs') http://git-wip-us.apache.org/repos/asf/ambari/blob/3332dad4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 index ac731ce..3813a0a 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 @@ -20,8 +20,8 @@ solr.zkhosts={{zookeeper_quorum}}{{logsearch_solr_znode}} solr.core.logs={{logsearch_solr_collection_service_logs}} solr.service_logs.split_interval_mins={{service_logs_collection_splits_interval_mins}} -solr.service_logs.shards={{logsearch_numshards}} -solr.service_logs.replication_factor={{logsearch_repfactor}} +solr.service_logs.shards={{logsearch_service_logs_numshards}} +solr.service_logs.replication_factor={{logsearch_service_logs_repfactor}} solr.servicelogs.fields={{logsearch_service_logs_fields}} @@ -31,13 +31,13 @@ auditlog.solr.core.logs={{logsearch_solr_collection_audit_logs}} auditlog.solr.url={{solr_audit_logs_url}} solr.audit_logs.split_interval_mins={{audit_logs_collection_splits_interval_mins}} -solr.audit_logs.shards={{logsearch_numshards}} -solr.audit_logs.replication_factor={{logsearch_repfactor}} +solr.audit_logs.shards={{logsearch_audit_logs_numshards}} +solr.audit_logs.replication_factor={{logsearch_audit_logs_repfactor}} # History logs solr.core.history=history solr.history.config_name=history -solr.history.replication_factor={{logsearch_repfactor}} +solr.history.replication_factor=1 #Logfeeder Settings http://git-wip-us.apache.org/repos/asf/ambari/blob/3332dad4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 index 09858dc..7d0319d 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 @@ -23,7 +23,7 @@ "destination":"solr", "zk_hosts":"{{zookeeper_quorum}}{{logsearch_solr_znode}}", "collection":"{{logsearch_solr_collection_service_logs}}", - "number_of_shards": "{{logsearch_numshards}}", + "number_of_shards": "{{logsearch_service_logs_numshards}}", "splits_interval_mins": "{{service_logs_collection_splits_interval_mins}}", "conditions":{ "fields":{ @@ -42,7 +42,7 @@ "destination":"solr", "zk_hosts":"{{zookeeper_quorum}}{{logsearch_solr_znode}}", "collection":"{{logsearch_solr_collection_audit_logs}}", - "number_of_shards": "{{logsearch_numshards}}", + "number_of_shards": "{{logsearch_audit_logs_numshards}}", "splits_interval_mins": "{{audit_logs_collection_splits_interval_mins}}", "conditions":{ "fields":{ http://git-wip-us.apache.org/repos/asf/ambari/blob/3332dad4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json index 248bbd3..259fb28 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json @@ -39,8 +39,8 @@ "column-index": "1", "row-span": "1", "column-span": "1", - "section-columns": "3", - "section-rows": "1", + "section-columns": "1", + "section-rows": "5", "subsections": [ { "name": "subsection-logsearch-server-col1", @@ -81,11 +81,19 @@ "subsection-name": "subsection-logsearch-server-col1" }, { - "config": "logsearch-properties/logsearch.collection.numshards", + "config": "logsearch-properties/logsearch.collection.service.logs.numshards", + "subsection-name": "subsection-logsearch-server-col1" + }, + { + "config": "logsearch-properties/logsearch.collection.service.logs.replication.factor", + "subsection-name": "subsection-logsearch-server-col1" + }, + { + "config": "logsearch-properties/logsearch.collection.audit.logs.numshards", "subsection-name": "subsection-logsearch-server-col1" }, { - "config": "logsearch-properties/logsearch.collection.replication.factor", + "config": "logsearch-properties/logsearch.collection.audit.logs.replication.factor", "subsection-name": "subsection-logsearch-server-col1" } ] @@ -137,7 +145,29 @@ } }, { - "config": "logsearch-properties/logsearch.collection.numshards", + "config": "logsearch-properties/logsearch.collection.service.logs.numshards", + "widget": { + "type": "slider", + "units": [ + { + "unit-name": "int" + } + ] + } + }, + { + "config": "logsearch-properties/logsearch.collection.service.logs.replication.factor", + "widget": { + "type": "slider", + "units": [ + { + "unit-name": "int" + } + ] + } + }, + { + "config": "logsearch-properties/logsearch.collection.audit.logs.numshards", "widget": { "type": "slider", "units": [ @@ -148,7 +178,7 @@ } }, { - "config": "logsearch-properties/logsearch.collection.replication.factor", + "config": "logsearch-properties/logsearch.collection.audit.logs.replication.factor", "widget": { "type": "slider", "units": [ http://git-wip-us.apache.org/repos/asf/ambari/blob/3332dad4/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 73e9044..36d6e9f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -690,16 +690,23 @@ class HDP23StackAdvisor(HDP22StackAdvisor): if logsearchSolrHosts is not None and len(logsearchSolrHosts) > 0 \ and "logsearch-properties" in services["configurations"]: + recommendedMinShards = len(logsearchSolrHosts) + recommendedShards = 2 * len(logsearchSolrHosts) + recommendedMaxShards = 3 * len(logsearchSolrHosts) # recommend number of shard putLogsearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties") - putLogsearchAttribute('logsearch.collection.numshards', 'minimum', len(logsearchSolrHosts)) - putLogsearchAttribute('logsearch.collection.numshards', 'maximum', 3 * len(logsearchSolrHosts)) - putLogsearchProperty("logsearch.collection.numshards", 2 * len(logsearchSolrHosts)) + putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards) + putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards) + putLogsearchProperty("logsearch.collection.service.logs.numshards", recommendedShards) + + putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards) + putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards) + putLogsearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards) # recommend replication factor replicationReccomendFloat = math.log(len(logsearchSolrHosts), 5) recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat)) - putLogsearchProperty("logsearch.collection.replication.factor", recommendedReplicationFactor) - + putLogsearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor) + putLogsearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor) def getServiceConfigurationValidators(self): parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators() http://git-wip-us.apache.org/repos/asf/ambari/blob/3332dad4/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 95be86b..9511738 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -2937,8 +2937,10 @@ class TestHDP23StackAdvisor(TestCase): configurations = { "logsearch-properties": { "properties": { - "logsearch.collection.numshards" : "5", - "logsearch.collection.replication.factor": "0" + "logsearch.collection.service.logs.numshards" : "5", + "logsearch.collection.service.logs.replication.factor": "0", + "logsearch.collection.audit.logs.numshards" : "5", + "logsearch.collection.audit.logs.replication.factor": "0" } } } @@ -2954,11 +2956,17 @@ class TestHDP23StackAdvisor(TestCase): expected = { 'logsearch-properties': { 'properties': { - "logsearch.collection.numshards" : "2", - "logsearch.collection.replication.factor": "1" + "logsearch.collection.service.logs.numshards" : "2", + "logsearch.collection.service.logs.replication.factor": "1", + "logsearch.collection.audit.logs.numshards" : "2", + "logsearch.collection.audit.logs.replication.factor": "1" }, "property_attributes": { - "logsearch.collection.numshards": { + "logsearch.collection.service.logs.numshards": { + "minimum": "1", + "maximum": "3" + }, + "logsearch.collection.audit.logs.numshards": { "minimum": "1", "maximum": "3" }
