AMBARI-19756. Add common log rotation settings to hive log4j2 (Madhuvanthi Radhakrishnan via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c977113c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c977113c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c977113c Branch: refs/heads/branch-dev-patch-upgrade Commit: c977113ccb18218befbb86479353f60bda4eb1d7 Parents: 1eb4ca5 Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Sun Jan 29 22:48:19 2017 -0800 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Sun Jan 29 22:48:19 2017 -0800 ---------------------------------------------------------------------- .../package/scripts/hive_interactive.py | 4 +-- .../0.12.0.2.0/package/scripts/params_linux.py | 8 ++++++ .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 20 ++++++++++++++ .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 12 +++++++++ .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 12 +++++++++ .../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 12 +++++++++ .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 11 ++++++++ .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 11 ++++++++ .../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 11 ++++++++ .../stacks/HDP/2.4/upgrades/config-upgrade.xml | 20 ++++++++++++++ .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 14 ++++++++++ .../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 12 +++++++++ .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 11 ++++++++ .../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 11 ++++++++ .../services/HIVE/configuration/hive-log4j2.xml | 25 ++++++++++++++++- .../HIVE/configuration/llap-cli-log4j2.xml | 25 ++++++++++++++++- .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 28 ++++++++++++++++++++ .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 14 +++++++++- .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 6 +++++ .../stacks/2.5/HIVE/test_hive_server_int.py | 8 +++--- 20 files changed, 266 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py index 335a850..760752f 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py @@ -234,14 +234,14 @@ def hive_interactive(name=None): mode=mode_identified, group=params.user_group, owner=params.hive_user, - content=params.llap_cli_log4j2) + content=InlineTemplate(params.llap_cli_log4j2)) hive_log4j2_filename = 'hive-log4j2.properties' File(format("{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, - content=params.hive_log4j2) + content=InlineTemplate(params.hive_log4j2)) hive_exec_log4j2_filename = 'hive-exec-log4j2.properties' File(format("{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"), http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 4907f93..adac228 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -603,6 +603,14 @@ has_hive_interactive = len(hive_interactive_hosts) > 0 hive_llap_log_maxfilesize = default('/configurations/llap-daemon-log4j/hive_llap_log_maxfilesize', 256) hive_llap_log_maxbackupindex = default('/configurations/llap-daemon-log4j/hive_llap_log_maxbackupindex', 240) +#hive log4j2 properties +hive2_log_maxfilesize = default('/configurations/hive-log4j2/hive2_log_maxfilesize', 256) +hive2_log_maxbackupindex = default('/configurations/hive-log4j2/hive2_log_maxbackupindex', 30) + +#llap cli log4j2 properties +llap_cli_log_maxfilesize = default('/configurations/llap-cli-log4j2/llap_cli_log_maxfilesize', 256) +llap_cli_log_maxbackupindex = default('/configurations/llap-cli-log4j2/llap_cli_log_maxbackupindex', 30) + if has_hive_interactive: llap_daemon_log4j = config['configurations']['llap-daemon-log4j']['content'] llap_cli_log4j2 = config['configurations']['llap-cli-log4j2']['content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index efa6e70..1f3b5ad 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -191,9 +191,29 @@ <regex-replace key="content" find="property.llap.daemon.log.maxfilesize = ([0-9]+)MB" replace-with="property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB"/> <regex-replace key="content" find="property.llap.daemon.log.maxbackupindex = ([0-9]+)" replace-with="property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}"/> </definition> + </changes> + </component> + <component name="HIVE_SERVER_INTERACTIVE"> + <changes> + <definition xsi:type="configure" id="hive_log4j2_parameterize" summary="Parameterizing Hive Log4J2 Properties"> + <type>hive-log4j2</type> + <set key="hive2_log_maxfilesize" value="256"/> + <set key = "hive2_log_maxbackupindex" value="30"/> + <regex-replace key="content" find="appender.DRFA.strategy.max = ([0-9]+)" replace-with="appender.DRFA.strategy.max = {{hive2_log_maxbackupindex}}"/> + <replace key="content" find="appender.DRFA.strategy.type = DefaultRolloverStrategy" replace-with="appender.DRFA.strategy.type = DefaultRolloverStrategy
appender.DRFA.policies.fsize.type = SizeBasedTriggeringPolicy
appender.DRFA.policies.fsize.size = {{hive2_log_maxfilesize}}MB"/> + </definition> + + <definition xsi:type="configure" id="llap_cli_log4j2_parameterize" summary="Parameterizing LLAP Cli Log4J2 Properties"> + <type>llap-cli-log4j2</type> + <set key="llap_cli_log_maxfilesize" value="256"/> + <set key = "llap_cli_log_maxbackupindex" value="30"/> + <regex-replace key="content" find="appender.DRFA.strategy.max = ([0-9]+)" replace-with="appender.DRFA.strategy.max = {{llap_cli_log_maxbackupindex}}"/> + <replace key="content" find="appender.DRFA.strategy.type = DefaultRolloverStrategy" replace-with="appender.DRFA.strategy.type = DefaultRolloverStrategy
appender.DRFA.policies.fsize.type = SizeBasedTriggeringPolicy
appender.DRFA.policies.fsize.size = {{llap_cli_log_maxfilesize}}MB"/> + </definition> </changes> </component> + <component name="WEBHCAT_SERVER"> <changes> <definition xsi:type="configure" id="hdp_2_4_0_0_webhcat_server_update_configuration_paths" summary="Updating Configuration Paths"> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml index 619fa5d..9e13036 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml @@ -338,6 +338,18 @@ <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Parameterizing LLAP Cli Log4J2 Properties"> + <task xsi:type="configure" id="llap_cli_log4j2_parameterize"> + <summary>Updating the LLAP Cli Log4J2 properties to include parameterizations</summary> + </task> + </execute-stage> + + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Parameterizing Hive Log4J2 Properties"> + <task xsi:type="configure" id="hive_log4j2_parameterize"> + <summary>Updating the Hive Log4J2 properties to include parameterizations</summary> + </task> + </execute-stage> + <!--OOZIE--> <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml index b6d98aa..a3407e0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml @@ -374,6 +374,18 @@ </task> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Parameterizing LLAP Cli Log4J2 Properties"> + <task xsi:type="configure" id="llap_cli_log4j2_parameterize"> + <summary>Updating the LLAP Cli Log4J2 properties to include parameterizations</summary> + </task> + </execute-stage> + + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Parameterizing Hive Log4J2 Properties"> + <task xsi:type="configure" id="hive_log4j2_parameterize"> + <summary>Updating the Hive Log4J2 properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties"> <task xsi:type="configure" id="webhcat_log4j_parameterize"> <summary>Updating the Webhcat Log4J properties to include parameterizations</summary> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml index 03fd683..a9b1480 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml @@ -400,6 +400,18 @@ </task> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Parameterizing LLAP Cli Log4J2 Properties"> + <task xsi:type="configure" id="llap_cli_log4j2_parameterize"> + <summary>Updating the LLAP Cli Log4J2 properties to include parameterizations</summary> + </task> + </execute-stage> + + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Parameterizing Hive Log4J2 Properties"> + <task xsi:type="configure" id="hive_log4j2_parameterize"> + <summary>Updating the Hive Log4J2 properties to include parameterizations</summary> + </task> + </execute-stage> + <!--OOZIE--> <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml index 743dd1b..57f7984 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml @@ -789,6 +789,17 @@ <task xsi:type="restart-task" /> </upgrade> </component> + + <component name="HIVE_SERVER_INTERACTIVE"> + <pre-upgrade> + <task xsi:type="configure" id="hive_log4j2_parameterize"/> + <task xsi:type="configure" id="llap_cli_log4j2_parameterize"/> + </pre-upgrade> + <pre-downgrade/> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> </service> <service name="SLIDER"> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml index 8db7669..6d45362 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml @@ -916,6 +916,17 @@ <task xsi:type="restart-task" /> </upgrade> </component> + + <component name="HIVE_SERVER_INTERACTIVE"> + <pre-upgrade> + <task xsi:type="configure" id="hive_log4j2_parameterize"/> + <task xsi:type="configure" id="llap_cli_log4j2_parameterize"/> + </pre-upgrade> + <pre-downgrade/> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> </service> <service name="SLIDER"> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml index 981947f..a65b32c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml @@ -917,6 +917,17 @@ <task xsi:type="restart-task" /> </upgrade> </component> + + <component name="HIVE_SERVER_INTERACTIVE"> + <pre-upgrade> + <task xsi:type="configure" id="hive_log4j2_parameterize"/> + <task xsi:type="configure" id="llap_cli_log4j2_parameterize"/> + </pre-upgrade> + <pre-downgrade/> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> </service> <service name="SLIDER"> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml index 0a6ea5e..7179099 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml @@ -92,6 +92,26 @@ <regex-replace key="content" find="property.llap.daemon.log.maxfilesize = ([0-9]+)MB" replace-with="property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB"/> <regex-replace key="content" find="property.llap.daemon.log.maxbackupindex = ([0-9]+)" replace-with="property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}"/> </definition> + </changes> + </component> + + <component name="HIVE_SERVER_INTERACTIVE"> + <changes> + <definition xsi:type="configure" id="hive_log4j2_parameterize" summary="Parameterizing Hive Log4J2 Properties"> + <type>hive-log4j2</type> + <set key="hive2_log_maxfilesize" value="256"/> + <set key = "hive2_log_maxbackupindex" value="30"/> + <regex-replace key="content" find="appender.DRFA.strategy.max = ([0-9]+)" replace-with="appender.DRFA.strategy.max = {{hive2_log_maxbackupindex}}"/> + <replace key="content" find="appender.DRFA.strategy.type = DefaultRolloverStrategy" replace-with="appender.DRFA.strategy.type = DefaultRolloverStrategy
appender.DRFA.policies.fsize.type = SizeBasedTriggeringPolicy
appender.DRFA.policies.fsize.size = {{hive2_log_maxfilesize}}MB"/> + </definition> + + <definition xsi:type="configure" id="llap_cli_log4j2_parameterize" summary="Parameterizing LLAP Cli Log4J2 Properties"> + <type>llap-cli-log4j2</type> + <set key="llap_cli_log_maxfilesize" value="256"/> + <set key = "llap_cli_log_maxbackupindex" value="30"/> + <regex-replace key="content" find="appender.DRFA.strategy.max = ([0-9]+)" replace-with="appender.DRFA.strategy.max = {{llap_cli_log_maxbackupindex}}"/> + <replace key="content" find="appender.DRFA.strategy.type = DefaultRolloverStrategy" replace-with="appender.DRFA.strategy.type = DefaultRolloverStrategy
appender.DRFA.policies.fsize.type = SizeBasedTriggeringPolicy
appender.DRFA.policies.fsize.size = {{llap_cli_log_maxfilesize}}MB"/> + </definition> </changes> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml index d41c4eb..67e9314 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml @@ -428,17 +428,31 @@ <summary>Updating the Hive Log4J properties to include parameterizations</summary> </task> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive LLAP Log4J Properties"> <task xsi:type="configure" id="hive_llap_log4j_parameterize"> <summary>Updating the Hive llap Log4J properties to include parameterizations</summary> </task> </execute-stage> + <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties"> <task xsi:type="configure" id="webhcat_log4j_parameterize"> <summary>Updating the Webhcat Log4J properties to include parameterizations</summary> </task> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Parameterizing LLAP Cli Log4J2 Properties"> + <task xsi:type="configure" id="llap_cli_log4j2_parameterize"> + <summary>Updating the LLAP Cli Log4J2 properties to include parameterizations</summary> + </task> + </execute-stage> + + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Parameterizing Hive Log4J2 Properties"> + <task xsi:type="configure" id="hive_log4j2_parameterize"> + <summary>Updating the Hive Log4J2 properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- HBASE --> <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for Hbase Master"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml index e5c51ad..7b9b062 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml @@ -440,6 +440,18 @@ </task> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Parameterizing LLAP Cli Log4J2 Properties"> + <task xsi:type="configure" id="llap_cli_log4j2_parameterize"> + <summary>Updating the LLAP Cli Log4J2 properties to include parameterizations</summary> + </task> + </execute-stage> + + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Parameterizing Hive Log4J2 Properties"> + <task xsi:type="configure" id="hive_log4j2_parameterize"> + <summary>Updating the Hive Log4J2 properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- HBASE --> <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for Hbase Master"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml index d2c6ff5..1ee9ea2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml @@ -907,6 +907,17 @@ <task xsi:type="restart-task" /> </upgrade> </component> + + <component name="HIVE_SERVER_INTERACTIVE"> + <pre-upgrade> + <task xsi:type="configure" id="hive_log4j2_parameterize"/> + <task xsi:type="configure" id="llap_cli_log4j2_parameterize"/> + </pre-upgrade> + <pre-downgrade/> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> </service> <service name="SLIDER"> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml index 8b463dd..781149c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml @@ -913,6 +913,17 @@ <task xsi:type="restart-task" /> </upgrade> </component> + + <component name="HIVE_SERVER_INTERACTIVE"> + <pre-upgrade> + <task xsi:type="configure" id="hive_log4j2_parameterize"/> + <task xsi:type="configure" id="llap_cli_log4j2_parameterize"/> + </pre-upgrade> + <pre-downgrade/> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> </service> <service name="SLIDER"> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-log4j2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-log4j2.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-log4j2.xml index abf307a..01d556f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-log4j2.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-log4j2.xml @@ -21,6 +21,27 @@ --> <configuration supports_final="false" supports_adding_forbidden="false"> <property> + <name>hive2_log_maxfilesize</name> + <value>256</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Hive Log2: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive2_log_maxbackupindex</name> + <value>30</value> + <description>The number of backup files</description> + <display-name>Hive Log2: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>content</name> <display-name>hive-log4j2 template</display-name> <description>Custom hive-log4j2.properties</description> @@ -74,7 +95,9 @@ appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy appender.DRFA.policies.time.interval = 1 appender.DRFA.policies.time.modulate = true appender.DRFA.strategy.type = DefaultRolloverStrategy -appender.DRFA.strategy.max = 30 +appender.DRFA.strategy.max = {{hive2_log_maxbackupindex}} +appender.DRFA.policies.fsize.type = SizeBasedTriggeringPolicy +appender.DRFA.policies.fsize.size = {{hive2_log_maxfilesize}}MB # list of all loggers loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-cli-log4j2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-cli-log4j2.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-cli-log4j2.xml index 67f5830..ff93265 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-cli-log4j2.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-cli-log4j2.xml @@ -21,6 +21,27 @@ --> <configuration supports_final="false" supports_adding_forbidden="false"> <property> + <name>llap_cli_log_maxfilesize</name> + <value>256</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>LLAP Client Log2: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>llap_cli_log_maxbackupindex</name> + <value>30</value> + <description>The number of backup files</description> + <display-name>LLAP Client Log2: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>content</name> <display-name>llap-cli-log4j2 template</display-name> <description>Custom llap-cli-log4j2.properties</description> @@ -74,7 +95,9 @@ appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy appender.DRFA.policies.time.interval = 1 appender.DRFA.policies.time.modulate = true appender.DRFA.strategy.type = DefaultRolloverStrategy -appender.DRFA.strategy.max = 30 +appender.DRFA.strategy.max = {{llap_cli_log_maxbackupindex}} +appender.DRFA.policies.fsize.type = SizeBasedTriggeringPolicy +appender.DRFA.policies.fsize.size = {{llap_cli_log_maxfilesize}}MB # list of all loggers loggers = ZooKeeper, DataNucleus, Datastore, JPOX, HadoopConf http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index ddffc79..4bda941 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -315,6 +315,34 @@ <type>hive-interactive-env</type> <insert key="llap_java_opts" value="{{heap_dump_opts}}" insert-type="append" newline-before="false" newline-after="false" /> </definition> + + <definition xsi:type="configure" id="hive_log4j2_parameterize" summary="Parameterizing Hive Log4J2 Properties"> + <type>hive-log4j2</type> + <set key="hive2_log_maxfilesize" value="256"/> + <set key = "hive2_log_maxbackupindex" value="30"/> + <regex-replace key="content" find="appender.DRFA.strategy.max = ([0-9]+)" replace-with="appender.DRFA.strategy.max = {{hive2_log_maxbackupindex}}"/> + <replace key="content" find="appender.DRFA.strategy.type = DefaultRolloverStrategy" replace-with="appender.DRFA.strategy.type = DefaultRolloverStrategy
appender.DRFA.policies.fsize.type = SizeBasedTriggeringPolicy
appender.DRFA.policies.fsize.size = {{hive2_log_maxfilesize}}MB"/> + </definition> + + <definition xsi:type="configure" id="llap_cli_log4j2_parameterize" summary="Parameterizing LLAP Cli Log4J2 Properties"> + <type>llap-cli-log4j2</type> + <set key="llap_cli_log_maxfilesize" value="256"/> + <set key = "llap_cli_log_maxbackupindex" value="30"/> + <regex-replace key="content" find="appender.DRFA.strategy.max = ([0-9]+)" replace-with="appender.DRFA.strategy.max = {{llap_cli_log_maxbackupindex}}"/> + <replace key="content" find="appender.DRFA.strategy.type = DefaultRolloverStrategy" replace-with="appender.DRFA.strategy.type = DefaultRolloverStrategy
appender.DRFA.policies.fsize.type = SizeBasedTriggeringPolicy
appender.DRFA.policies.fsize.size = {{llap_cli_log_maxfilesize}}MB"/> + </definition> + </changes> + + </component> + <component name = "WEBHCAT_SERVER"> + <changes> + <definition xsi:type="configure" id="webhcat_log4j_parameterize" summary="Parameterizing Webhcat Log4J Properties"> + <type>webhcat-log4j</type> + <set key="webhcat_log_maxfilesize" value="256"/> + <set key = "webhcat_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxFileSize = {{webhcat_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxBackupIndex = {{webhcat_log_maxbackupindex}}"/> + </definition> </changes> </component> <component name = "WEBHCAT_SERVER"> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index ba33a7d..8092506 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -444,7 +444,19 @@ <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Appending Java heap dump options for HiveSever2 Interactive"> <task xsi:type="configure" id="hdp_2_6_0_0_hive_llap_append_java_heap_dump_options"/> - </execute-stage> + </execute-stage> + + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Parameterizing LLAP Cli Log4J2 Properties"> + <task xsi:type="configure" id="llap_cli_log4j2_parameterize"> + <summary>Updating the LLAP Cli Log4J2 properties to include parameterizations</summary> + </task> + </execute-stage> + + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Parameterizing Hive Log4J2 Properties"> + <task xsi:type="configure" id="hive_log4j2_parameterize"> + <summary>Updating the Hive Log4J2 properties to include parameterizations</summary> + </task> + </execute-stage> </group> <!-- http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index 790e50c..a739551 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -783,6 +783,8 @@ <task xsi:type="configure" id="hdp_2_6_0_0_hive_llap_append_java_heap_dump_options"/> <task xsi:type="configure" id="hdp_2_6_0_0_hive_llap_append_heap_dump_options"/> <task xsi:type="configure" id="webhcat_log4j_parameterize" /> + <task xsi:type="configure" id="hive_log4j2_parameterize"/> + <task xsi:type="configure" id="llap_cli_log4j2_parameterize"/> </pre-upgrade> <pre-downgrade /> @@ -793,6 +795,10 @@ </component> <component name="WEBHCAT_SERVER"> + <pre-upgrade> + <task xsi:type="configure" id="webhcat_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/c977113c/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py index 2bc28e5..fb97612 100644 --- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py @@ -566,13 +566,13 @@ class TestHiveServerInteractive(RMFTestCase): mode=0600, ) self.assertResourceCalled('File', os.path.join(conf_dir, 'llap-cli-log4j2.properties'), - content='con\ntent', + content=InlineTemplate('con\ntent'), owner='hive', group='hadoop', mode=0600, ) self.assertResourceCalled('File', os.path.join(conf_dir, 'hive-log4j2.properties'), - content='con\ntent', # Test new line + content=InlineTemplate('con\ntent'), # Test new line owner='hive', group='hadoop', mode=0600, @@ -631,13 +631,13 @@ class TestHiveServerInteractive(RMFTestCase): mode=0644, ) self.assertResourceCalled('File', os.path.join(conf_dir, 'llap-cli-log4j2.properties'), - content='con\ntent', + content=InlineTemplate('con\ntent'), owner='hive', group='hadoop', mode=0644, ) self.assertResourceCalled('File', os.path.join(conf_dir, 'hive-log4j2.properties'), - content='con\ntent', # Test new line + content=InlineTemplate('con\ntent'), # Test new line owner='hive', group='hadoop', mode=0644,