AMBARI-19660. Add log rotation settings - handle upgrade scenario - Storm, Hive (Madhuvanthi Radhakrishnan via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/78fefdd4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/78fefdd4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/78fefdd4 Branch: refs/heads/branch-dev-patch-upgrade Commit: 78fefdd4c327390062d16a15594b32bb816f65cf Parents: 1b630eb Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Tue Jan 24 14:05:34 2017 -0800 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Tue Jan 24 14:08:16 2017 -0800 ---------------------------------------------------------------------- .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 46 +++++++++++++++++++ .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 29 +++++++++++- .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 29 ++++++++++++ .../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 30 ++++++++++++- .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 6 ++- .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 5 +++ .../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 45 ++++++++++++++++++- .../stacks/HDP/2.4/upgrades/config-upgrade.xml | 44 ++++++++++++++++++ .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 28 ++++++++++++ .../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 27 +++++++++++ .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 9 +++- .../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 8 ++++ .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 47 ++++++++++++++++++++ .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 30 ++++++++++++- .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 8 ++++ 15 files changed, 384 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index c9adb8c..efa6e70 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -177,6 +177,21 @@ <transfer operation="delete" delete-key="atlas.hook.hive.maxThreads" /> </definition> + <definition xsi:type="configure" id="hive_log4j_parameterize" summary="Parameterizing Hive Log4J Properties"> + <type>hive-log4j</type> + <set key="hive_log_maxfilesize" value="256"/> + <set key = "hive_log_maxbackupindex" value="30"/> + <regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/> + <replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB"/> + </definition> + <definition xsi:type="configure" id="hive_llap_log4j_parameterize" summary="Parameterizing Hive llap Log4J Properties"> + <type>llap-daemon-log4j</type> + <set key="hive_llap_log_maxfilesize" value="256"/> + <set key = "hive_llap_log_maxbackupindex" value="240"/> + <regex-replace key="content" find="property.llap.daemon.log.maxfilesize = ([0-9]+)MB" replace-with="property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB"/> + <regex-replace key="content" find="property.llap.daemon.log.maxbackupindex = ([0-9]+)" replace-with="property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}"/> + </definition> + </changes> </component> <component name="WEBHCAT_SERVER"> @@ -198,6 +213,14 @@ <replace key="templeton.hcat" find="/usr/hdp/current/hive-client" replace-with="/usr/hdp/${hdp.version}/hive"/> <set key="templeton.hive.extra.files" value="/usr/hdp/${hdp.version}/tez/conf/tez-site.xml,/usr/hdp/${hdp.version}/tez,/usr/hdp/${hdp.version}/tez/lib"/> </definition> + + <definition xsi:type="configure" id="webhcat_log4j_parameterize" summary="Parameterizing Webhcat Log4J Properties"> + <type>webhcat-log4j</type> + <set key="webhcat_log_maxfilesize" value="256"/> + <set key = "webhcat_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxFileSize = {{webhcat_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxBackupIndex = {{webhcat_log_maxbackupindex}}"/> + </definition> </changes> </component> </service> @@ -547,6 +570,29 @@ if-type="storm-site" if-value="15000" /> </definition> + + <definition xsi:type="configure" id="storm_worker_log4j_parameterize" summary="Parameterizing Storm Worker Log4J Properties"> + <type>storm-worker-log4j</type> + <set key="storm_wrkr_a1_maxfilesize" value="100"/> + <set key="storm_wrkr_a1_maxbackupindex" value="9"/> + <set key="storm_wrkr_out_maxfilesize" value="100"/> + <set key="storm_wrkr_out_maxbackupindex" value="4"/> + <set key="storm_wrkr_err_maxfilesize" value="100"/> + <set key="storm_wrkr_err_maxbackupindex" value="4"/> + <regex-replace key="content" find="A1"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)" + replace-with="A1"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/> + <regex-replace key="content" find="STDOUT"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)" + replace-with="STDOUT"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/> + <regex-replace key="content" find="STDERR"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)" + replace-with="STDERR"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/> + </definition> + <definition xsi:type="configure" id="storm_cluster_log4j_parameterize" summary="Parameterizing Storm Cluster Log4J Properties"> + <type>storm-cluster-log4j</type> + <set key="storm_a1_maxfilesize" value="100"/> + <set key="storm_a1_maxbackupindex" value="9"/> + <regex-replace key="content" find="A1" immediateFlush="false"
 fileName="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
 filePattern="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)" + replace-with="A1" immediateFlush="false"
 fileName="${sys:storm.log.dir}/${sys:logfile.name}"
 filePattern="${sys:storm.log.dir}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml index d674af7..619fa5d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml @@ -318,6 +318,22 @@ <task xsi:type="configure" id="hdp_2_4_0_0_webhcat_server_update_configuration_paths"/> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties"> + <task xsi:type="configure" id="hive_log4j_parameterize"> + <summary>Updating the Hive Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive LLAP Log4J Properties"> + <task xsi:type="configure" id="hive_llap_log4j_parameterize"> + <summary>Updating the Hive llap Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties"> + <task xsi:type="configure" id="webhcat_log4j_parameterize"> + <summary>Updating the Webhcat Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server"> <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/> </execute-stage> @@ -368,7 +384,18 @@ <task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/> </execute-stage> - <!--ZOOKEEPER--> + <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm worker Log4J Properties"> + <task xsi:type="configure" id="storm_worker_log4j_parameterize"> + <summary>Updating the Storm worker Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm cluster Log4J Properties"> + <task xsi:type="configure" id="storm_cluster_log4j_parameterize"> + <summary>Updating the Storm cluster Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!--ZOOKEEPER--> <execute-stage service="ZOOKEEPER" component="ZOOKEEPER_SERVER" title="Parameterizing Zookeeper Log4J Properties"> <task xsi:type="configure" id="zookeeper_log4j_parameterize"> <summary>Updating the Zookeeper Log4J properties to include parameterizations</summary> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml index 7a05c99..b6d98aa 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml @@ -362,6 +362,24 @@ <task xsi:type="configure" id="hdp_2_5_0_0_webhcat_server_update_configuration_paths"/> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties"> + <task xsi:type="configure" id="hive_log4j_parameterize"> + <summary>Updating the Hive Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive LLAP Log4J Properties"> + <task xsi:type="configure" id="hive_llap_log4j_parameterize"> + <summary>Updating the Hive llap Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties"> + <task xsi:type="configure" id="webhcat_log4j_parameterize"> + <summary>Updating the Webhcat Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server"> <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/> </execute-stage> @@ -552,6 +570,17 @@ <task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/> </execute-stage> + <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm worker Log4J Properties"> + <task xsi:type="configure" id="storm_worker_log4j_parameterize"> + <summary>Updating the Storm worker Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm cluster Log4J Properties"> + <task xsi:type="configure" id="storm_cluster_log4j_parameterize"> + <summary>Updating the Storm cluster Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!--ATLAS--> <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties"> <task xsi:type="configure" id="atlas_log4j_parameterize"> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml index 2c2049b..03fd683 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml @@ -364,6 +364,12 @@ <task xsi:type="configure" id="hdp_2_5_0_0_webhcat_server_update_configuration_paths"/> </execute-stage> + <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties"> + <task xsi:type="configure" id="webhcat_log4j_parameterize"> + <summary>Updating the Webhcat Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server"> <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/> </execute-stage> @@ -383,6 +389,17 @@ </task> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties"> + <task xsi:type="configure" id="hive_log4j_parameterize"> + <summary>Updating the Hive Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive LLAP Log4J Properties"> + <task xsi:type="configure" id="hive_llap_log4j_parameterize"> + <summary>Updating the Hive llap Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!--OOZIE--> <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"> @@ -560,7 +577,18 @@ <task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/> </execute-stage> - <!--ATLAS--> + <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm worker Log4J Properties"> + <task xsi:type="configure" id="storm_worker_log4j_parameterize"> + <summary>Updating the Storm worker Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm cluster Log4J Properties"> + <task xsi:type="configure" id="storm_cluster_log4j_parameterize"> + <summary>Updating the Storm cluster Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!--ATLAS--> <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties"> <task xsi:type="configure" id="atlas_log4j_parameterize"> <summary>Updating the Atlas Log4J properties to include parameterizations</summary> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml index a37d171..743dd1b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml @@ -754,6 +754,8 @@ <component name="HIVE_SERVER"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/> + <task xsi:type="configure" id="hive_log4j_parameterize" /> + <task xsi:type="configure" id="hive_llap_log4j_parameterize" /> </pre-upgrade> <pre-downgrade/> @@ -766,6 +768,7 @@ <component name="WEBHCAT_SERVER"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_4_0_0_webhcat_server_update_configuration_paths"/> + <task xsi:type="configure" id="webhcat_log4j_parameterize" /> </pre-upgrade> <pre-downgrade/> @@ -933,7 +936,8 @@ <component name="NIMBUS"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_3_0_0_remove_empty_storm_topology_submission_notifier_plugin_class"/> - + <task xsi:type="configure" id="storm_worker_log4j_parameterize" /> + <task xsi:type="configure" id="storm_cluster_log4j_parameterize" /> <task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/> </pre-upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml index 47f58bd..8db7669 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml @@ -874,6 +874,8 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hive_audit_db" /> + <task xsi:type="configure" id="hive_log4j_parameterize" /> + <task xsi:type="configure" id="hive_llap_log4j_parameterize" /> <!-- Remove Atlas configs that were incorrectly added to hive-site instead of Atlas' application.properties. --> <task xsi:type="configure" id="hdp_2_5_0_0_remove_hive_atlas_configs" /> @@ -893,6 +895,7 @@ <component name="WEBHCAT_SERVER"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_5_0_0_webhcat_server_update_configuration_paths"/> + <task xsi:type="configure" id="webhcat_log4j_parameterize"/> </pre-upgrade> <pre-downgrade /> @@ -1091,6 +1094,8 @@ <function>delete_storm_local_data</function> </task> <task xsi:type="configure" id="hdp_2_5_0_0_add_storm_security_configs" /> + <task xsi:type="configure" id="storm_worker_log4j_parameterize" /> + <task xsi:type="configure" id="storm_cluster_log4j_parameterize" /> </pre-upgrade> <pre-downgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml index 8101682..981947f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml @@ -559,6 +559,10 @@ <processing> <service name="ZOOKEEPER"> <component name="ZOOKEEPER_SERVER"> + <pre-upgrade> + <task xsi:type="configure" id="zookeeper_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade /> <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -612,6 +616,7 @@ <script>scripts/ranger_admin.py</script> <function>setup_ranger_java_patches</function> </task> + <task xsi:type="configure" id="admin_log4j_parameterize" /> </pre-upgrade> <pre-downgrade/> <!-- no-op to prevent config changes on downgrade --> @@ -623,6 +628,19 @@ </component> <component name="RANGER_USERSYNC"> + <pre-upgrade> + <task xsi:type="configure" id="usersync_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade /> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + <component name="RANGER_TAGSYNC"> + <pre-upgrade> + <task xsi:type="configure" id="tagsync_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade /> <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -633,7 +651,7 @@ <component name="RANGER_KMS_SERVER"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db" /> - + <task xsi:type="configure" id="kms_log4j_parameterize" /> <task xsi:type="execute" hosts="any" sequential="true"> <summary>Upgrading Ranger KMS database schema</summary> <script>scripts/kms_server.py</script> @@ -660,6 +678,7 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_4_0_0_namenode_ha_adjustments"/> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" /> + <task xsi:type="configure" id="hdfs_log4j_parameterize" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> @@ -739,6 +758,7 @@ <summary>Calculating Yarn Properties for Spark Shuffle</summary> </task> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db" /> + <task xsi:type="configure" id="yarn_log4j_parameterize" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> @@ -772,6 +792,7 @@ <!-- These HBASE configs changed in HDP 2.3.4.0, but Ambari can't distinguish HDP 2.3.2.0 vs HDP 2.3.4.0, so easier to always do them. --> <task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db" /> + <task xsi:type="configure" id="hbase_log4j_parameterize" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> @@ -861,6 +882,8 @@ <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HiveEnvClasspathAction"> <summary>Update hive-env content</summary> </task> + <task xsi:type="configure" id="hive_log4j_parameterize" /> + <task xsi:type="configure" id="hive_llap_log4j_parameterize" /> </pre-upgrade> <pre-downgrade/> @@ -873,6 +896,7 @@ <component name="WEBHCAT_SERVER"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_5_0_0_webhcat_server_update_configuration_paths"/> + <task xsi:type="configure" id="webhcat_log4j_parameterize" /> </pre-upgrade> <pre-downgrade /> @@ -946,7 +970,7 @@ <component name="OOZIE_SERVER"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" /> - + <task xsi:type="configure" id="oozie_log4j_parameterize" /> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"> <summary>Adjusting Oozie properties</summary> </task> @@ -1000,6 +1024,7 @@ <component name="FALCON_SERVER"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"/> + <task xsi:type="configure" id="falcon_log4j_parameterize" /> </pre-upgrade> <pre-downgrade/> <upgrade> @@ -1018,6 +1043,7 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db" /> <task xsi:type="configure" id="hdp_2_5_0_0_add_protocol_compatibility" /> + <task xsi:type="configure" id="kafka_log4j_parameterize" /> </pre-upgrade> <pre-downgrade/> @@ -1032,6 +1058,8 @@ <component name="KNOX_GATEWAY"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_knox_audit_db" /> + <task xsi:type="configure" id="knox_ldap_log4j_parameterize" /> + <task xsi:type="configure" id="knox_gateway_log4j_parameterize" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> @@ -1067,6 +1095,8 @@ <function>delete_storm_local_data</function> </task> <task xsi:type="configure" id="hdp_2_5_0_0_add_storm_security_configs" /> + <task xsi:type="configure" id="storm_worker_log4j_parameterize" /> + <task xsi:type="configure" id="storm_cluster_log4j_parameterize" /> </pre-upgrade> <pre-downgrade> @@ -1192,5 +1222,16 @@ </upgrade> </component> </service> + <service name="ATLAS"> + <component name="ATLAS_SERVER"> + <pre-upgrade> + <task xsi:type="configure" id="atlas_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade /> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + </service> </processing> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml index 4451306..0a6ea5e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml @@ -78,6 +78,21 @@ <transfer operation="delete" delete-key="atlas.hook.hive.maxThreads" /> </definition> + <definition xsi:type="configure" id="hive_log4j_parameterize" summary="Parameterizing Hive Log4J Properties"> + <type>hive-log4j</type> + <set key="hive_log_maxfilesize" value="256"/> + <set key = "hive_log_maxbackupindex" value="30"/> + <regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/> + <replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB"/> + </definition> + <definition xsi:type="configure" id="hive_llap_log4j_parameterize" summary="Parameterizing Hive llap Log4J Properties"> + <type>llap-daemon-log4j</type> + <set key="hive_llap_log_maxfilesize" value="256"/> + <set key = "hive_llap_log_maxbackupindex" value="240"/> + <regex-replace key="content" find="property.llap.daemon.log.maxfilesize = ([0-9]+)MB" replace-with="property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB"/> + <regex-replace key="content" find="property.llap.daemon.log.maxbackupindex = ([0-9]+)" replace-with="property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}"/> + </definition> + </changes> </component> @@ -91,6 +106,13 @@ <replace key="templeton.hcat" find="/usr/hdp/current/hive-client" replace-with="/usr/hdp/${hdp.version}/hive"/> <set key="templeton.hive.extra.files" value="/usr/hdp/${hdp.version}/tez/conf/tez-site.xml,/usr/hdp/${hdp.version}/tez,/usr/hdp/${hdp.version}/tez/lib"/> </definition> + <definition xsi:type="configure" id="webhcat_log4j_parameterize" summary="Parameterizing Webhcat Log4J Properties"> + <type>webhcat-log4j</type> + <set key="webhcat_log_maxfilesize" value="256"/> + <set key = "webhcat_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxFileSize = {{webhcat_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxBackupIndex = {{webhcat_log_maxbackupindex}}"/> + </definition> </changes> </component> </service> @@ -381,6 +403,28 @@ if-type="storm-site" if-value="15000" /> </definition> + <definition xsi:type="configure" id="storm_worker_log4j_parameterize" summary="Parameterizing Storm Worker Log4J Properties"> + <type>storm-worker-log4j</type> + <set key="storm_wrkr_a1_maxfilesize" value="100"/> + <set key="storm_wrkr_a1_maxbackupindex" value="9"/> + <set key="storm_wrkr_out_maxfilesize" value="100"/> + <set key="storm_wrkr_out_maxbackupindex" value="4"/> + <set key="storm_wrkr_err_maxfilesize" value="100"/> + <set key="storm_wrkr_err_maxbackupindex" value="4"/> + <regex-replace key="content" find="A1"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)" + replace-with="A1"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/> + <regex-replace key="content" find="STDOUT"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)" + replace-with="STDOUT"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/> + <regex-replace key="content" find="STDERR"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)" + replace-with="STDERR"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/> + </definition> + <definition xsi:type="configure" id="storm_cluster_log4j_parameterize" summary="Parameterizing Storm Cluster Log4J Properties"> + <type>storm-cluster-log4j</type> + <set key="storm_a1_maxfilesize" value="100"/> + <set key="storm_a1_maxbackupindex" value="9"/> + <regex-replace key="content" find="A1" immediateFlush="false"
 fileName="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
 filePattern="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)" + replace-with="A1" immediateFlush="false"
 fileName="${sys:storm.log.dir}/${sys:logfile.name}"
 filePattern="${sys:storm.log.dir}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml index 1e4da76..d41c4eb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml @@ -423,6 +423,22 @@ </task> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties"> + <task xsi:type="configure" id="hive_log4j_parameterize"> + <summary>Updating the Hive Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive LLAP Log4J Properties"> + <task xsi:type="configure" id="hive_llap_log4j_parameterize"> + <summary>Updating the Hive llap Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties"> + <task xsi:type="configure" id="webhcat_log4j_parameterize"> + <summary>Updating the Webhcat Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- HBASE --> <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for Hbase Master"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db"/> @@ -479,6 +495,18 @@ <task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/> </execute-stage> + <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm worker Log4J Properties"> + <task xsi:type="configure" id="storm_worker_log4j_parameterize"> + <summary>Updating the Storm worker Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm cluster Log4J Properties"> + <task xsi:type="configure" id="storm_cluster_log4j_parameterize"> + <summary>Updating the Storm cluster Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!-- KAFKA --> <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Apply config changes for Kafka Broker"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml index 647cb45..e5c51ad 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml @@ -424,6 +424,22 @@ </task> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties"> + <task xsi:type="configure" id="hive_log4j_parameterize"> + <summary>Updating the Hive Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive LLAP Log4J Properties"> + <task xsi:type="configure" id="hive_llap_log4j_parameterize"> + <summary>Updating the Hive llap Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties"> + <task xsi:type="configure" id="webhcat_log4j_parameterize"> + <summary>Updating the Webhcat Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- HBASE --> <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for Hbase Master"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db"/> @@ -479,6 +495,17 @@ <task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/> </execute-stage> + <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm worker Log4J Properties"> + <task xsi:type="configure" id="storm_worker_log4j_parameterize"> + <summary>Updating the Storm worker Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm cluster Log4J Properties"> + <task xsi:type="configure" id="storm_cluster_log4j_parameterize"> + <summary>Updating the Storm cluster Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- KAFKA --> <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Apply config changes for Kafka Broker"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml index fac26de..d2c6ff5 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml @@ -875,7 +875,8 @@ <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HiveEnvClasspathAction"> <summary>Update hive-env content</summary> </task> - + <task xsi:type="configure" id="hive_log4j_parameterize" /> + <task xsi:type="configure" id="hive_llap_log4j_parameterize" /> </pre-upgrade> <pre-downgrade/> @@ -886,6 +887,10 @@ </component> <component name="WEBHCAT_SERVER"> + <pre-upgrade> + <task xsi:type="configure" id="webhcat_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -1042,6 +1047,8 @@ <task xsi:type="configure" id="hdp_2_5_0_0_add_storm_security_configs" /> <task xsi:type="configure" id="hdp_2_4_0_0_remove_empty_storm_topology_submission_notifier_plugin_class"/> <task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/> + <task xsi:type="configure" id="storm_worker_log4j_parameterize" /> + <task xsi:type="configure" id="storm_cluster_log4j_parameterize" /> </pre-upgrade> <pre-downgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml index d75cb24..8b463dd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml @@ -880,6 +880,8 @@ <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HiveEnvClasspathAction"> <summary>Update hive-env content</summary> </task> + <task xsi:type="configure" id="hive_log4j_parameterize" /> + <task xsi:type="configure" id="hive_llap_log4j_parameterize" /> </pre-upgrade> @@ -891,6 +893,10 @@ </component> <component name="WEBHCAT_SERVER"> + <pre-upgrade> + <task xsi:type="configure" id="webhcat_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -1047,6 +1053,8 @@ <task xsi:type="configure" id="hdp_2_5_0_0_add_storm_security_configs" /> <task xsi:type="configure" id="hdp_2_4_0_0_remove_empty_storm_topology_submission_notifier_plugin_class"/> <task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/> + <task xsi:type="configure" id="storm_worker_log4j_parameterize" /> + <task xsi:type="configure" id="storm_cluster_log4j_parameterize" /> </pre-upgrade> <pre-downgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 73e3c38..ddffc79 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -41,6 +41,28 @@ if-type="storm-site" if-value="15000" /> </definition> + <definition xsi:type="configure" id="storm_worker_log4j_parameterize" summary="Parameterizing Storm Worker Log4J Properties"> + <type>storm-worker-log4j</type> + <set key="storm_wrkr_a1_maxfilesize" value="100"/> + <set key="storm_wrkr_a1_maxbackupindex" value="9"/> + <set key="storm_wrkr_out_maxfilesize" value="100"/> + <set key="storm_wrkr_out_maxbackupindex" value="4"/> + <set key="storm_wrkr_err_maxfilesize" value="100"/> + <set key="storm_wrkr_err_maxbackupindex" value="4"/> + <regex-replace key="content" find="A1"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)" + replace-with="A1"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/> + <regex-replace key="content" find="STDOUT"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)" + replace-with="STDOUT"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/> + <regex-replace key="content" find="STDERR"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)" + replace-with="STDERR"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/> + </definition> + <definition xsi:type="configure" id="storm_cluster_log4j_parameterize" summary="Parameterizing Storm Cluster Log4J Properties"> + <type>storm-cluster-log4j</type> + <set key="storm_a1_maxfilesize" value="100"/> + <set key="storm_a1_maxbackupindex" value="9"/> + <regex-replace key="content" find="A1" immediateFlush="false"
 fileName="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
 filePattern="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)" + replace-with="A1" immediateFlush="false"
 fileName="${sys:storm.log.dir}/${sys:logfile.name}"
 filePattern="${sys:storm.log.dir}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/> + </definition> </changes> </component> </service> @@ -267,6 +289,20 @@ <type>tez-site</type> <insert key="tez.task.launch.cmd-opts" value="{{heap_dump_opts}}" insert-type="append" newline-before="false" newline-after="false" /> </definition> + <definition xsi:type="configure" id="hive_log4j_parameterize" summary="Parameterizing Hive Log4J Properties"> + <type>hive-log4j</type> + <set key="hive_log_maxfilesize" value="256"/> + <set key = "hive_log_maxbackupindex" value="30"/> + <regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/> + <replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB"/> + </definition> + <definition xsi:type="configure" id="hive_llap_log4j_parameterize" summary="Parameterizing Hive llap Log4J Properties"> + <type>llap-daemon-log4j</type> + <set key="hive_llap_log_maxfilesize" value="256"/> + <set key = "hive_llap_log_maxbackupindex" value="240"/> + <regex-replace key="content" find="property.llap.daemon.log.maxfilesize = ([0-9]+)MB" replace-with="property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB"/> + <regex-replace key="content" find="property.llap.daemon.log.maxbackupindex = ([0-9]+)" replace-with="property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}"/> + </definition> </changes> </component> <component name="HIVE_SERVER_INTERACTIVE"> @@ -281,6 +317,17 @@ </definition> </changes> </component> + <component name = "WEBHCAT_SERVER"> + <changes> + <definition xsi:type="configure" id="webhcat_log4j_parameterize" summary="Parameterizing Webhcat Log4J Properties"> + <type>webhcat-log4j</type> + <set key="webhcat_log_maxfilesize" value="256"/> + <set key = "webhcat_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxFileSize = {{webhcat_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxBackupIndex = {{webhcat_log_maxbackupindex}}"/> + </definition> + </changes> + </component> </service> </services> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 6c4da04..ba33a7d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -401,7 +401,35 @@ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus"> <task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/> </execute-stage> - + + <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm worker Log4J Properties"> + <task xsi:type="configure" id="storm_worker_log4j_parameterize"> + <summary>Updating the Storm worker Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm cluster Log4J Properties"> + <task xsi:type="configure" id="storm_cluster_log4j_parameterize"> + <summary>Updating the Storm cluster Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!--HIVE--> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties"> + <task xsi:type="configure" id="hive_log4j_parameterize"> + <summary>Updating the Hive Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive LLAP Log4J Properties"> + <task xsi:type="configure" id="hive_llap_log4j_parameterize"> + <summary>Updating the Hive llap Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties"> + <task xsi:type="configure" id="webhcat_log4j_parameterize"> + <summary>Updating the Webhcat Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Appending heap dump options for Hive"> <task xsi:type="configure" id="hdp_2_6_0_0_hive_append_heap_dump_options"/> </execute-stage> http://git-wip-us.apache.org/repos/asf/ambari/blob/78fefdd4/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index 7f9e986..790e50c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -767,6 +767,8 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_6_0_0_tez_append_heap_dump_options"/> <task xsi:type="configure" id="hdp_2_6_0_0_hive_append_heap_dump_options"/> + <task xsi:type="configure" id="hive_log4j_parameterize" /> + <task xsi:type="configure" id="hive_llap_log4j_parameterize" /> </pre-upgrade> <pre-downgrade /> @@ -780,6 +782,7 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_6_0_0_hive_llap_append_java_heap_dump_options"/> <task xsi:type="configure" id="hdp_2_6_0_0_hive_llap_append_heap_dump_options"/> + <task xsi:type="configure" id="webhcat_log4j_parameterize" /> </pre-upgrade> <pre-downgrade /> @@ -927,6 +930,11 @@ <service name="STORM"> <component name="NIMBUS"> + <pre-upgrade> + <task xsi:type="configure" id="storm_worker_log4j_parameterize" /> + <task xsi:type="configure" id="storm_cluster_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task" /> </upgrade>
