Repository: ambari Updated Branches: refs/heads/branch-2.5 d07c86c45 -> 38f0ee59a
AMBARI-19538. Add log rotation settings to Hive, Druid, Storm (Madhuvanthi Radhakrishnan via smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/38f0ee59 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/38f0ee59 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/38f0ee59 Branch: refs/heads/branch-2.5 Commit: 38f0ee59a5f0da2b0a52dc24aa3676f8556ab071 Parents: d07c86c Author: Sumit Mohanty <[email protected]> Authored: Fri Jan 13 15:58:45 2017 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Fri Jan 13 15:58:45 2017 -0800 ---------------------------------------------------------------------- .../0.9.2/configuration/druid-logrotate.xml | 24 ++++++- .../DRUID/0.9.2/package/scripts/params.py | 3 + .../0.12.0.2.0/configuration/hive-log4j.xml | 25 ++++++- .../0.12.0.2.0/configuration/webhcat-log4j.xml | 23 ++++++ .../HIVE/0.12.0.2.0/package/scripts/hive.py | 2 +- .../0.12.0.2.0/package/scripts/params_linux.py | 11 +++ .../HIVE/0.12.0.2.0/package/scripts/webhcat.py | 2 +- .../configuration/storm-cluster-log4j.xml | 25 ++++++- .../0.10.0/configuration/storm-worker-log4j.xml | 75 ++++++++++++++++++-- .../STORM/0.9.1/package/scripts/params_linux.py | 11 +++ .../1.0.1/configuration/storm-cluster-log4j.xml | 25 ++++++- .../1.0.1/configuration/storm-worker-log4j.xml | 75 ++++++++++++++++++-- .../HIVE/configuration/llap-daemon-log4j.xml | 25 ++++++- .../stacks/2.0.6/HIVE/test_hive_client.py | 4 +- .../stacks/2.0.6/HIVE/test_hive_metastore.py | 6 +- .../stacks/2.0.6/HIVE/test_hive_server.py | 4 +- .../stacks/2.0.6/HIVE/test_webhcat_server.py | 7 +- 17 files changed, 315 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml index f612766..b6c9142 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml @@ -21,13 +21,34 @@ --> <configuration supports_adding_forbidden="true"> <property> + <name>druid_log_maxfilesize</name> + <value>256</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Druid Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid_log_maxbackupindex</name> + <value>7</value> + <description>The number of backup files</description> + <display-name>Druid Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>content</name> <display-name>druid logrotate template</display-name> <description>Custom logrotate file</description> <value><![CDATA[ {{druid_log_dir}}/*.log { copytruncate - rotate 7 + rotate {{druid_log_maxbackupindex}} daily nocompress missingok @@ -35,6 +56,7 @@ create 660 druid users dateext dateformat -%Y-%m-%d-%s + size {{druid_log_maxfilesize}}MB } ]]></value> <value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py index c83b16e..032b019 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py @@ -86,6 +86,9 @@ log4j_props = config['configurations']['druid-log4j']['content'] druid_log_level = config['configurations']['druid-log4j']['druid_log_level'] metamx_log_level = config['configurations']['druid-log4j']['metamx_log_level'] root_log_level = config['configurations']['druid-log4j']['root_log_level'] + +druid_log_maxbackupindex = default('/configurations/druid-logrotate/druid_log_maxbackupindex', 7) +druid_log_maxfilesize = default('/configurations/druid-logrotate/druid_log_maxfilesize', 256) logrotate_props = config['configurations']['druid-logrotate']['content'] # Metadata storage http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-log4j.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-log4j.xml index cc837bb..3473b8c 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-log4j.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-log4j.xml @@ -21,6 +21,27 @@ --> <configuration supports_final="false" supports_adding_forbidden="true"> <property> + <name>hive_log_maxfilesize</name> + <value>256</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Hive Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_log_maxbackupindex</name> + <value>30</value> + <description>The number of backup files</description> + <display-name>Hive Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>content</name> <display-name>hive-log4j template</display-name> <description>Custom log4j.properties</description> @@ -69,9 +90,11 @@ log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file} log4j.appender.DRFA.DatePattern=.yyyy-MM-dd # 30-day backup -#log4j.appender.DRFA.MaxBackupIndex=30 +#log4j.appender.DRFA.MaxBackupIndex= {{hive_log_maxbackupindex}} +log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + # Pattern format: Date LogLevel LoggerName LogMessage #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n # Debugging Pattern format http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-log4j.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-log4j.xml index 96562df..4ec22770 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-log4j.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-log4j.xml @@ -21,6 +21,27 @@ --> <configuration supports_final="false" supports_adding_forbidden="true"> <property> + <name>webhcat_log_maxfilesize</name> + <value>256</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Webhcat Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>webhcat_log_maxbackupindex</name> + <value>20</value> + <description>The number of backup files</description> + <display-name>Webhcat Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>content</name> <display-name>webhcat-log4j template</display-name> <description>Custom webhcat-log4j.properties</description> @@ -54,6 +75,8 @@ log4j.threshhold = DEBUG log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender log4j.appender.standard.File = ${webhcat.log.dir}/${webhcat.log.file} +log4j.appender.standard.MaxFileSize = {{webhcat_log_maxfilesize}}MB +log4j.appender.standard.MaxBackupIndex = {{webhcat_log_maxbackupindex}} # Rollver at midnight log4j.appender.DRFA.DatePattern = .yyyy-MM-dd http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py index 58f01e6..4ac3f8e 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py @@ -426,7 +426,7 @@ def fill_conf_dir(component_conf_dir): mode=mode_identified_for_file, group=params.user_group, owner=params.hive_user, - content=params.log4j_props + content=InlineTemplate(params.log4j_props) ) elif (os.path.exists("{component_conf_dir}/{log4j_filename}.template")): File(format("{component_conf_dir}/{log4j_filename}"), http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index a515b62..c55287e 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -441,6 +441,12 @@ hcat_pid_dir = status_params.hcat_pid_dir hcat_log_dir = config['configurations']['hive-env']['hcat_log_dir'] hcat_env_sh_template = config['configurations']['hcat-env']['content'] +#Hive log4j properties +webhcat_log_maxfilesize = default("/configurations/webhcat-log4j/webhcat_log_maxfilesize", 256) +webhcat_log_maxbackupindex = default("/configurations/webhcat-log4j/webhcat_log_maxbackupindex", 20) +hive_log_maxfilesize = default("/configurations/hive-log4j/hive_log_maxfilesize", 256) +hive_log_maxbackupindex = default("/configurations/hive-log4j/hive_log_maxbackupindex", 30) + #hive-log4j.properties.template if (('hive-log4j' in config['configurations']) and ('content' in config['configurations']['hive-log4j'])): log4j_props = config['configurations']['hive-log4j']['content'] @@ -582,6 +588,11 @@ HdfsResource = functools.partial( # Hive Interactive related hive_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', []) has_hive_interactive = len(hive_interactive_hosts) > 0 + +#llap log4j properties +hive_llap_log_maxfilesize = default('/configurations/llap-daemon-log4j/hive_llap_log_maxfilesize', 256) +hive_llap_log_maxbackupindex = default('/configurations/llap-daemon-log4j/hive_llap_log_maxbackupindex', 240) + if has_hive_interactive: llap_daemon_log4j = config['configurations']['llap-daemon-log4j']['content'] llap_cli_log4j2 = config['configurations']['llap-cli-log4j2']['content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py index aa116d6..b519b4e 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py @@ -128,7 +128,7 @@ def webhcat(): mode=0644, group=params.user_group, owner=params.webhcat_user, - content=params.log4j_webhcat_props + content=InlineTemplate(params.log4j_webhcat_props) ) elif (os.path.exists("{config_dir}/{log4j_webhcat_filename}.template")): File(format("{config_dir}/{log4j_webhcat_filename}"), http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-cluster-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-cluster-log4j.xml b/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-cluster-log4j.xml index 58e1791..92d6c38 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-cluster-log4j.xml +++ b/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-cluster-log4j.xml @@ -21,6 +21,27 @@ --> <configuration supports_final="false" supports_adding_forbidden="true"> <property> + <name>storm_a1_maxfilesize</name> + <value>100</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Storm Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>storm_a1_maxbackupindex</name> + <value>9</value> + <description>The number of backup files</description> + <display-name>Storm Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>content</name> <display-name>storm-cluster-log4j template</display-name> <description>Custom cluster.xml</description> @@ -55,9 +76,9 @@ <pattern>${pattern}</pattern> </PatternLayout> <Policies> - <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB --> </Policies> - <DefaultRolloverStrategy max="9"/> + <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/> </RollingFile> <RollingFile name="ACCESS" fileName="${sys:storm.log.dir}/access.log" http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-worker-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-worker-log4j.xml b/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-worker-log4j.xml index adf57fa..45bf289 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-worker-log4j.xml +++ b/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-worker-log4j.xml @@ -21,6 +21,69 @@ --> <configuration supports_final="false" supports_adding_forbidden="true"> <property> + <name>storm_wrkr_a1_maxfilesize</name> + <value>100</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Storm Worker Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>storm_wrkr_a1_maxbackupindex</name> + <value>9</value> + <description>The number of backup files</description> + <display-name>Storm Worker Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>storm_wrkr_out_maxfilesize</name> + <value>100</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Storm Worker Standard out Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>storm_wrkr_out_maxbackupindex</name> + <value>4</value> + <description>The number of backup files</description> + <display-name>Storm Worker Standard out Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>storm_wrkr_err_maxfilesize</name> + <value>100</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Storm Worker Standard Error Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>storm_wrkr_err_maxbackupindex</name> + <value>4</value> + <description>The number of backup files</description> + <display-name>Storm Worker Standard Error Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>content</name> <display-name>storm-worker-log4j template</display-name> <description>Custom worker.xml</description> @@ -55,9 +118,9 @@ <pattern>${pattern}</pattern> </PatternLayout> <Policies> - <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB --> </Policies> - <DefaultRolloverStrategy max="9"/> + <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/> </RollingFile> <RollingFile name="STDOUT" fileName="${sys:storm.log.dir}/${sys:logfile.name}.out" @@ -66,9 +129,9 @@ <pattern>${patternNoTime}</pattern> </PatternLayout> <Policies> - <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB --> </Policies> - <DefaultRolloverStrategy max="4"/> + <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/> </RollingFile> <RollingFile name="STDERR" fileName="${sys:storm.log.dir}/${sys:logfile.name}.err" @@ -77,9 +140,9 @@ <pattern>${patternNoTime}</pattern> </PatternLayout> <Policies> - <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB --> </Policies> - <DefaultRolloverStrategy max="4"/> + <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/> </RollingFile> <Syslog name="syslog" format="RFC5424" host="localhost" port="514" protocol="UDP" appName="[${sys:storm.id}:${sys:worker.port}]" mdcId="mdc" includeMDC="true" http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py index c167d7c..dbb26f6 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py @@ -252,6 +252,17 @@ repo_config_username = config['configurations']['ranger-storm-plugin-properties' ranger_env = config['configurations']['ranger-env'] ranger_plugin_properties = config['configurations']['ranger-storm-plugin-properties'] policy_user = storm_user + +#Storm log4j properties +storm_a1_maxfilesize = default('/configurations/storm-cluster-log4j/storm_a1_maxfilesize', 100) +storm_a1_maxbackupindex = default('/configurations/storm-cluster-log4j/storm_a1_maxbackupindex', 9) +storm_wrkr_a1_maxfilesize = default('/configurations/storm-worker-log4j/storm_wrkr_a1_maxfilesize', 100) +storm_wrkr_a1_maxbackupindex = default('/configurations/storm-worker-log4j/storm_wrkr_a1_maxbackupindex', 9) +storm_wrkr_out_maxfilesize = default('/configurations/storm-worker-log4j/storm_wrkr_out_maxfilesize', 100) +storm_wrkr_out_maxbackupindex = default('/configurations/storm-worker-log4j/storm_wrkr_out_maxbackupindex', 4) +storm_wrkr_err_maxfilesize = default('/configurations/storm-worker-log4j/storm_wrkr_err_maxfilesize', 100) +storm_wrkr_err_maxbackupindex = default('/configurations/storm-worker-log4j/storm_wrkr_err_maxbackupindex', 4) + storm_cluster_log4j_content = config['configurations']['storm-cluster-log4j']['content'] storm_worker_log4j_content = config['configurations']['storm-worker-log4j']['content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-cluster-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-cluster-log4j.xml b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-cluster-log4j.xml index 1709f0d..1cb17f0 100644 --- a/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-cluster-log4j.xml +++ b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-cluster-log4j.xml @@ -21,6 +21,27 @@ --> <configuration supports_final="false" supports_adding_forbidden="true"> <property> + <name>storm_a1_maxfilesize</name> + <value>100</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Storm Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>storm_a1_maxbackupindex</name> + <value>9</value> + <description>The number of backup files</description> + <display-name>Storm Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>content</name> <display-name>storm-cluster-log4j template</display-name> <description>Custom cluster.xml</description> @@ -53,9 +74,9 @@ <pattern>${pattern}</pattern> </PatternLayout> <Policies> - <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB --> </Policies> - <DefaultRolloverStrategy max="9"/> + <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/> </RollingFile> <RollingFile name="WEB-ACCESS" immediateFlush="false" fileName="${sys:storm.log.dir}/access-web-${sys:daemon.name}.log" http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-worker-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-worker-log4j.xml b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-worker-log4j.xml index a1029a2..c3de7cd 100644 --- a/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-worker-log4j.xml +++ b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-worker-log4j.xml @@ -21,6 +21,69 @@ --> <configuration supports_final="false" supports_adding_forbidden="true"> <property> + <name>storm_wrkr_a1_maxfilesize</name> + <value>100</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Storm Worker Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>storm_wrkr_a1_maxbackupindex</name> + <value>9</value> + <description>The number of backup files</description> + <display-name>Storm Worker Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>storm_wrkr_out_maxfilesize</name> + <value>100</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Storm Worker Standard out Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>storm_wrkr_out_maxbackupindex</name> + <value>4</value> + <description>The number of backup files</description> + <display-name>Storm Worker Standard out Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>storm_wrkr_err_maxfilesize</name> + <value>100</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Storm Worker Standard Error Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>storm_wrkr_err_maxbackupindex</name> + <value>4</value> + <description>The number of backup files</description> + <display-name>Storm Worker Standard Error Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>content</name> <display-name>storm-worker-log4j template</display-name> <description>Custom worker.xml</description> @@ -56,9 +119,9 @@ <pattern>${pattern}</pattern> </PatternLayout> <Policies> - <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB --> </Policies> - <DefaultRolloverStrategy max="9"/> + <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/> </RollingFile> <RollingFile name="STDOUT" fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out" @@ -67,9 +130,9 @@ <pattern>${patternNoTime}</pattern> </PatternLayout> <Policies> - <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB --> </Policies> - <DefaultRolloverStrategy max="4"/> + <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/> </RollingFile> <RollingFile name="STDERR" fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err" @@ -78,9 +141,9 @@ <pattern>${patternNoTime}</pattern> </PatternLayout> <Policies> - <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB --> </Policies> - <DefaultRolloverStrategy max="4"/> + <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/> </RollingFile> <RollingFile name="METRICS" fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.metrics" http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml index 6953530..4790f0f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml @@ -21,6 +21,27 @@ --> <configuration supports_final="false" supports_adding_forbidden="true"> <property> + <name>hive_llap_log_maxfilesize</name> + <value>256</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Hive LLAP Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_llap_log_maxbackupindex</name> + <value>240</value> + <description>The number of backup files</description> + <display-name>Hive LLAP Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>content</name> <display-name>llap-deamon-log4j template</display-name> <description>Custom llap-daemon-log4j2.properties</description> @@ -58,8 +79,8 @@ property.llap.daemon.root.logger = console property.llap.daemon.log.dir = . property.llap.daemon.log.file = llapdaemon.log property.llap.daemon.historylog.file = llapdaemon_history.log -property.llap.daemon.log.maxfilesize = 256MB -property.llap.daemon.log.maxbackupindex = 240 +property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB +property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}} # list of all appenders appenders = console, RFA, HISTORYAPPENDER, query-routing http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py index 2653254..d102b9d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py @@ -67,7 +67,7 @@ class TestHiveClient(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/usr/hdp/current/hive-client/conf/hive-log4j.properties', - content = 'log4jproperties\nline2', + content = InlineTemplate('log4jproperties\nline2'), owner = 'hive', group = 'hadoop', mode = 0644, @@ -147,7 +147,7 @@ class TestHiveClient(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/usr/hdp/current/hive-client/conf/hive-log4j.properties', - content = 'log4jproperties\nline2', + content = InlineTemplate('log4jproperties\nline2'), owner = 'hive', group = 'hadoop', mode = 0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py index bf9c208..821248e 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py @@ -184,7 +184,7 @@ class TestHiveMetastore(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-log4j.properties', - content = 'log4jproperties\nline2', + content = InlineTemplate('log4jproperties\nline2'), owner = 'hive', group = 'hadoop', mode = 0644, @@ -302,7 +302,7 @@ class TestHiveMetastore(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-log4j.properties', - content = 'log4jproperties\nline2', + content = InlineTemplate('log4jproperties\nline2'), owner = 'hive', group = 'hadoop', mode = 0644, @@ -451,7 +451,7 @@ class TestHiveMetastore(RMFTestCase): owner = 'hive') self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-log4j.properties', - content = 'log4jproperties\nline2', + content = InlineTemplate('log4jproperties\nline2'), mode = 420, group = 'hadoop', owner = 'hive') http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py index 6e97d15..ec3e8f0 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py @@ -447,7 +447,7 @@ class TestHiveServer(RMFTestCase): mode=0644, ) self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-log4j.properties', - content='log4jproperties\nline2', + content=InlineTemplate('log4jproperties\nline2'), owner='hive', group='hadoop', mode=0644, @@ -653,7 +653,7 @@ class TestHiveServer(RMFTestCase): mode=0644, ) self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-log4j.properties', - content='log4jproperties\nline2', + content=InlineTemplate('log4jproperties\nline2'), owner='hive', group='hadoop', mode=0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py index 2b26dd8..f93ea09 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py @@ -22,6 +22,7 @@ from mock.mock import MagicMock, patch from stacks.utils.RMFTestCase import * from resource_management.core.exceptions import Fail + @patch("os.path.isfile", new = MagicMock(return_value=True)) @patch("glob.glob", new = MagicMock(return_value=["one", "two"])) class TestWebHCatServer(RMFTestCase): @@ -227,7 +228,7 @@ class TestWebHCatServer(RMFTestCase): create_parents = True ) self.assertResourceCalled('File', '/etc/hive-webhcat/conf/webhcat-log4j.properties', - content = 'log4jproperties\nline2', + content = InlineTemplate('log4jproperties\nline2'), owner = 'hcat', group = 'hadoop', mode = 0644, @@ -273,7 +274,7 @@ class TestWebHCatServer(RMFTestCase): create_parents = True ) self.assertResourceCalled('File', '/etc/hive-webhcat/conf/webhcat-log4j.properties', - content = 'log4jproperties\nline2', + content = InlineTemplate('log4jproperties\nline2'), owner = 'hcat', group = 'hadoop', mode = 0644, @@ -531,7 +532,7 @@ class TestWebHCatServer(RMFTestCase): create_parents = True) self.assertResourceCalled('File', '/usr/hdp/current/hive-webhcat/etc/webhcat/webhcat-log4j.properties', - content = 'log4jproperties\nline2', + content = InlineTemplate('log4jproperties\nline2'), owner = 'hcat', group = 'hadoop', mode = 0644)
