AMBARI-19541. Add log rotation settings - handle HDP upgrade scenario (Madhuvanthi Radhakrishnan via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4dac2783 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4dac2783 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4dac2783 Branch: refs/heads/trunk Commit: 4dac27831c2498e12fff05d33a95444e3abffdff Parents: b2ba7dd Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Tue Jan 17 14:15:57 2017 -0800 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Tue Jan 17 14:15:57 2017 -0800 ---------------------------------------------------------------------- .../upgrade/ConfigUpgradeChangeDefinition.java | 110 ++++++++++- .../state/stack/upgrade/ConfigureTask.java | 9 +- .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 148 +++++++++++++++ .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 90 +++++++++ .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 88 ++++++++- .../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 91 ++++++++- .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 52 ++++- .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 43 ++++- .../stacks/HDP/2.4/upgrades/config-upgrade.xml | 174 ++++++++++++++++- .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 97 +++++++++- .../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 87 ++++++++- .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 40 +++- .../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 40 +++- .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 190 ++++++++++++++++++- .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 95 ++++++++++ .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 41 ++++ .../src/main/resources/upgrade-config.xsd | 14 +- 17 files changed, 1394 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4dac2783/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java index 5428ea7..31df790 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java @@ -20,6 +20,9 @@ package org.apache.ambari.server.state.stack.upgrade; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -28,6 +31,8 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlType; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -130,6 +135,9 @@ public class ConfigUpgradeChangeDefinition { @XmlElement(name="replace") private List<Replace> replacements; + @XmlElement(name="regex-replace") + private List<RegexReplace> regexReplacements; + /** * @return the config type */ @@ -198,6 +206,50 @@ public class ConfigUpgradeChangeDefinition { } /** + * @return the replacement tokens, never {@code null} + */ + public List<Replace> getRegexReplacements(Cluster cluster) { + + if (null == regexReplacements) { + + return Collections.emptyList(); + } + + List<Replace> list = new ArrayList<>(); + for (RegexReplace regexReplaceObj : regexReplacements) { + if (null == regexReplaceObj.key || null == regexReplaceObj.find || null == regexReplaceObj.replaceWith) { + LOG.warn(String.format("Replacement %s is invalid", regexReplaceObj)); + continue; + } + + try{ + Config config = cluster.getDesiredConfigByType(configType); + + Map<String, String> properties = config.getProperties(); + String content = properties.get(regexReplaceObj.key); + + Pattern REGEX = Pattern.compile(regexReplaceObj.find, Pattern.MULTILINE); + + Matcher patternMatchObj = REGEX.matcher(content); + if (patternMatchObj.find() && patternMatchObj.groupCount()==1) { + regexReplaceObj.find = patternMatchObj.group(); + Replace rep = regexReplaceObj.copyToReplaceObject(); + list.add(rep); + } + + }catch(Exception e){ + String message = ""; + message = "ConfigUpgradeChangeDefinition: getRegexReplacements : Error while fetching config properties "; + LOG.error(message, e); + + } + + } + return list; + } + + + /** * Used for configuration updates that should mask their values from being * printed in plain text. */ @@ -379,4 +431,60 @@ public class ConfigUpgradeChangeDefinition { '}'; } } -} \ No newline at end of file + + /** + * Used to replace strings in a key with other strings. More complex + * scenarios are possible with regex. + */ + @XmlAccessorType(XmlAccessType.FIELD) + @XmlType(name = "regex-replace") + public static class RegexReplace extends Masked{ + /** + * The key name + */ + @XmlAttribute(name="key") + public String key; + + /** + * The string to find + */ + @XmlAttribute(name="find") + public String find; + + /** + * The string to replace + */ + @XmlAttribute(name="replace-with") + public String replaceWith; + + @Override + public String toString() { + return "RegexReplace{" + + "key='" + key + '\'' + + ", find='" + find + '\'' + + ", replaceWith='" + replaceWith + '\'' + + ", ifKey='" + ifKey + '\'' + + ", ifType='" + ifType + '\'' + + ", ifValue='" + ifValue + '\'' + + ", ifKeyState='" + ifKeyState + '\'' + + '}'; + } + + /*** + * Copies a RegexReplace type object to Replace object. + * @return Replace object + */ + public Replace copyToReplaceObject(){ + Replace rep = new Replace(); + rep.find = this.find; + rep.key = this.key; + rep.replaceWith = this.replaceWith; + rep.ifKey = this.ifKey; + rep.ifType = this.ifType; + rep.ifValue = this.ifValue; + rep.ifKeyState = this.ifKeyState; + + return rep; + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/4dac2783/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java index d7bb338..f256eb0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java @@ -211,7 +211,12 @@ public class ConfigureTask extends ServerSideActionTask { } // replacements - List<Replace> replacements = definition.getReplacements(); + + List<Replace> replacements = new ArrayList<Replace>(); + replacements.addAll(definition.getReplacements()); + //Fetch the replacements that used regex to find a string + replacements.addAll(definition.getRegexReplacements(cluster)); + if( null != replacements && !replacements.isEmpty() ){ List<Replace> allowedReplacements = getValidReplacements(cluster, definition.getConfigType(), replacements); configParameters.put(ConfigureTask.PARAMETER_REPLACEMENTS, m_gson.toJson(allowedReplacements)); @@ -319,4 +324,4 @@ public class ConfigureTask extends ServerSideActionTask { return config.getProperties().get(propertyKey); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ambari/blob/4dac2783/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index fd821aa..c9adb8c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -64,6 +64,19 @@ <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> </definition> + <!-- HBase Rolling properties for log4j need to be parameterized. --> + <definition xsi:type="configure" id="hbase_log4j_parameterize" summary="Parameterizing HBase Log4J Properties"> + <type>hbase-log4j</type> + <set key="hbase_log_maxfilesize" value="256"/> + <set key="hbase_log_maxbackupindex" value="20"/> + <set key="hbase_security_log_maxfilesize" value="256"/> + <set key="hbase_security_log_maxbackupindex" value="20"/> + <regex-replace key="content" find="hbase.log.maxfilesize=([0-9]+)MB" replace-with="hbase.log.maxfilesize={{hbase_log_maxfilesize}}MB"/> + <regex-replace key="content" find="hbase.log.maxbackupindex=([0-9]+)" replace-with="hbase.log.maxbackupindex={{hbase_log_maxbackupindex}}"/> + <regex-replace key="content" find="hbase.security.log.maxfilesize=([0-9]+)MB" replace-with="hbase.security.log.maxfilesize={{hbase_security_log_maxfilesize}}MB"/> + <regex-replace key="content" find="hbase.security.log.maxbackupindex=([0-9]+)" replace-with="hbase.security.log.maxbackupindex={{hbase_security_log_maxbackupindex}}"/> + </definition> + </changes> </component> </service> @@ -232,9 +245,40 @@ <type>ranger-env</type> <transfer operation="delete" delete-key="bind_anonymous" /> </definition> + <definition xsi:type="configure" id="admin_log4j_parameterize" summary="Parameterizing Ranger Log4J Properties"> + <type>admin-log4j</type> + <set key="ranger_xa_log_maxfilesize" value="256"/> + <set key="ranger_xa_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.xa_log_appender.MaxFileSize={{ranger_xa_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.xa_log_appender.MaxBackupIndex={{ranger_xa_log_maxbackupindex}}"/> + </definition> </changes> </component> + + <component name="RANGER_USERSYNC"> + <changes> + <definition xsi:type="configure" id="usersync_log4j_parameterize" summary="Parameterizing Ranger Usersync Log4J Properties"> + <type>usersync-log4j</type> + <set key="ranger_usersync_log_maxfilesize" value="256"/> + <set key="ranger_usersync_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxFileSize = {{ranger_usersync_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxBackupIndex = {{ranger_usersync_log_maxbackupindex}}"/> + </definition> + </changes> + </component> + + <component name="RANGER_TAGSYNC"> + <changes> + <definition xsi:type="configure" id="tagsync_log4j_parameterize" summary="Parameterizing Ranger Tagsync Log4J Properties"> + <type>tagsync-log4j</type> + <set key="ranger_tagsync_log_maxfilesize" value="256"/> + <set key="ranger_tagsync_log_number_of_backup_files" value="20"/> + <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxFileSize = {{ranger_tagsync_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxBackupIndex = {{ranger_tagsync_log_number_of_backup_files}}"/> + </definition> + </changes> + </component> </service> <service name="RANGER_KMS"> @@ -250,6 +294,17 @@ <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> </definition> + <definition xsi:type="configure" id="kms_log4j_parameterize" summary="Parameterizing Ranger KMS Log4J Properties"> + <type>kms-log4j</type> + <set key="ranger_kms_log_maxfilesize" value="256"/> + <set key="ranger_kms_log_maxbackupindex" value="20"/> + <set key="ranger_kms_audit_log_maxfilesize" value="256"/> + <set key="ranger_kms_audit_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms.MaxFileSize = {{ranger_kms_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms.MaxBackupIndex = {{ranger_kms_log_maxbackupindex}}"/> + <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms-audit.MaxFileSize = {{ranger_kms_audit_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms-audit.MaxBackupIndex = {{ranger_kms_audit_log_maxbackupindex}}"/> + </definition> </changes> </component> </service> @@ -274,6 +329,19 @@ <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> </definition> + + <!-- HDFS Rolling properties for log4j need to be parameterized. --> + <definition xsi:type="configure" id="hdfs_log4j_parameterize" summary="Parameterizing Hdfs Log4J Properties"> + <type>hdfs-log4j</type> + <set key="hadoop_log_max_backup_size" value="256"/> + <set key="hadoop_log_number_of_backup_files" value="10"/> + <set key="hadoop_security_log_max_backup_size" value="256"/> + <set key="hadoop_security_log_number_of_backup_files" value="20"/> + <regex-replace key="content" find="log4j.appender.RFA.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RFA.MaxFileSize={{hadoop_log_max_backup_size}}MB"/> + <regex-replace key="content" find="log4j.appender.RFA.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RFA.MaxBackupIndex={{hadoop_log_number_of_backup_files}}"/> + <regex-replace key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB" replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/> + <regex-replace key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)" replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/> + </definition> </changes> </component> </service> @@ -337,6 +405,12 @@ <type>oozie-site</type> <replace key="oozie.services" find="org.apache.oozie.service.CoordinatorStoreService," replace-with="" /> </definition> + <!-- Oozie Rolling properties for log4j need to be parameterized. --> + <definition xsi:type="configure" id="oozie_log4j_parameterize" summary="Parameterizing Oozie Log4J Properties"> + <type>oozie-log4j</type> + <set key="oozie_log_maxhistory" value="720"/> + <regex-replace key="content" find="^log4j.appender.oozie.RollingPolicy.MaxHistory=([0-9]+)" replace-with="log4j.appender.oozie.RollingPolicy.MaxHistory={{oozie_log_maxhistory}}"/> + </definition> </changes> </component> </service> @@ -359,6 +433,17 @@ <set key="inter.broker.protocol.version" value="0.9.0.0" /> <set key="log.message.format.version" value="0.9.0.0" /> </definition> + <definition xsi:type="configure" id="kafka_log4j_parameterize" summary="Parameterizing Kafka Log4J Properties"> + <type>kafka-log4j</type> + <set key="kafka_log_maxfilesize" value="256"/> + <set key="kafka_log_maxbackupindex" value="20"/> + <set key="controller_log_maxfilesize" value="256"/> + <set key="controller_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kafkaAppender.MaxFileSize = {{kafka_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kafkaAppender.MaxBackupIndex = {{kafka_log_maxbackupindex}}"/> + <replace key="content" find="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.controllerAppender.MaxFileSize = {{controller_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.controllerAppender.MaxBackupIndex = {{controller_log_maxbackupindex}}"/> + </definition> </changes> </component> </service> @@ -376,6 +461,14 @@ <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> </definition> + <!-- Yarn Rolling properties for log4j need to be parameterized. --> + <definition xsi:type="configure" id="yarn_log4j_parameterize" summary="Parameterizing Yarn Log4J Properties"> + <type>yarn-log4j</type> + <set key="yarn_rm_summary_log_max_backup_size" value="256"/> + <set key="yarn_rm_summary_log_number_of_backup_files" value="20"/> + <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/> + <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/> + </definition> </changes> </component> </service> @@ -471,6 +564,20 @@ <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> </definition> + <definition xsi:type="configure" id="knox_gateway_log4j_parameterize" summary="Parameterizing Knox Gateway Log4J Properties"> + <type>gateway-log4j</type> + <set key="knox_gateway_log_maxfilesize" value="256"/> + <set key="knox_gateway_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxFileSize = {{knox_gateway_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxBackupIndex = {{knox_gateway_log_maxbackupindex}}"/> + </definition> + <definition xsi:type="configure" id="knox_ldap_log4j_parameterize" summary="Parameterizing Knox Ldap Log4J Properties"> + <type>ldap-log4j</type> + <set key="knox_ldap_log_maxfilesize" value="256"/> + <set key="knox_ldap_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxFileSize = {{knox_ldap_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxBackupIndex = {{knox_ldap_log_maxbackupindex}}"/> + </definition> </changes> </component> </service> @@ -482,8 +589,49 @@ <type>falcon-startup.properties</type> <set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.extensions.ExtensionService, org.apache.falcon.service.LifecyclePolicyMap, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService, org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}"/> </definition> + <definition xsi:type="configure" id="falcon_log4j_parameterize" summary="Parameterizing Falcon Log4J Properties"> + <type>falcon-log4j</type> + <set key="falcon_log_maxfilesize" value="256"/> + <set key="falcon_log_maxbackupindex" value="20"/> + <set key="falcon_security_log_maxfilesize" value="256"/> + <set key="falcon_security_log_maxbackupindex" value="20"/> + <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxFileSize" value="{{falcon_log_maxfilesize}}MB" />"/> + <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxBackupIndex" value="{{falcon_log_maxbackupindex}}" />"/> + <replace key="content" find="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxFileSize" value="{{falcon_security_log_maxfilesize}}MB"/>"/> + <replace key="content" find="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxBackupIndex" value="{{falcon_security_log_maxbackupindex}}"/>"/> + </definition> + </changes> + </component> + </service> + + <service name="ATLAS"> + <component name="ATLAS_SERVER"> + <changes> + <definition xsi:type="configure" id="atlas_log4j_parameterize" summary="Parameterizing Atlas Log4J Properties"> + <type>atlas-log4j</type> + <set key="atlas_log_max_backup_size" value="256"/> + <set key="atlas_log_number_of_backup_files" value="20"/> + <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">\n<param name="MaxFileSize" value="{{atlas_log_max_backup_size}}MB" />"/> + <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">\n<param name="MaxFileSize" value="{{atlas_log_number_of_backup_files}}" />"/> + </definition> + </changes> + </component> + </service> + + <service name="ZOOKEEPER"> + <component name="ZOOKEEPER_SERVER"> + <changes> + <!-- Zookeeper Rolling properties for log4j need to be parameterized. --> + <definition xsi:type="configure" id="zookeeper_log4j_parameterize" summary="Parameterizing ZooKeeper Log4J Properties"> + <type>zookeeper-log4j</type> + <set key="zookeeper_log_max_backup_size" value="10"/> + <set key="zookeeper_log_number_of_backup_files" value="10"/> + <regex-replace key="content" find="^log4j.appender.ROLLINGFILE.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.ROLLINGFILE.MaxFileSize={{zookeeper_log_max_backup_size}}MB"/> + <regex-replace key="content" find="^#log4j.appender.ROLLINGFILE.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.ROLLINGFILE.MaxBackupIndex={{zookeeper_log_number_of_backup_files}}"/> + </definition> </changes> </component> </service> + </services> </upgrade-config-changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/4dac2783/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml index 8dff078..d674af7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml @@ -258,6 +258,12 @@ <task xsi:type="configure" id="hdp_2_4_0_0_namenode_ha_adjustments"/> </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Parameterizing Hdfs Log4J Properties"> + <task xsi:type="configure" id="hdfs_log4j_parameterize"> + <summary>Updating the Hdfs Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- YARN --> <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig"> @@ -265,6 +271,13 @@ </task> </execute-stage> + <!--Yarn--> + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Parameterizing Yarn Log4J Properties Resource Manager"> + <task xsi:type="configure" id="yarn_log4j_parameterize"> + <summary>Updating the Yarn Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"> <summary>Verifying LZO codec path for mapreduce</summary> @@ -277,6 +290,12 @@ <task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/> </execute-stage> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Parameterizing HBase Log4J Properties"> + <task xsi:type="configure" id="hbase_log4j_parameterize"> + <summary>Updating the Hbase Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"> <summary>Update HBase Env Configuration</summary> @@ -310,6 +329,12 @@ </task> </execute-stage> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Parameterizing Oozie Log4J Properties"> + <task xsi:type="configure" id="oozie_log4j_parameterize"> + <summary>Updating the Oozie Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server"> <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" /> </execute-stage> @@ -342,6 +367,71 @@ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus"> <task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/> </execute-stage> + + <!--ZOOKEEPER--> + <execute-stage service="ZOOKEEPER" component="ZOOKEEPER_SERVER" title="Parameterizing Zookeeper Log4J Properties"> + <task xsi:type="configure" id="zookeeper_log4j_parameterize"> + <summary>Updating the Zookeeper Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!--ATLAS--> + <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties"> + <task xsi:type="configure" id="atlas_log4j_parameterize"> + <summary>Updating the Atlas Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!--KAFKA--> + <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Parameterizing Kafka Log4J Properties"> + <task xsi:type="configure" id="kafka_log4j_parameterize"> + <summary>Updating the Kafka Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!--RANGER--> + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Parameterizing Ranger Admin Log4J Properties"> + <task xsi:type="configure" id="admin_log4j_parameterize"> + <summary>Updating the Ranger admin Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="RANGER" component="RANGER_USERSYNC" title="Parameterizing Ranger Usersync Log4J Properties"> + <task xsi:type="configure" id="usersync_log4j_parameterize"> + <summary>Updating the Ranger usersync Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="RANGER" component="RANGER_TAGSYNC" title="Parameterizing Ranger Tagsync Log4J Properties"> + <task xsi:type="configure" id="tagsync_log4j_parameterize"> + <summary>Updating the Ranger tagsync Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!--RANGER-KMS--> + <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Parameterizing Ranger Kms Log4J Properties"> + <task xsi:type="configure" id="kms_log4j_parameterize"> + <summary>Updating the KMS Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!--KNOX--> + <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Parameterizing Knox Gateway Log4J Properties"> + <task xsi:type="configure" id="knox_gateway_log4j_parameterize"> + <summary>Updating the Knox Gateway Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Parameterizing Knox Ldap Log4J Properties"> + <task xsi:type="configure" id="knox_ldap_log4j_parameterize"> + <summary>Updating the Knox Ldap Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!--FALCON--> + <execute-stage service="FALCON" component="FALCON_SERVER" title="Parameterizing Falcon Log4J Properties"> + <task xsi:type="configure" id="falcon_log4j_parameterize"> + <summary>Updating the Falcon Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + </group> <!-- http://git-wip-us.apache.org/repos/asf/ambari/blob/4dac2783/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml index 11cd31a..7a05c99 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml @@ -279,6 +279,12 @@ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db"/> </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Parameterizing Hdfs Log4J Properties"> + <task xsi:type="configure" id="hdfs_log4j_parameterize"> + <summary>Updating the Hdfs Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- YARN --> <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig"> @@ -290,6 +296,12 @@ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/> </execute-stage> + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Parameterizing Yarn Log4J Properties Resource Manager"> + <task xsi:type="configure" id="yarn_log4j_parameterize"> + <summary>Updating the Yarn Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle"> <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/> </execute-stage> @@ -310,6 +322,12 @@ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db"/> </execute-stage> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Parameterizing HBase Log4J Properties"> + <task xsi:type="configure" id="hbase_log4j_parameterize"> + <summary>Updating the Hbase Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"> <summary>Update HBase Env Configuration</summary> @@ -374,15 +392,34 @@ <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" /> </execute-stage> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Parameterizing Oozie Log4J Properties"> + <task xsi:type="configure" id="oozie_log4j_parameterize"> + <summary>Updating the Oozie Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!--FALCON--> <execute-stage service="FALCON" component="FALCON_SERVER" title="Apply config changes for Falcon"> <task xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"/> </execute-stage> + <execute-stage service="FALCON" component="FALCON_SERVER" title="Parameterizing Falcon Log4J Properties"> + <task xsi:type="configure" id="falcon_log4j_parameterize"> + <summary>Updating the Falcon Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!--KAFKA--> <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Apply config changes for Kafka Broker"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db"/> </execute-stage> + <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Parameterizing Kafka Log4J Properties"> + <task xsi:type="configure" id="kafka_log4j_parameterize"> + <summary>Updating the Kafka Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- SPARK --> <execute-stage service="SPARK" component="SPARK_JOBHISTORYSERVER" title="Apply config changes for Spark JobHistoryServer"> <task xsi:type="configure" id="hdp_2_4_0_0_spark_jobhistoryserver"/> @@ -424,6 +461,12 @@ <task xsi:type="configure" id="hdp_2_5_0_0_set_external_solrCloud_flag"/> </execute-stage> + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Parameterizing Ranger Admin Log4J Properties"> + <task xsi:type="configure" id="admin_log4j_parameterize"> + <summary>Updating the Ranger admin Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Calculating Ranger Properties"> <condition xsi:type="security" type="kerberos"/> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.RangerKerberosConfigCalculation"> @@ -437,6 +480,17 @@ </task> </execute-stage> + <execute-stage service="RANGER" component="RANGER_USERSYNC" title="Parameterizing Ranger Usersync Log4J Properties"> + <task xsi:type="configure" id="usersync_log4j_parameterize"> + <summary>Updating the Ranger usersync Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="RANGER" component="RANGER_TAGSYNC" title="Parameterizing Ranger Tagsync Log4J Properties"> + <task xsi:type="configure" id="tagsync_log4j_parameterize"> + <summary>Updating the Ranger tagsync Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- RANGER KMS --> <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config changes for Ranger KMS Server"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db"/> @@ -449,11 +503,28 @@ </task> </execute-stage> + <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Parameterizing Ranger Kms Log4J Properties"> + <task xsi:type="configure" id="kms_log4j_parameterize"> + <summary>Updating the KMS Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- KNOX --> <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for Knox Gateway"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_knox_audit_db"/> </execute-stage> + <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Parameterizing Knox Gateway Log4J Properties"> + <task xsi:type="configure" id="knox_gateway_log4j_parameterize"> + <summary>Updating the Knox Gateway Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Parameterizing Knox Ldap Log4J Properties"> + <task xsi:type="configure" id="knox_ldap_log4j_parameterize"> + <summary>Updating the Knox Ldap Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- STORM --> <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Storm Nimbus"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_storm_audit_db"/> @@ -480,7 +551,22 @@ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus"> <task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/> </execute-stage> - </group> + + <!--ATLAS--> + <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties"> + <task xsi:type="configure" id="atlas_log4j_parameterize"> + <summary>Updating the Atlas Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!--ZOOKEEPER--> + <execute-stage service="ZOOKEEPER" component="ZOOKEEPER_SERVER" title="Parameterizing Zookeeper Log4J Properties"> + <task xsi:type="configure" id="zookeeper_log4j_parameterize"> + <summary>Updating the Zookeeper Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + </group> <!-- After processing this group, the user-specified Kerberos descriptor will be updated to work with http://git-wip-us.apache.org/repos/asf/ambari/blob/4dac2783/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml index 549e5a4..2c2049b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml @@ -280,6 +280,12 @@ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db"/> </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Parameterizing Hdfs Log4J Properties"> + <task xsi:type="configure" id="hdfs_log4j_parameterize"> + <summary>Updating the Hdfs Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- YARN --> <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig"> @@ -295,6 +301,12 @@ <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/> </execute-stage> + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Parameterizing Yarn Log4J Properties Resource Manager"> + <task xsi:type="configure" id="yarn_log4j_parameterize"> + <summary>Updating the Yarn Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"> <summary>Verifying LZO codec path for mapreduce</summary> @@ -311,6 +323,13 @@ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db"/> </execute-stage> + <!--HBASE--> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Parameterizing HBase Log4J Properties"> + <task xsi:type="configure" id="hbase_log4j_parameterize"> + <summary>Updating the Hbase Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"> <summary>Update HBase Env Configuration</summary> @@ -375,15 +394,34 @@ <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" /> </execute-stage> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Parameterizing Oozie Log4J Properties"> + <task xsi:type="configure" id="oozie_log4j_parameterize"> + <summary>Updating the Oozie Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!--FALCON--> <execute-stage service="FALCON" component="FALCON_SERVER" title="Apply config changes for Falcon"> <task xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"/> </execute-stage> + <execute-stage service="FALCON" component="FALCON_SERVER" title="Parameterizing Falcon Log4J Properties"> + <task xsi:type="configure" id="falcon_log4j_parameterize"> + <summary>Updating the Falcon Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!--KAFKA--> <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Apply config changes for Kafka Broker"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db"/> </execute-stage> + <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Parameterizing Kafka Log4J Properties"> + <task xsi:type="configure" id="kafka_log4j_parameterize"> + <summary>Updating the Kafka Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- SPARK --> <execute-stage service="SPARK" component="SPARK_JOBHISTORYSERVER" title="Apply config changes for Spark JobHistoryServer"> <task xsi:type="configure" id="hdp_2_4_0_0_spark_jobhistoryserver"/> @@ -425,6 +463,12 @@ <task xsi:type="configure" id="hdp_2_5_0_0_set_external_solrCloud_flag"/> </execute-stage> + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Parameterizing Ranger Admin Log4J Properties"> + <task xsi:type="configure" id="admin_log4j_parameterize"> + <summary>Updating the Ranger admin Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Calculating Ranger Properties"> <condition xsi:type="security" type="kerberos"/> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.RangerKerberosConfigCalculation"> @@ -442,11 +486,29 @@ <task xsi:type="configure" id="hdp_2_6_0_0_remove_bind_anonymous"/> </execute-stage> + <execute-stage service="RANGER" component="RANGER_USERSYNC" title="Parameterizing Ranger Usersync Log4J Properties"> + <task xsi:type="configure" id="usersync_log4j_parameterize"> + <summary>Updating the Ranger usersync Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <execute-stage service="RANGER" component="RANGER_TAGSYNC" title="Parameterizing Ranger Tagsync Log4J Properties"> + <task xsi:type="configure" id="tagsync_log4j_parameterize"> + <summary>Updating the Ranger tagsync Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- RANGER KMS --> <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config changes for Ranger KMS Server"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db"/> </execute-stage> + <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config changes for Ranger kms"> + <task xsi:type="configure" id="kms_log4j_parameterize"> + <summary>Updating the KMS Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Calculating Ranger Properties"> <condition xsi:type="security" type="kerberos"/> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.RangerKmsProxyConfig"> @@ -459,6 +521,18 @@ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_knox_audit_db"/> </execute-stage> + <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Parameterizing Knox Gateway Log4J Properties"> + <task xsi:type="configure" id="knox_gateway_log4j_parameterize"> + <summary>Updating the Knox Gateway Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Parameterizing Knox Ldap Log4J Properties"> + <task xsi:type="configure" id="knox_ldap_log4j_parameterize"> + <summary>Updating the Knox Ldap Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + <!-- STORM --> <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Storm Nimbus"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_storm_audit_db"/> @@ -485,7 +559,22 @@ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus"> <task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/> </execute-stage> - </group> + + <!--ATLAS--> + <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties"> + <task xsi:type="configure" id="atlas_log4j_parameterize"> + <summary>Updating the Atlas Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + <!--ZOOKEEPER--> + <execute-stage service="ZOOKEEPER" component="ZOOKEEPER_SERVER" title="Parameterizing Zookeeper Log4J Properties"> + <task xsi:type="configure" id="zookeeper_log4j_parameterize"> + <summary>Updating the Zookeeper Log4J properties to include parameterizations</summary> + </task> + </execute-stage> + + </group> <!-- After processing this group, the user-specified Kerberos descriptor will be updated to work with http://git-wip-us.apache.org/repos/asf/ambari/blob/4dac2783/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml index 0a7bcea..a37d171 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml @@ -470,6 +470,10 @@ <processing> <service name="ZOOKEEPER"> <component name="ZOOKEEPER_SERVER"> + <pre-upgrade> + <task xsi:type="configure" id="zookeeper_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade /> <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -509,6 +513,9 @@ <script>scripts/ranger_admin.py</script> <function>setup_ranger_java_patches</function> </task> + + <task xsi:type="configure" id="admin_log4j_parameterize" /> + </pre-upgrade> <pre-downgrade copy-upgrade="true" /> @@ -520,6 +527,20 @@ </component> <component name="RANGER_USERSYNC"> + <pre-upgrade> + <task xsi:type="configure" id="usersync_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade/> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + + <component name="RANGER_TAGSYNC"> + <pre-upgrade> + <task xsi:type="configure" id="tagsync_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade /> <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -529,6 +550,7 @@ <service name="RANGER_KMS"> <component name="RANGER_KMS_SERVER"> <pre-upgrade> + <task xsi:type="configure" id="kms_log4j_parameterize" /> <task xsi:type="execute" hosts="any" sequential="true"> <summary>Upgrading Ranger KMS database schema</summary> <script>scripts/kms_server.py</script> @@ -555,6 +577,7 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_4_0_0_namenode_ha_adjustments"/> + <task xsi:type="configure" id="hdfs_log4j_parameterize" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> @@ -630,6 +653,7 @@ <component name="RESOURCEMANAGER"> <pre-upgrade> + <task xsi:type="configure" id="yarn_log4j_parameterize" /> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig"> <summary>Calculating Yarn Properties for Spark Shuffle</summary> </task> @@ -659,6 +683,7 @@ <pre-upgrade> <!-- These HBASE configs changed in HDP 2.3.4.0, but Ambari can't distinguish HDP 2.3.2.0 vs HDP 2.3.4.0, so easier to always do them. --> <task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/> + <task xsi:type="configure" id="hbase_log4j_parameterize" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> @@ -814,7 +839,7 @@ <component name="OOZIE_SERVER"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" /> - + <task xsi:type="configure" id="oozie_log4j_parameterize" /> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"> <summary>Adjusting Oozie properties</summary> </task> @@ -863,6 +888,10 @@ <service name="FALCON"> <component name="FALCON_SERVER"> + <pre-upgrade> + <task xsi:type="configure" id="falcon_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -876,6 +905,10 @@ <service name="KAFKA"> <component name="KAFKA_BROKER"> + <pre-upgrade> + <task xsi:type="configure" id="kafka_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -884,6 +917,10 @@ <service name="KNOX"> <component name="KNOX_GATEWAY"> + <pre-upgrade> + <task xsi:type="configure" id="knox_gateway_log4j_parameterize" /> + <task xsi:type="configure" id="knox_ldap_log4j_parameterize" /> + </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> <upgrade> @@ -963,5 +1000,18 @@ </upgrade> </component> </service> + + <service name="ATLAS"> + <component name="ATLAS_SERVER"> + <pre-upgrade> + <task xsi:type="configure" id="atlas_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade /> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + </service> + </processing> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/4dac2783/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml index 7827533..47f58bd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml @@ -558,6 +558,10 @@ <processing> <service name="ZOOKEEPER"> <component name="ZOOKEEPER_SERVER"> + <pre-upgrade> + <task xsi:type="configure" id="zookeeper_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -573,6 +577,7 @@ <service name="RANGER"> <component name="RANGER_ADMIN"> <pre-upgrade> + <task xsi:type="configure" id="admin_log4j_parameterize" /> <task xsi:type="execute" hosts="all"> <summary>Stop Ranger Admin</summary> <script>scripts/ranger_admin.py</script> @@ -621,6 +626,20 @@ </component> <component name="RANGER_USERSYNC"> + <pre-upgrade> + <task xsi:type="configure" id="usersync_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade/> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + + <component name="RANGER_TAGSYNC"> + <pre-upgrade> + <task xsi:type="configure" id="tagsync_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade /> <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -631,7 +650,7 @@ <component name="RANGER_KMS_SERVER"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db" /> - + <task xsi:type="configure" id="kms_log4j_parameterize" /> <task xsi:type="execute" hosts="any" sequential="true"> <summary>Upgrading Ranger KMS database schema</summary> <script>scripts/kms_server.py</script> @@ -658,6 +677,7 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_4_0_0_namenode_ha_adjustments"/> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" /> + <task xsi:type="configure" id="hdfs_log4j_parameterize" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> @@ -733,6 +753,7 @@ <component name="RESOURCEMANAGER"> <pre-upgrade> + <task xsi:type="configure" id="yarn_log4j_parameterize" /> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig"> <summary>Calculating Yarn Properties for Spark Shuffle</summary> </task> @@ -770,6 +791,7 @@ <!-- These HBASE configs changed in HDP 2.3.4.0, but Ambari can't distinguish HDP 2.3.2.0 vs HDP 2.3.4.0, so easier to always do them. --> <task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db" /> + <task xsi:type="configure" id="hbase_log4j_parameterize" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> @@ -944,7 +966,7 @@ <component name="OOZIE_SERVER"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" /> - + <task xsi:type="configure" id="oozie_log4j_parameterize" /> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"> <summary>Adjusting Oozie properties</summary> </task> @@ -998,6 +1020,7 @@ <component name="FALCON_SERVER"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"/> + <task xsi:type="configure" id="falcon_log4j_parameterize" /> </pre-upgrade> <pre-downgrade/> <upgrade> @@ -1016,6 +1039,7 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db" /> <task xsi:type="configure" id="hdp_2_5_0_0_add_protocol_compatibility" /> + <task xsi:type="configure" id="kafka_log4j_parameterize" /> </pre-upgrade> <pre-downgrade/> @@ -1030,6 +1054,8 @@ <component name="KNOX_GATEWAY"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_knox_audit_db" /> + <task xsi:type="configure" id="knox_gateway_log4j_parameterize" /> + <task xsi:type="configure" id="knox_ldap_log4j_parameterize" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> @@ -1190,5 +1216,18 @@ </upgrade> </component> </service> + + <service name="ATLAS"> + <component name="ATLAS_SERVER"> + <pre-upgrade> + <task xsi:type="configure" id="atlas_log4j_parameterize" /> + </pre-upgrade> + <pre-downgrade /> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + </service> + </processing> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/4dac2783/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml index fa09448..4451306 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml @@ -139,6 +139,37 @@ <transfer operation="delete" delete-key="bind_anonymous" /> </definition> + <definition xsi:type="configure" id="admin_log4j_parameterize" summary="Parameterizing Ranger Log4J Properties"> + <type>admin-log4j</type> + <set key="ranger_xa_log_maxfilesize" value="256"/> + <set key="ranger_xa_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.xa_log_appender.MaxFileSize={{ranger_xa_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.xa_log_appender.MaxBackupIndex={{ranger_xa_log_maxbackupindex}}"/> + </definition> + + </changes> + </component> + + <component name="RANGER_USERSYNC"> + <changes> + <definition xsi:type="configure" id="usersync_log4j_parameterize" summary="Parameterizing Ranger Usersync Log4J Properties"> + <type>usersync-log4j</type> + <set key="ranger_usersync_log_maxfilesize" value="256"/> + <set key="ranger_usersync_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxFileSize = {{ranger_usersync_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxBackupIndex = {{ranger_usersync_log_maxbackupindex}}"/> + </definition> + </changes> + </component> + <component name="RANGER_TAGSYNC"> + <changes> + <definition xsi:type="configure" id="tagsync_log4j_parameterize" summary="Parameterizing Ranger Tagsync Log4J Properties"> + <type>tagsync-log4j</type> + <set key="ranger_tagsync_log_maxfilesize" value="256"/> + <set key="ranger_tagsync_log_number_of_backup_files" value="20"/> + <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxFileSize = {{ranger_tagsync_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxBackupIndex = {{ranger_tagsync_log_number_of_backup_files}}"/> + </definition> </changes> </component> </service> @@ -156,6 +187,17 @@ <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> </definition> + <definition xsi:type="configure" id="kms_log4j_parameterize" summary="Parameterizing Ranger KMS Log4J Properties"> + <type>kms-log4j</type> + <set key="ranger_kms_log_maxfilesize" value="256"/> + <set key="ranger_kms_log_maxbackupindex" value="20"/> + <set key="ranger_kms_audit_log_maxfilesize" value="256"/> + <set key="ranger_kms_audit_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms.MaxFileSize = {{ranger_kms_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms.MaxBackupIndex = {{ranger_kms_log_maxbackupindex}}"/> + <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms-audit.MaxFileSize = {{ranger_kms_audit_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms-audit.MaxBackupIndex = {{ranger_kms_audit_log_maxbackupindex}}"/> + </definition> </changes> </component> </service> @@ -179,6 +221,18 @@ <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> </definition> + <!-- HDFS Rolling properties for log4j need to be parameterized. --> + <definition xsi:type="configure" id="hdfs_log4j_parameterize" summary="Parameterizing Hdfs Log4J Properties"> + <type>hdfs-log4j</type> + <set key="hadoop_log_max_backup_size" value="256"/> + <set key="hadoop_log_number_of_backup_files" value="10"/> + <set key="hadoop_security_log_max_backup_size" value="256"/> + <set key="hadoop_security_log_number_of_backup_files" value="20"/> + <regex-replace key="content" find="log4j.appender.RFA.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RFA.MaxFileSize={{hadoop_log_max_backup_size}}MB"/> + <regex-replace key="content" find="log4j.appender.RFA.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RFA.MaxBackupIndex={{hadoop_log_number_of_backup_files}}"/> + <regex-replace key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB" replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/> + <regex-replace key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)" replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/> + </definition> </changes> </component> </service> @@ -196,6 +250,14 @@ <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> </definition> + <!-- Yarn Rolling properties for log4j need to be parameterized. --> + <definition xsi:type="configure" id="yarn_log4j_parameterize" summary="Parameterizing Yarn Log4J Properties"> + <type>yarn-log4j</type> + <set key="yarn_rm_summary_log_max_backup_size" value="256"/> + <set key="yarn_rm_summary_log_number_of_backup_files" value="20"/> + <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/> + <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/> + </definition> </changes> </component> @@ -228,6 +290,17 @@ <set key="inter.broker.protocol.version" value="0.9.0.0" /> <set key="log.message.format.version" value="0.9.0.0" /> </definition> + <definition xsi:type="configure" id="kafka_log4j_parameterize" summary="Parameterizing Kafka Log4J Properties"> + <type>kafka-log4j</type> + <set key="kafka_log_maxfilesize" value="256"/> + <set key="kafka_log_maxbackupindex" value="20"/> + <set key="controller_log_maxfilesize" value="256"/> + <set key="controller_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kafkaAppender.MaxFileSize = {{kafka_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kafkaAppender.MaxBackupIndex = {{kafka_log_maxbackupindex}}"/> + <replace key="content" find="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.controllerAppender.MaxFileSize = {{controller_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.controllerAppender.MaxBackupIndex = {{controller_log_maxbackupindex}}"/> + </definition> </changes> </component> </service> @@ -325,6 +398,18 @@ <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> </definition> + <!-- HBase Rolling properties for log4j need to be parameterized. --> + <definition xsi:type="configure" id="hbase_log4j_parameterize" summary="Parameterizing HBase Log4J Properties"> + <type>hbase-log4j</type> + <set key="hbase_log_maxfilesize" value="256"/> + <set key="hbase_log_maxbackupindex" value="20"/> + <set key="hbase_security_log_maxfilesize" value="256"/> + <set key="hbase_security_log_maxbackupindex" value="20"/> + <regex-replace key="content" find="hbase.log.maxfilesize=([0-9]+)MB" replace-with="hbase.log.maxfilesize={{hbase_log_maxfilesize}}MB"/> + <regex-replace key="content" find="hbase.log.maxbackupindex=([0-9]+)" replace-with="hbase.log.maxbackupindex={{hbase_log_maxbackupindex}}"/> + <regex-replace key="content" find="hbase.security.log.maxfilesize=([0-9]+)MB" replace-with="hbase.security.log.maxfilesize={{hbase_security_log_maxfilesize}}MB"/> + <regex-replace key="content" find="hbase.security.log.maxbackupindex=([0-9]+)" replace-with="hbase.security.log.maxbackupindex={{hbase_security_log_maxbackupindex}}"/> + </definition> </changes> </component> </service> @@ -342,6 +427,20 @@ <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> </definition> + <definition xsi:type="configure" id="knox_gateway_log4j_parameterize" summary="Parameterizing Knox Gateway Log4J Properties"> + <type>gateway-log4j</type> + <set key="knox_gateway_log_maxfilesize" value="256"/> + <set key="knox_gateway_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxFileSize = {{knox_gateway_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxBackupIndex = {{knox_gateway_log_maxbackupindex}}"/> + </definition> + <definition xsi:type="configure" id="knox_ldap_log4j_parameterize" summary="Parameterizing Knox Ldap Log4J Properties"> + <type>ldap-log4j</type> + <set key="knox_ldap_log_maxfilesize" value="256"/> + <set key="knox_ldap_log_maxbackupindex" value="20"/> + <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxFileSize = {{knox_ldap_log_maxfilesize}}MB"/> + <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxBackupIndex = {{knox_ldap_log_maxbackupindex}}"/> + </definition> </changes> </component> </service> @@ -353,8 +452,81 @@ <type>falcon-startup.properties</type> <set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.extensions.ExtensionService, org.apache.falcon.service.LifecyclePolicyMap, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService, org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}"/> </definition> + <definition xsi:type="configure" id="falcon_log4j_parameterize" summary="Parameterizing Falcon Log4J Properties"> + <type>falcon-log4j</type> + <set key="falcon_log_maxfilesize" value="256"/> + <set key="falcon_log_maxbackupindex" value="20"/> + <set key="falcon_security_log_maxfilesize" value="256"/> + <set key="falcon_security_log_maxbackupindex" value="20"/> + <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxFileSize" value="{{falcon_log_maxfilesize}}MB" />"/> + <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxBackupIndex" value="{{falcon_log_maxbackupindex}}" />"/> + <replace key="content" find="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxFileSize" value="{{falcon_security_log_maxfilesize}}MB"/>"/> + <replace key="content" find="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxBackupIndex" value="{{falcon_security_log_maxbackupindex}}"/>"/> + </definition> + </changes> + </component> + </service> + + <service name="SPARK"> + <component name="SPARK_JOBHISTORYSERVER"> + <changes> + <definition xsi:type="configure" id="hdp_2_5_0_0_spark_jobhistoryserver"> + <type>spark-defaults</type> + <transfer operation="delete" delete-key="spark.yarn.max.executor.failures" /> + </definition> + </changes> + </component> + <component name="SPARK_CLIENT"> + <changes> + <definition xsi:type="configure" id="hdp_2_5_0_0_spark_client"> + <type>spark-defaults</type> + <transfer operation="delete" delete-key="spark.yarn.max.executor.failures" /> + </definition> + </changes> + </component> + </service> + + <service name="OOZIE"> + <component name="OOZIE_SERVER"> + <changes> + <!-- Oozie Rolling properties for log4j need to be parameterized. --> + <definition xsi:type="configure" id="oozie_log4j_parameterize" summary="Parameterizing Oozie Log4J Properties"> + <type>oozie-log4j</type> + <set key="oozie_log_maxhistory" value="720"/> + <regex-replace key="content" find="^log4j.appender.oozie.RollingPolicy.MaxHistory=([0-9]+)" replace-with="log4j.appender.oozie.RollingPolicy.MaxHistory={{oozie_log_maxhistory}}"/> + </definition> + </changes> + </component> + </service> + + <service name="ZOOKEEPER"> + <component name="ZOOKEEPER_SERVER"> + <changes> + <!-- Zookeeper Rolling properties for log4j need to be parameterized. --> + <definition xsi:type="configure" id="zookeeper_log4j_parameterize" summary="Parameterizing ZooKeeper Log4J Properties"> + <type>zookeeper-log4j</type> + <set key="zookeeper_log_max_backup_size" value="10"/> + <set key="zookeeper_log_number_of_backup_files" value="10"/> + <regex-replace key="content" find="^log4j.appender.ROLLINGFILE.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.ROLLINGFILE.MaxFileSize={{zookeeper_log_max_backup_size}}MB"/> + <regex-replace key="content" find="^#log4j.appender.ROLLINGFILE.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.ROLLINGFILE.MaxBackupIndex={{zookeeper_log_number_of_backup_files}}"/> + </definition> + </changes> + </component> + </service> + + <service name="ATLAS"> + <component name="ATLAS_SERVER"> + <changes> + <definition xsi:type="configure" id="atlas_log4j_parameterize" summary="Parameterizing Atlas Log4J Properties"> + <type>atlas-log4j</type> + <set key="atlas_log_max_backup_size" value="256"/> + <set key="atlas_log_number_of_backup_files" value="20"/> + <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">\n<param name="MaxFileSize" value="{{atlas_log_max_backup_size}}MB" />"/> + <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">\n<param name="MaxFileSize" value="{{atlas_log_number_of_backup_files}}" />"/> + </definition> </changes> </component> </service> - </services> + + </services> </upgrade-config-changes>