Repository: ambari Updated Branches: refs/heads/trunk 1b66baac0 -> 8c3d2acd4
AMBARI-11006 - YARN Upgrade Pack For HDP-2.2 To HDP-2.3 (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8c3d2acd Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8c3d2acd Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8c3d2acd Branch: refs/heads/trunk Commit: 8c3d2acd445dde75f0c6af7f13ef874aa071603d Parents: 1b66baa Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Thu May 7 22:44:50 2015 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Thu May 7 23:48:37 2015 -0400 ---------------------------------------------------------------------- .../serveraction/upgrades/ConfigureAction.java | 30 ++++++++++---- .../state/stack/upgrade/ConfigureTask.java | 3 ++ .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 43 +++++++++++++++++++- .../GLUSTERFS/configuration/hadoop-env.xml | 2 +- .../HIVE/configuration/webhcat-site.xml | 2 +- .../services/HDFS/configuration/hadoop-env.xml | 4 +- .../ranger-yarn-plugin-properties.xml | 4 +- .../upgrades/ConfigureActionTest.java | 22 ++++++++++ 8 files changed, 95 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java index a812169..c064ac4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java @@ -42,6 +42,7 @@ import org.apache.ambari.server.state.ConfigMergeHelper.ThreeWayValue; import org.apache.ambari.server.state.DesiredConfig; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.stack.upgrade.ConfigureTask; +import org.apache.commons.lang.StringUtils; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; @@ -186,9 +187,17 @@ public class ConfigureAction extends AbstractServerAction { for (ConfigureTask.Transfer transfer : transfers) { switch (transfer.operation) { case COPY: - if (null == transfer.fromType && base.containsKey(transfer.fromKey)) { - newValues.put(transfer.toKey, base.get(transfer.fromKey)); - changedValues = true; + // if copying from the current configuration type, then first + // determine if the key already exists; if it does not, then set a + // default if a default exists + if (null == transfer.fromType) { + if (base.containsKey(transfer.fromKey)) { + newValues.put(transfer.toKey, base.get(transfer.fromKey)); + changedValues = true; + } else if (StringUtils.isNotBlank(transfer.defaultValue)) { + newValues.put(transfer.toKey, transfer.defaultValue); + changedValues = true; + } } else { // !!! copying from another configuration Config other = cluster.getDesiredConfigByType(transfer.fromType); @@ -199,14 +208,23 @@ public class ConfigureAction extends AbstractServerAction { if (otherValues.containsKey(transfer.fromKey)) { newValues.put(transfer.toKey, otherValues.get(transfer.fromKey)); changedValues = true; + } else if (StringUtils.isNotBlank(transfer.defaultValue)) { + newValues.put(transfer.toKey, transfer.defaultValue); + changedValues = true; } } } break; case MOVE: + // if the value existed previously, then update the maps with the new + // key; otherwise if there is a default value specified, set the new + // key with the default if (newValues.containsKey(transfer.fromKey)) { newValues.put(transfer.toKey, newValues.remove(transfer.fromKey)); changedValues = true; + } else if (StringUtils.isNotBlank(transfer.defaultValue)) { + newValues.put(transfer.toKey, transfer.defaultValue); + changedValues = true; } break; @@ -261,9 +279,7 @@ public class ConfigureAction extends AbstractServerAction { config.persist(false); return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", - MessageFormat.format("Updated ''{0}'' with ''{1}={2}''", - configType, key, value), - ""); + MessageFormat.format("Updated configuration ''{0}''", configType), ""); } // !!! values are different and within the same stack. create a new @@ -279,7 +295,7 @@ public class ConfigureAction extends AbstractServerAction { m_configHelper.createConfigType(cluster, m_controller, configType, newValues, auditName, serviceVersionNote); - String message = "Updated ''{0}'' with ''{1}={2}''"; + String message = "Updated configuration ''{0}'' with ''{1}={2}''"; message = MessageFormat.format(message, configType, key, value); return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", message, ""); http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java index 1f921c9..2b68a28 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java @@ -193,6 +193,9 @@ public class ConfigureTask extends ServerSideActionTask { @XmlAttribute(name = "preserve-edits") public boolean preserveEdits = false; + @XmlAttribute(name = "default-value") + public String defaultValue; + @XmlElement(name = "keep-key") public List<String> keepKeys = new ArrayList<String>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml index 225cc63..abd95aa 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml @@ -351,6 +351,20 @@ <service name="MAPREDUCE2"> <component name="HISTORYSERVER"> + <pre-upgrade> + <task xsi:type="configure"> + <type>mapred-site</type> + <transfer operation="MOVE" from-key="mapreduce.job.speculative.speculativecap" to-key="mapreduce.job.speculative.speculative-cap-running-tasks" default-value="0.1" /> + <transfer operation="DELETE" delete-key="mapreduce.task.tmp.dir" /> + </task> + + <task xsi:type="configure"> + <type>mapred-site</type> + <key>mapreduce.fileoutputcommitter.algorithm.version</key> + <value>1</value> + </task> + </pre-upgrade> + <upgrade> <task xsi:type="restart" /> </upgrade> @@ -365,12 +379,38 @@ <service name="YARN"> <component name="APP_TIMELINE_SERVER"> + <pre-upgrade> + <task xsi:type="configure"> + <type>yarn-site</type> + <key>yarn.timeline-service.recovery.enabled</key> + <value>true</value> + </task> + + <task xsi:type="configure"> + <type>yarn-site</type> + <transfer operation="COPY" from-key="yarn.timeline-service.leveldb-timeline-store.path" to-key="yarn.timeline-service.leveldb-state-store.path"/> + </task> + + <task xsi:type="configure"> + <type>yarn-site</type> + <key>yarn.timeline-service.state-store-class</key> + <value>org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore</value> + </task> + </pre-upgrade> + <upgrade> <task xsi:type="restart" /> </upgrade> </component> <component name="RESOURCEMANAGER"> + <pre-upgrade> + <task xsi:type="configure"> + <type>yarn-site</type> + <key>yarn.node-labels.enabled</key> + <value>false</value> + </task> + </pre-upgrade> <upgrade> <task xsi:type="restart" /> </upgrade> @@ -575,8 +615,7 @@ </task> <task xsi:type="configure" summary="Setting nimbus.seeds from nimbus.host"> <type>storm-site</type> - <transfer operation="COPY" from-key="nimbus.host" - to-type="storm-site" to-key="nimbus.seeds" /> + <transfer operation="COPY" from-key="nimbus.host" to-key="nimbus.seeds" /> </task> </pre-upgrade> <upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/GLUSTERFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/GLUSTERFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/GLUSTERFS/configuration/hadoop-env.xml index bce6b53..e9078e2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/GLUSTERFS/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/GLUSTERFS/configuration/hadoop-env.xml @@ -180,7 +180,7 @@ MAPREDUCE_LIBS={{mapreduce_libs_path}} export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} if [ -d "/usr/lib/tez" ]; then - export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/tez/*:/usr/lib/tez/lib/*:/etc/tez/conf + export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/tez/*:/usr/lib/tez/lib/*:/usr/hdp/current/tez-client/conf fi # Setting path to hdfs command line http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/configuration/webhcat-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/configuration/webhcat-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/configuration/webhcat-site.xml index a7caae4..4293ea5 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/configuration/webhcat-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/configuration/webhcat-site.xml @@ -31,7 +31,7 @@ limitations under the License. <property> <name>templeton.hadoop.conf.dir</name> - <value>/etc/hadoop/conf</value> + <value>/usr/hdp/current/hadoop-client/conf</value> <description>The path to the Hadoop configuration.</description> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml index cb641d6..108805f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml @@ -140,9 +140,9 @@ export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} # added to the HADOOP_CLASSPATH if [ -d "/usr/hdp/current/tez-client" ]; then - if [ -d "/etc/tez/conf/" ]; then + if [ -d "/usr/hdp/current/tez-client/conf/" ]; then # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP. - export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf/ + export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/usr/hdp/current/tez-client/conf/ fi fi http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/ranger-yarn-plugin-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/ranger-yarn-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/ranger-yarn-plugin-properties.xml index 93035d1..7fa47d9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/ranger-yarn-plugin-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/ranger-yarn-plugin-properties.xml @@ -210,7 +210,7 @@ <property> <name>SSL_KEYSTORE_FILE_PATH</name> - <value>/etc/hadoop/conf/ranger-plugin-keystore.jks</value> + <value>/usr/hdp/current/hadoop-client/conf/ranger-plugin-keystore.jks</value> <description></description> </property> @@ -222,7 +222,7 @@ <property> <name>SSL_TRUSTSTORE_FILE_PATH</name> - <value>/etc/hadoop/conf/ranger-plugin-truststore.jks</value> + <value>/usr/hdp/current/hadoop-client/conf/ranger-plugin-truststore.jks</value> <description></description> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java index ecd55db..d9638bc 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java @@ -247,6 +247,7 @@ public class ConfigureActionTest { commandParams.put(ConfigureTask.PARAMETER_KEY, "initLimit"); commandParams.put(ConfigureTask.PARAMETER_VALUE, "11"); + // normal copy List<ConfigureTask.Transfer> transfers = new ArrayList<ConfigureTask.Transfer>(); ConfigureTask.Transfer transfer = new ConfigureTask.Transfer(); transfer.operation = TransferOperation.COPY; @@ -254,12 +255,29 @@ public class ConfigureActionTest { transfer.toKey = "copyKey"; transfers.add(transfer); + // copy with default + transfer = new ConfigureTask.Transfer(); + transfer.operation = TransferOperation.COPY; + transfer.fromKey = "copiedFromMissingKeyWithDefault"; + transfer.toKey = "copiedToMissingKeyWithDefault"; + transfer.defaultValue = "defaultValue"; + transfers.add(transfer); + + // normal move transfer = new ConfigureTask.Transfer(); transfer.operation = TransferOperation.MOVE; transfer.fromKey = "moveIt"; transfer.toKey = "movedKey"; transfers.add(transfer); + // move with default + transfer = new ConfigureTask.Transfer(); + transfer.operation = TransferOperation.MOVE; + transfer.fromKey = "movedFromKeyMissingWithDefault"; + transfer.toKey = "movedToMissingWithDefault"; + transfer.defaultValue = "defaultValue2"; + transfers.add(transfer); + transfer = new ConfigureTask.Transfer(); transfer.operation = TransferOperation.DELETE; transfer.deleteKey = "deleteIt"; @@ -300,6 +318,10 @@ public class ConfigureActionTest { assertFalse(map.containsKey("moveIt")); assertTrue(map.containsKey("movedKey")); assertFalse(map.containsKey("deletedKey")); + assertTrue(map.containsKey("copiedToMissingKeyWithDefault")); + assertEquals("defaultValue", map.get("copiedToMissingKeyWithDefault")); + assertTrue(map.containsKey("movedToMissingWithDefault")); + assertEquals("defaultValue2", map.get("movedToMissingWithDefault")); transfers.clear(); transfer = new ConfigureTask.Transfer();
