Repository: ambari Updated Branches: refs/heads/branch-2.2 6f1c1b447 -> 311382a8c
Revert "AMBARI-14678. RU - Oozie HA, Service Check fails since the pre-upgrade directive has multiple tasks that should run only on the specified hosts (alejandro)" This reverts commit 872a9ad5a280034e77e81e223b9b924b268e2c86 due to Unit Test failures in branch-2.2: https://builds.apache.org/job/Ambari-branch-2.2/218/ Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/311382a8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/311382a8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/311382a8 Branch: refs/heads/branch-2.2 Commit: 311382a8c27ac7a3ffcabe0f7270a8a30eef1178 Parents: 6f1c1b4 Author: Yusaku Sako <yus...@hortonworks.com> Authored: Wed Jan 20 13:35:43 2016 -0800 Committer: Yusaku Sako <yus...@hortonworks.com> Committed: Wed Jan 20 13:35:43 2016 -0800 ---------------------------------------------------------------------- .../stack/upgrade/StageWrapperBuilder.java | 2 +- .../ambari/server/state/stack/upgrade/Task.java | 8 - .../server/state/stack/upgrade/TaskWrapper.java | 13 -- .../state/stack/upgrade/TaskWrapperBuilder.java | 10 -- .../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml | 5 +- .../HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml | 29 ++- .../HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml | 29 ++- .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml | 29 ++- .../stacks/HDP/2.2/upgrades/upgrade-2.2.xml | 8 +- .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 8 +- .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 8 +- .../HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml | 29 ++- .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 29 ++- .../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 8 +- .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 8 +- .../HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml | 29 ++- .../stacks/HDP/2.4/upgrades/upgrade-2.4.xml | 8 +- .../ambari/server/state/UpgradeHelperTest.java | 166 ++++------------- .../HDP/2.0.7/services/OOZIE/metainfo.xml | 176 ------------------- .../HDP/2.1.1/upgrades/upgrade_bucket_test.xml | 20 ++- .../upgrades/upgrade_nonrolling_new_stack.xml | 6 +- .../stacks/HDP/2.1.1/upgrades/upgrade_test.xml | 45 +---- 22 files changed, 239 insertions(+), 434 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java index a394cd0..7efe608 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java @@ -169,7 +169,7 @@ public abstract class StageWrapperBuilder { */ protected String getStageText(String prefix, String component, Set<String> hosts, int batchNum, int totalBatches) { String stageText = getStageText(prefix, component, hosts); - String batchText = 1 == totalBatches? "" : String.format(" (Batch %s of %s)", batchNum, totalBatches); + String batchText = 1 == totalBatches? "" : String.format(" ( Batch %s of %s )", batchNum, totalBatches); return stageText + batchText; } http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java index f365a7f..980f88e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java @@ -17,7 +17,6 @@ */ package org.apache.ambari.server.state.stack.upgrade; -import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlSeeAlso; @@ -35,13 +34,6 @@ public abstract class Task { public String summary; /** - * Whether the task needs to run sequentially, i.e., on its own stage. - * If false, will be grouped with other tasks. - */ - @XmlAttribute(name = "sequential") - public boolean isSequential = false; - - /** * @return the type of the task */ public abstract Type getType(); http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java index 5fdf91c..69b3f8b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java @@ -112,17 +112,4 @@ public class TaskWrapper { return component; } - /** - * @return true if any task is sequential, otherwise, return false. - */ - public boolean isAnyTaskSequential() { - for (Task t : getTasks()) { - if (t.isSequential) { - return true; - } - } - - return false; - } - } http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java index 81a3a4d..057c310 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Set; import org.apache.ambari.server.stack.HostsType; -import org.apache.ambari.server.utils.StageUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,16 +47,8 @@ public class TaskWrapperBuilder { * @param params additional parameters */ public static List<TaskWrapper> getTaskList(String service, String component, HostsType hostsType, List<Task> tasks, Map<String, String> params) { - // Ok if Ambari Server is not part of the cluster hosts since this is only used in the calculation of how many batches - // to create. - String ambariServerHostname = StageUtils.getHostName(); - List<TaskWrapper> collection = new ArrayList<TaskWrapper>(); for (Task t : tasks) { - if (t.getType().equals(Task.Type.CONFIGURE) || t.getType().equals(Task.Type.MANUAL)) { - collection.add(new TaskWrapper(service, component, Collections.singleton(ambariServerHostname), params, t)); - continue; - } if (t.getType().equals(Task.Type.EXECUTE)) { ExecuteTask et = (ExecuteTask) t; if (et.hosts == ExecuteHostType.MASTER) { @@ -79,7 +70,6 @@ public class TaskWrapperBuilder { continue; } } - // Otherwise, meant to run on ALL hosts. } collection.add(new TaskWrapper(service, component, hostsType.hosts, params, t)); http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml index e581506..90e3b37 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml @@ -793,15 +793,14 @@ from HDP 2.1->2.3. --> <task xsi:type="configure_function"/> - - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> </pre-upgrade> <pre-downgrade> - <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Create a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml index 098682b..4febe52 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml @@ -415,6 +415,31 @@ </service> </group> + <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. --> + <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database"> + <direction>UPGRADE</direction> + <skippable>true</skippable> + <supports-auto-skip-failure>false</supports-auto-skip-failure> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>upgrade_oozie_database_and_sharelib</function> + </task> + </execute-stage> + </group> + + <!-- Only create the ShareLib folder during a Downgrade. --> + <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib"> + <direction>DOWNGRADE</direction> + <skippable>true</skippable> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>create_sharelib</function> + </task> + </execute-stage> + </group> + <group xsi:type="restart" name="OOZIE" title="Oozie"> <service-check>false</service-check> <skippable>true</skippable> @@ -749,14 +774,14 @@ <service name="OOZIE"> <component name="OOZIE_SERVER"> <pre-upgrade> - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> </pre-upgrade> <pre-downgrade> - <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Create a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml index 6dd132a..5014728 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml @@ -689,6 +689,31 @@ </service> </group> + <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. --> + <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database"> + <direction>UPGRADE</direction> + <skippable>true</skippable> + <supports-auto-skip-failure>false</supports-auto-skip-failure> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>upgrade_oozie_database_and_sharelib</function> + </task> + </execute-stage> + </group> + + <!-- Only create the ShareLib folder during a Downgrade. --> + <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib"> + <direction>DOWNGRADE</direction> + <skippable>true</skippable> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>create_sharelib</function> + </task> + </execute-stage> + </group> + <group xsi:type="restart" name="OOZIE" title="Oozie"> <service-check>false</service-check> <skippable>true</skippable> @@ -1037,14 +1062,14 @@ <service name="OOZIE"> <component name="OOZIE_SERVER"> <pre-upgrade> - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> </pre-upgrade> <pre-downgrade> - <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Create a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml index c5d74e2..8feb7cb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml @@ -753,6 +753,31 @@ </service> </group> + <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. --> + <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database"> + <direction>UPGRADE</direction> + <skippable>true</skippable> + <supports-auto-skip-failure>false</supports-auto-skip-failure> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>upgrade_oozie_database_and_sharelib</function> + </task> + </execute-stage> + </group> + + <!-- Only create the ShareLib folder during a Downgrade. --> + <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib"> + <direction>DOWNGRADE</direction> + <skippable>true</skippable> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>create_sharelib</function> + </task> + </execute-stage> + </group> + <group xsi:type="restart" name="OOZIE" title="Oozie"> <service-check>false</service-check> <skippable>true</skippable> @@ -1128,14 +1153,14 @@ <service name="OOZIE"> <component name="OOZIE_SERVER"> <pre-upgrade> - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> </pre-upgrade> <pre-downgrade> - <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Create a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml index d2f6f48..89505f8 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml @@ -594,24 +594,24 @@ <service name="OOZIE"> <component name="OOZIE_SERVER"> <pre-upgrade> - <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers"> + <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers"> <script>scripts/oozie_server.py</script> <function>stop</function> </task> - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> </pre-upgrade> <pre-downgrade> - <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers"> + <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers"> <script>scripts/oozie_server.py</script> <function>stop</function> </task> - <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Create a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml index 09ab2b9..9ac2ca6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml @@ -746,24 +746,24 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/> - <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers"> + <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers"> <script>scripts/oozie_server.py</script> <function>stop</function> </task> - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> </pre-upgrade> <pre-downgrade> - <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers"> + <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers"> <script>scripts/oozie_server.py</script> <function>stop</function> </task> - <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Create a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml index 6b5bcb1..4345f43 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml @@ -753,24 +753,24 @@ <task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/> <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" /> - <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers"> + <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers"> <script>scripts/oozie_server.py</script> <function>stop</function> </task> - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> </pre-upgrade> <pre-downgrade> - <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers"> + <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers"> <script>scripts/oozie_server.py</script> <function>stop</function> </task> - <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Create a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml index d490ce0..4d2fb6b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml @@ -473,6 +473,31 @@ </service> </group> + <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. --> + <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database"> + <direction>UPGRADE</direction> + <skippable>true</skippable> + <supports-auto-skip-failure>false</supports-auto-skip-failure> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>upgrade_oozie_database_and_sharelib</function> + </task> + </execute-stage> + </group> + + <!-- Only create the ShareLib folder during a Downgrade. --> + <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib"> + <direction>DOWNGRADE</direction> + <skippable>true</skippable> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>create_sharelib</function> + </task> + </execute-stage> + </group> + <group xsi:type="restart" name="OOZIE" title="Oozie"> <service-check>false</service-check> <skippable>true</skippable> @@ -854,14 +879,14 @@ <service name="OOZIE"> <component name="OOZIE_SERVER"> <pre-upgrade> - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> </pre-upgrade> <pre-downgrade> - <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Create a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml index 45bf744..d6d221b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml @@ -492,6 +492,31 @@ </service> </group> + <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. --> + <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database"> + <direction>UPGRADE</direction> + <skippable>true</skippable> + <supports-auto-skip-failure>false</supports-auto-skip-failure> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>upgrade_oozie_database_and_sharelib</function> + </task> + </execute-stage> + </group> + + <!-- Only create the ShareLib folder during a Downgrade. --> + <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib"> + <direction>DOWNGRADE</direction> + <skippable>true</skippable> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>create_sharelib</function> + </task> + </execute-stage> + </group> + <group xsi:type="restart" name="OOZIE" title="Oozie"> <service-check>false</service-check> <skippable>true</skippable> @@ -879,14 +904,14 @@ <service name="OOZIE"> <component name="OOZIE_SERVER"> <pre-upgrade> - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> </pre-upgrade> <pre-downgrade> - <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Create a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml index bc0ce3f..3d7b6ed 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml @@ -689,24 +689,24 @@ <service name="OOZIE"> <component name="OOZIE_SERVER"> <pre-upgrade> - <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers"> + <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers"> <script>scripts/oozie_server.py</script> <function>stop</function> </task> - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> </pre-upgrade> <pre-downgrade> - <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers"> + <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers"> <script>scripts/oozie_server.py</script> <function>stop</function> </task> - <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Create a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml index 682815f..930a026 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml @@ -669,24 +669,24 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" /> - <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers"> + <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers"> <script>scripts/oozie_server.py</script> <function>stop</function> </task> - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> </pre-upgrade> <pre-downgrade> - <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers"> + <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers"> <script>scripts/oozie_server.py</script> <function>stop</function> </task> - <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Create a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml index 2600eae..a9ee860 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml @@ -474,6 +474,31 @@ </service> </group> + <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. --> + <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database"> + <direction>UPGRADE</direction> + <skippable>true</skippable> + <supports-auto-skip-failure>false</supports-auto-skip-failure> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>upgrade_oozie_database_and_sharelib</function> + </task> + </execute-stage> + </group> + + <!-- Only create the ShareLib folder during a Downgrade. --> + <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib"> + <direction>DOWNGRADE</direction> + <skippable>true</skippable> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>create_sharelib</function> + </task> + </execute-stage> + </group> + <group xsi:type="restart" name="OOZIE" title="Oozie"> <service-check>false</service-check> <skippable>true</skippable> @@ -855,14 +880,14 @@ <service name="OOZIE"> <component name="OOZIE_SERVER"> <pre-upgrade> - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> </pre-upgrade> <pre-downgrade> - <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Create a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml index b91e27a..b8bd1ee 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml @@ -687,24 +687,24 @@ <service name="OOZIE"> <component name="OOZIE_SERVER"> <pre-upgrade> - <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers"> + <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers"> <script>scripts/oozie_server.py</script> <function>stop</function> </task> - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> </pre-upgrade> <pre-downgrade> - <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers"> + <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers"> <script>scripts/oozie_server.py</script> <function>stop</function> </task> - <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Create a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java index 320744f..b9406ba 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java @@ -52,8 +52,6 @@ import org.apache.ambari.server.state.stack.UpgradePack; import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition; import org.apache.ambari.server.state.stack.upgrade.ConfigureTask; import org.apache.ambari.server.state.stack.upgrade.Direction; -import org.apache.ambari.server.state.stack.upgrade.ExecuteTask; -import org.apache.ambari.server.state.stack.upgrade.Grouping; import org.apache.ambari.server.state.stack.upgrade.ManualTask; import org.apache.ambari.server.state.stack.upgrade.StageWrapper; import org.apache.ambari.server.state.stack.upgrade.Task; @@ -194,14 +192,13 @@ public class UpgradeHelperTest { List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(7, groups.size()); + assertEquals(6, groups.size()); assertEquals("PRE_CLUSTER", groups.get(0).name); assertEquals("ZOOKEEPER", groups.get(1).name); assertEquals("CORE_MASTER", groups.get(2).name); assertEquals("CORE_SLAVES", groups.get(3).name); assertEquals("HIVE", groups.get(4).name); - assertEquals("OOZIE", groups.get(5).name); UpgradeGroupHolder holder = groups.get(2); boolean found = false; @@ -231,7 +228,7 @@ public class UpgradeHelperTest { assertTrue(mt.structuredOut.contains("DATANODE")); assertTrue(mt.structuredOut.contains("NODEMANAGER")); - UpgradeGroupHolder postGroup = groups.get(6); + UpgradeGroupHolder postGroup = groups.get(5); assertEquals("POST_CLUSTER", postGroup.name); assertEquals("Finalize Upgrade", postGroup.title); assertEquals(3, postGroup.items.size()); @@ -242,7 +239,7 @@ public class UpgradeHelperTest { assertEquals(4, groups.get(0).items.size()); assertEquals(6, groups.get(1).items.size()); - assertEquals(9, groups.get(2).items.size()); + assertEquals(8, groups.get(2).items.size()); assertEquals(8, groups.get(3).items.size()); } @@ -298,7 +295,7 @@ public class UpgradeHelperTest { List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(7, groups.size()); + assertEquals(6, groups.size()); UpgradeGroupHolder mastersGroup = groups.get(2); assertEquals("CORE_MASTER", mastersGroup.name); @@ -347,16 +344,14 @@ public class UpgradeHelperTest { List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(7, groups.size()); + assertEquals(6, groups.size()); assertEquals("PRE_CLUSTER", groups.get(0).name); assertEquals("ZOOKEEPER", groups.get(1).name); assertEquals("CORE_MASTER", groups.get(2).name); assertEquals("CORE_SLAVES", groups.get(3).name); - assertEquals("HIVE", groups.get(4).name); - assertEquals("OOZIE", groups.get(5).name); - UpgradeGroupHolder postGroup = groups.get(6); + UpgradeGroupHolder postGroup = groups.get(5); assertEquals("POST_CLUSTER", postGroup.name); assertEquals("Finalize Upgrade", postGroup.title); assertEquals(3, postGroup.items.size()); @@ -366,7 +361,7 @@ public class UpgradeHelperTest { assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType()); assertEquals(6, groups.get(1).items.size()); - assertEquals(9, groups.get(2).items.size()); + assertEquals(8, groups.get(2).items.size()); assertEquals(7, groups.get(3).items.size()); } @@ -384,17 +379,17 @@ public class UpgradeHelperTest { List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(7, groups.size()); + assertEquals(6, groups.size()); - assertEquals("PRE_CLUSTER", groups.get(0).name); - assertEquals("OOZIE", groups.get(1).name); - assertEquals("HIVE", groups.get(2).name); - assertEquals("CORE_SLAVES", groups.get(3).name); - assertEquals("CORE_MASTER", groups.get(4).name); - assertEquals("ZOOKEEPER", groups.get(5).name); + UpgradeGroupHolder preGroup = groups.get(0); + assertEquals("PRE_CLUSTER", preGroup.name); + assertEquals("HIVE", groups.get(1).name); + assertEquals("CORE_SLAVES", groups.get(2).name); + assertEquals("CORE_MASTER", groups.get(3).name); + assertEquals("ZOOKEEPER", groups.get(4).name); - UpgradeGroupHolder postGroup = groups.get(6); + UpgradeGroupHolder postGroup = groups.get(5); assertEquals("POST_CLUSTER", postGroup.name); assertEquals("Finalize Downgrade", postGroup.title); assertEquals(3, postGroup.items.size()); @@ -404,10 +399,10 @@ public class UpgradeHelperTest { assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType()); assertEquals(4, groups.get(0).items.size()); - assertEquals(8, groups.get(1).items.size()); - assertEquals(3, groups.get(2).items.size()); - assertEquals(8, groups.get(3).items.size()); - assertEquals(8, groups.get(4).items.size()); + assertEquals(2, groups.get(1).items.size()); + assertEquals(8, groups.get(2).items.size()); + assertEquals(7, groups.get(3).items.size()); + assertEquals(5, groups.get(4).items.size()); } @Test @@ -427,17 +422,7 @@ public class UpgradeHelperTest { assertEquals(1, groups.size()); UpgradeGroupHolder group = groups.iterator().next(); - // Pre: - // Manual task = 1 - // 2x - Execute task on all 3 = 6 - - // Post: - // Execute task on all 3 = 3 - // 2x - Manual task = 2 - // 3x - Execute task on all 3 = 9 - - // Service Check = 1 - assertEquals(22, group.items.size()); + assertEquals(6, group.items.size()); } @Test @@ -454,7 +439,7 @@ public class UpgradeHelperTest { List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(7, groups.size()); + assertEquals(6, groups.size()); // grab the manual task out of ZK which has placeholder text UpgradeGroupHolder zookeeperGroup = groups.get(1); @@ -482,12 +467,13 @@ public class UpgradeHelperTest { List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(7, groups.size()); + assertEquals(6, groups.size()); // grab the configure task out of Hive UpgradeGroupHolder hiveGroup = groups.get(4); assertEquals("HIVE", hiveGroup.name); - ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get(0).getTasks().get(0); + ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get( + 1).getTasks().get(0); // now change the thrift port to http to have the 2nd condition invoked Map<String, String> hiveConfigs = new HashMap<String, String>(); @@ -539,8 +525,10 @@ public class UpgradeHelperTest { assertEquals("delete-if-key-present", transfers.get(8).deleteKey); assertEquals("delete-if-key-absent", transfers.get(9).deleteKey); + } + @Test public void testConfigureTask() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); @@ -557,7 +545,7 @@ public class UpgradeHelperTest { List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(7, groups.size()); + assertEquals(6, groups.size()); // grab the configure task out of Hive UpgradeGroupHolder hiveGroup = groups.get(4); @@ -632,12 +620,12 @@ public class UpgradeHelperTest { List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(7, groups.size()); + assertEquals(6, groups.size()); // grab the configure task out of Hive UpgradeGroupHolder hiveGroup = groups.get(4); assertEquals("HIVE", hiveGroup.name); - ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get(0).getTasks().get(0); + ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(1).getTasks().get(0); Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup); assertFalse(configProperties.isEmpty()); @@ -712,7 +700,7 @@ public class UpgradeHelperTest { UpgradeGroupHolder holder = groups.get(3); assertEquals(holder.name, "SERVICE_CHECK_1"); - assertEquals(7, holder.items.size()); + assertEquals(6, holder.items.size()); int numServiceChecksActual = 0; for (StageWrapper sw : holder.items) { for(Service service : services) { @@ -839,7 +827,7 @@ public class UpgradeHelperTest { assertEquals(4, groups.get(0).items.size()); assertEquals(5, groups.get(1).items.size()); - assertEquals(9, groups.get(2).items.size()); + assertEquals(8, groups.get(2).items.size()); assertEquals(8, groups.get(3).items.size()); } @@ -892,7 +880,6 @@ public class UpgradeHelperTest { c.addService(serviceFactory.createNew(c, "YARN")); c.addService(serviceFactory.createNew(c, "ZOOKEEPER")); c.addService(serviceFactory.createNew(c, "HIVE")); - c.addService(serviceFactory.createNew(c, "OOZIE")); Service s = c.getService("HDFS"); ServiceComponent sc = s.addServiceComponent("NAMENODE"); @@ -921,16 +908,6 @@ public class UpgradeHelperTest { sc = s.addServiceComponent("HIVE_SERVER"); sc.addServiceComponentHost("h2"); - s = c.getService("OOZIE"); - // Oozie Server HA - sc = s.addServiceComponent("OOZIE_SERVER"); - sc.addServiceComponentHost("h2"); - sc.addServiceComponentHost("h3"); - sc = s.addServiceComponent("OOZIE_CLIENT"); - sc.addServiceComponentHost("h1"); - sc.addServiceComponentHost("h2"); - sc.addServiceComponentHost("h3"); - // set some desired configs Map<String, String> hiveConfigs = new HashMap<String, String>(); hiveConfigs.put("hive.server2.transport.mode", "binary"); @@ -976,23 +953,12 @@ public class UpgradeHelperTest { expect(m_masterHostResolver.getMasterAndHosts("YARN", "RESOURCEMANAGER")).andReturn(type).anyTimes(); type = new HostsType(); - expect(m_masterHostResolver.getMasterAndHosts("YARN", "APP_TIMELINE_SERVER")).andReturn(type).anyTimes(); - - type = new HostsType(); type.hosts.addAll(Arrays.asList("h1", "h3")); expect(m_masterHostResolver.getMasterAndHosts("YARN", "NODEMANAGER")).andReturn(type).anyTimes(); expect(m_masterHostResolver.getMasterAndHosts("HIVE", "HIVE_SERVER")).andReturn( type).anyTimes(); - type = new HostsType(); - type.hosts.addAll(Arrays.asList("h2", "h3")); - expect(m_masterHostResolver.getMasterAndHosts("OOZIE", "OOZIE_SERVER")).andReturn(type).anyTimes(); - - type = new HostsType(); - type.hosts.addAll(Arrays.asList("h1", "h2", "h3")); - expect(m_masterHostResolver.getMasterAndHosts("OOZIE", "OOZIE_CLIENT")).andReturn(type).anyTimes(); - expect(m_masterHostResolver.getCluster()).andReturn(c).anyTimes(); replay(m_masterHostResolver); @@ -1000,76 +966,6 @@ public class UpgradeHelperTest { return c; } - /** - * Test that multiple execute tasks with an annotation of synchronized="true" each run in their own stage. - */ - @Test - public void testUpgradeWithMultipleTasksInOwnStage() throws Exception { - Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - assertNotNull(upgrade); - assertTrue(upgrade.getType() == UpgradeType.ROLLING); - - List<Grouping> upgradePackGroups = upgrade.getGroups(Direction.UPGRADE); - - boolean foundService = false; - for (Grouping group : upgradePackGroups) { - if (group.title.equals("Oozie")) { - foundService = true; - } - } - assertTrue(foundService); - - makeCluster(); - - UpgradeContext context = new UpgradeContext(m_masterHostResolver, HDP_21, - HDP_21, UPGRADE_VERSION, Direction.UPGRADE, UpgradeType.ROLLING); - - List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); - - // The upgrade pack has 2 tasks for Oozie in the pre-upgrade group. - // The first task runs on "all", i.e., both Oozie Servers, whereas the - // second task runs on "any", i.e., exactly one. - int numPrepareStages = 0; - for (UpgradeGroupHolder group : groups) { - if (group.name.equals("OOZIE")) { - assertTrue(group.items.size() > 0); - for (StageWrapper sw : group.items) { - - if (sw.getText().equalsIgnoreCase("Preparing Oozie Server on h2 (Batch 1 of 2)") || - sw.getText().equalsIgnoreCase("Preparing Oozie Server on h3 (Batch 2 of 2)")) { - numPrepareStages++; - List<TaskWrapper> taskWrappers = sw.getTasks(); - assertEquals(1, taskWrappers.size()); - List<Task> tasks = taskWrappers.get(0).getTasks(); - assertEquals(1, taskWrappers.get(0).getHosts().size()); - assertEquals(1, tasks.size()); - - ExecuteTask task = (ExecuteTask) tasks.get(0); - assertTrue("scripts/oozie_server.py".equalsIgnoreCase(task.script)); - assertTrue("stop".equalsIgnoreCase(task.function)); - } - - if (sw.getText().equalsIgnoreCase("Preparing Oozie Server on h2")) { - numPrepareStages++; - List<TaskWrapper> taskWrappers = sw.getTasks(); - assertEquals(1, taskWrappers.size()); - List<Task> tasks = taskWrappers.get(0).getTasks(); - assertEquals(1, taskWrappers.get(0).getHosts().size()); - assertEquals(1, tasks.size()); - - ExecuteTask task = (ExecuteTask) tasks.get(0); - assertTrue("scripts/oozie_server_upgrade.py".equalsIgnoreCase(task.script)); - assertTrue("upgrade_oozie_database_and_sharelib".equalsIgnoreCase(task.function)); - } - } - } - } - assertEquals(3, numPrepareStages); - } - @Test public void testDowngradeAfterPartialUpgrade() throws Exception { http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/OOZIE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/OOZIE/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/OOZIE/metainfo.xml deleted file mode 100644 index 6010659..0000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/OOZIE/metainfo.xml +++ /dev/null @@ -1,176 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<metainfo> - <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>OOZIE</name> - <displayName>Oozie</displayName> - <comment>System for workflow coordination and execution of Apache Hadoop jobs. This also includes the installation of the optional Oozie Web Console which relies on and will install the <a target="_blank" href="http://www.sencha.com/legal/open-source-faq/">ExtJS</a> Library.</comment> - <version>4.2.0.2.3</version> - - <components> - <component> - <name>OOZIE_SERVER</name> - <displayName>Oozie Server</displayName> - <category>MASTER</category> - <cardinality>1+</cardinality> - <versionAdvertised>true</versionAdvertised> - <dependencies> - <dependency> - <name>HDFS/HDFS_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - <dependency> - <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - <dependency> - <name>YARN/YARN_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - </dependencies> - <commandScript> - <script>scripts/oozie_server.py</script> - <scriptType>PYTHON</scriptType> - <timeout>1200</timeout> - </commandScript> - </component> - - <component> - <name>OOZIE_CLIENT</name> - <displayName>Oozie Client</displayName> - <category>CLIENT</category> - <cardinality>1+</cardinality> - <versionAdvertised>true</versionAdvertised> - <dependencies> - <dependency> - <name>HDFS/HDFS_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - <dependency> - <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - </dependencies> - <commandScript> - <script>scripts/oozie_client.py</script> - <scriptType>PYTHON</scriptType> - </commandScript> - <configFiles> - <configFile> - <type>xml</type> - <fileName>oozie-site.xml</fileName> - <dictionaryName>oozie-site</dictionaryName> - </configFile> - <configFile> - <type>env</type> - <fileName>oozie-env.sh</fileName> - <dictionaryName>oozie-env</dictionaryName> - </configFile> - <configFile> - <type>env</type> - <fileName>oozie-log4j.properties</fileName> - <dictionaryName>oozie-log4j</dictionaryName> - </configFile> - </configFiles> - </component> - </components> - - <commandScript> - <script>scripts/service_check.py</script> - <scriptType>PYTHON</scriptType> - <timeout>300</timeout> - </commandScript> - - <requiredServices> - <service>YARN</service> - </requiredServices> - - <configuration-dependencies> - <config-type>oozie-site</config-type> - <config-type>oozie-env</config-type> - <config-type>oozie-log4j</config-type> - <config-type>yarn-site</config-type> - <config-type>hive-site</config-type> - <config-type>tez-site</config-type> - </configuration-dependencies> - - <osSpecifics> - <osSpecific> - <osFamily>any</osFamily> - <packages> - <package> - <name>zip</name> - </package> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - </package> - <package> - <name>extjs</name> - </package> - </packages> - </osSpecific> - <osSpecific> - <osFamily>redhat5,redhat6,redhat7,suse11</osFamily> - <packages> - <package> - <name>oozie_2_3_*</name> - </package> - <package> - <name>falcon_2_3_*</name> - </package> - </packages> - </osSpecific> - <osSpecific> - <osFamily>debian7,ubuntu12,ubuntu14</osFamily> - <packages> - <package> - <name>oozie-2-3-.*</name> - </package> - <package> - <name>falcon-2-3-.*</name> - </package> - <package> - <name>extjs</name> - </package> - <package> - <name>libxml2-utils</name> - </package> - </packages> - </osSpecific> - </osSpecifics> - </service> - </services> -</metainfo> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml index 085ba25..0e6d914 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml @@ -46,34 +46,36 @@ <component name="ZOOKEEPER_SERVER"> <pre-upgrade> <task xsi:type="manual"> - <message>pre-manual 1 task to run on all hosts</message> + <message>this is pre</message> </task> <task xsi:type="execute"> - <command>pre-ls 1 to run on all hosts</command> + <command>ls</command> </task> <task xsi:type="execute"> - <command>pre-ls 2 to run on all hosts</command> + <command>ls</command> </task> + </pre-upgrade> <post-upgrade> <task xsi:type="execute"> - <command>post-ls 1 to run on all hosts</command> + <command>ls</command> </task> <task xsi:type="manual"> - <message>post-manual 1 task to run on all hosts</message> + <message>this is pre</message> </task> <task xsi:type="manual"> - <message>post-manual 2 task to run on all hosts</message> + <message>this is pre</message> </task> <task xsi:type="execute"> - <command>post-ls 2 to run on all hosts</command> + <command>ls</command> </task> <task xsi:type="execute"> - <command>post-ls 3 to run on all hosts</command> + <command>ls</command> </task> + <task xsi:type="execute"> - <command>post-ls 4 to run on all hosts</command> + <command>ls</command> </task> </post-upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml index 868144e..ad6174c 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml @@ -442,7 +442,7 @@ <skippable>true</skippable> <supports-auto-skip-failure>false</supports-auto-skip-failure> <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database"> - <task xsi:type="execute" hosts="any" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> @@ -454,7 +454,7 @@ <direction>DOWNGRADE</direction> <skippable>true</skippable> <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib"> - <task xsi:type="execute" hosts="any" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>create_sharelib</function> </task> @@ -809,7 +809,7 @@ <service name="OOZIE"> <component name="OOZIE_SERVER"> <pre-upgrade> - <task xsi:type="execute" hosts="any" summary="Upgrading the Oozie database and creating a new sharelib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/311382a8/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml index 237da77..4f71b4a 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml @@ -101,17 +101,7 @@ <component>HIVE_SERVER</component> <component>WEBHCAT_SERVER</component> </service> - </group> - - <group name="OOZIE" title="Oozie"> - <skippable>true</skippable> - <supports-auto-skip-failure>false</supports-auto-skip-failure> - <service-check>false</service-check> - <service name="OOZIE"> - <component>OOZIE_SERVER</component> - <component>OOZIE_CLIENT</component> - </service> - </group> + </group> <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}"> <execute-stage title="Confirm Finalize"> @@ -209,38 +199,13 @@ <summary>HiveServer Port Availability</summary> <message>The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> </task> + <task xsi:type="configure" id="hdp_2_1_1_set_transport_mode"/> - <task xsi:type="configure" id="hdp_2_1_1_hive_server_foo"/> - </pre-upgrade> - </component> - </service> - <service name="OOZIE"> - <component name="OOZIE_SERVER"> - <pre-upgrade> - <!-- This is important, do not remove it since UpgradeHelperTest.java : - testUpgradeWithMultipleTasksWithMultipleHostTypes() asserts - that these tasks each run on their own stage. --> - <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers"> - <script>scripts/oozie_server.py</script> - <function>stop</function> - </task> + <task xsi:type="configure" id="hdp_2_1_1_hive_server_foo"/> - <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib"> - <script>scripts/oozie_server_upgrade.py</script> - <function>upgrade_oozie_database_and_sharelib</function> - </task> </pre-upgrade> - <upgrade> - <task xsi:type="restart-task" /> - </upgrade> - </component> - - <component name="OOZIE_CLIENT"> - <upgrade> - <task xsi:type="restart-task" /> - </upgrade> - </component> - </service> + </component> + </service> </processing> </upgrade>
