Repository: ambari Updated Branches: refs/heads/trunk 8263f2f1b -> f4a4d0d9a
AMBARI-11048 - Oozie Upgrade Pack For HDP-2.2 to HDP-2.3 (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f4a4d0d9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f4a4d0d9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f4a4d0d9 Branch: refs/heads/trunk Commit: f4a4d0d9a224d1d679dbc21286eb57a3ecb129a9 Parents: 8263f2f Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Thu May 14 12:18:08 2015 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Thu May 14 13:40:40 2015 -0400 ---------------------------------------------------------------------- .../internal/UpgradeResourceProvider.java | 4 - .../serveraction/upgrades/ConfigureAction.java | 4 +- .../package/alerts/alert_check_oozie_server.py | 12 +- .../4.0.0.2.0/package/scripts/oozie_service.py | 17 ++- .../OOZIE/5.0.0.2.3/configuration/oozie-env.xml | 142 +++++++++++++++++++ .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 52 +++++-- .../stacks/2.0.6/OOZIE/test_oozie_server.py | 4 + .../python/stacks/2.2/KNOX/test_knox_gateway.py | 2 + 8 files changed, 216 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index bdb2784..ca72934 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -1041,10 +1041,6 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider detail.append(String.format("/%s to %s", key, value)); } - if (!transfers.isEmpty()) { - detail.append(String.format("; transferring %d properties", transfers.size())); - } - itemDetail = detail.toString(); if (null != ct.summary) { http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java index ce03861..645c4bb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java @@ -209,7 +209,9 @@ public class ConfigureAction extends AbstractServerAction { changedValues = true; // append standard output - outputBuffer.append(MessageFormat.format("Copied {0}/{1}\n", configType, key)); + outputBuffer.append(MessageFormat.format("Copied {0}/{1}\n", configType, + transfer.toKey)); + } else if (StringUtils.isNotBlank(transfer.defaultValue)) { newValues.put(transfer.toKey, transfer.defaultValue); changedValues = true; http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py index c0dc18a..a4a24b7 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py @@ -39,6 +39,8 @@ OOZIE_URL_KEY = '{{oozie-site/oozie.base.url}}' SECURITY_ENABLED = '{{cluster-env/security_enabled}}' OOZIE_PRINCIPAL = '{{oozie-site/oozie.authentication.kerberos.principal}}' OOZIE_KEYTAB = '{{oozie-site/oozie.authentication.kerberos.keytab}}' +OOZIE_CONF_DIR = '/usr/hdp/current/oozie-server/conf' +OOZIE_CONF_DIR_LEGACY = '/etc/oozie/conf' class KerberosPropertiesNotFound(Exception): pass @@ -107,7 +109,15 @@ def get_check_command(oozie_url, host_name, configurations): # kinit Execute(kinit_command, environment=kerberos_env) - command = format("source /etc/oozie/conf/oozie-env.sh ; oozie admin -oozie {oozie_url} -status") + + # oozie configuration directory uses a symlink when > HDP 2.2 + oozie_config_directory = OOZIE_CONF_DIR_LEGACY + if os.path.exists(OOZIE_CONF_DIR): + oozie_config_directory = OOZIE_CONF_DIR + + command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format( + oozie_config_directory, oozie_url) + return (command, kerberos_env) def execute(configurations={}, parameters={}, host_name=None): http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py index 34ac6e5..ef721ba 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py @@ -44,6 +44,8 @@ def oozie_service(action = 'start', rolling_restart=False): """ import params + environment={'OOZIE_CONFIG': params.conf_dir} + if params.security_enabled: if params.oozie_principal is None: oozie_principal_with_host = 'missing_principal' @@ -88,13 +90,16 @@ def oozie_service(action = 'start', rolling_restart=False): params.oozie_user) Execute( cmd2, user = params.oozie_user, not_if = not_if_command, path = params.execute_path ) - - Execute( start_cmd, user = params.oozie_user, not_if = no_op_test ) + + # start oozie + Execute( start_cmd, environment=environment, user = params.oozie_user, + not_if = no_op_test ) elif action == 'stop': stop_cmd = format("cd {oozie_tmp_dir} && {oozie_home}/bin/oozie-stop.sh") - Execute(stop_cmd, only_if = no_op_test, user = params.oozie_user) - File(params.pid_file, action = "delete") - - + # stop oozie + Execute(stop_cmd, environment=environment, only_if = no_op_test, + user = params.oozie_user) + + File(params.pid_file, action = "delete") http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/configuration/oozie-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/configuration/oozie-env.xml b/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/configuration/oozie-env.xml new file mode 100644 index 0000000..ac12d7c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/configuration/oozie-env.xml @@ -0,0 +1,142 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>oozie_user</name> + <value>oozie</value> + <property-type>USER</property-type> + <description>Oozie User.</description> + </property> + <property> + <name>oozie_database</name> + <value>New Derby Database</value> + <description>Oozie Server Database.</description> + </property> + <property> + <name>oozie_derby_database</name> + <value>Derby</value> + <description>Oozie Derby Database</description> + </property> + <property> + <name>oozie_data_dir</name> + <value>/hadoop/oozie/data</value> + <description>Data directory in which the Oozie DB exists</description> + </property> + <property> + <name>oozie_log_dir</name> + <value>/var/log/oozie</value> + <description>Directory for oozie logs</description> + </property> + <property> + <name>oozie_pid_dir</name> + <value>/var/run/oozie</value> + <description>Directory in which the pid files for oozie reside.</description> + </property> + <property> + <name>oozie_admin_port</name> + <value>11001</value> + <description>The admin port Oozie server runs.</description> + </property> + <property> + <name>oozie_heapsize</name> + <value>2048</value> + <description>Oozie heap size.</description> + </property> + <property> + <name>oozie_permsize</name> + <value>256</value> + <description>Oozie permanent generation size.</description> + </property> + + <!-- oozie-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for oozie-env.sh file</description> + <value> +#!/bin/bash + +if [ -d "/usr/lib/bigtop-tomcat" ]; then + export OOZIE_CONFIG=${OOZIE_CONFIG:-{{conf_dir}}} + export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}} + export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie} + export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat +fi + +#Set JAVA HOME +export JAVA_HOME={{java_home}} + +export JRE_HOME=${JAVA_HOME} + +# Set Oozie specific environment variables here. + +# Settings for the Embedded Tomcat that runs Oozie +# Java System properties for Oozie should be specified in this variable +# +{% if java_version < 8 %} +export CATALINA_OPTS="$CATALINA_OPTS -Xmx{{oozie_heapsize}} -XX:MaxPermSize={{oozie_permsize}}" +{% else %} +export CATALINA_OPTS="$CATALINA_OPTS -Xmx{{oozie_heapsize}}" +{% endif %} +# Oozie configuration file to load from Oozie configuration directory +# +# export OOZIE_CONFIG_FILE=oozie-site.xml + +# Oozie logs directory +# +export OOZIE_LOG={{oozie_log_dir}} + +# Oozie pid directory +# +export CATALINA_PID={{pid_file}} + +#Location of the data for oozie +export OOZIE_DATA={{oozie_data_dir}} + +# Oozie Log4J configuration file to load from Oozie configuration directory +# +# export OOZIE_LOG4J_FILE=oozie-log4j.properties + +# Reload interval of the Log4J configuration file, in seconds +# +# export OOZIE_LOG4J_RELOAD=10 + +# The port Oozie server runs +# +export OOZIE_HTTP_PORT={{oozie_server_port}} + +# The admin port Oozie server runs +# +export OOZIE_ADMIN_PORT={{oozie_server_admin_port}} + +# The host name Oozie server runs on +# +# export OOZIE_HTTP_HOSTNAME=`hostname -f` + +# The base URL for callback URLs to Oozie +# +# export OOZIE_BASE_URL="http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie"; +export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64 + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml index 64a621e..eb89dbd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml @@ -21,7 +21,6 @@ <target>2.3.*.*</target> <target-stack>HDP-2.3</target-stack> <order> - <group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Backups"> <direction>UPGRADE</direction> <execute-stage service="HDFS" component="NAMENODE" title="Pre Upgrade HDFS"> @@ -63,7 +62,7 @@ <component>ZOOKEEPER_SERVER</component> </service> </group> - + <group name="RANGER" title="Ranger"> <skippable>true</skippable> <service name="RANGER"> @@ -91,9 +90,9 @@ <service name="HBASE"> <component>HBASE_MASTER</component> </service> - + </group> - + <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> @@ -107,13 +106,13 @@ <service>AMBARI_METRICS</service> </exclude> </group> - + <group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated"> <service-check>false</service-check> <service name="HDFS"> <component>DATANODE</component> </service> - + <service name="HBASE"> <component>HBASE_REGIONSERVER</component> </service> @@ -128,7 +127,7 @@ <message>The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services.</message> </batch> </group> - + <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> @@ -266,7 +265,6 @@ </service> </group> - <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}"> <skippable>true</skippable> <execute-stage title="Check Unhealthy Hosts" id="unhealthy-hosts"> @@ -274,7 +272,7 @@ <message>The following hosts were unhealthy and should be resolved before finalizing can be completed: {{hosts.unhealthy}}</message> </task> </execute-stage> - + <execute-stage title="Confirm Finalize"> <direction>UPGRADE</direction> <task xsi:type="manual"> @@ -577,6 +575,30 @@ <service name="OOZIE"> <component name="OOZIE_SERVER"> + <pre-upgrade> + <task xsi:type="configure"> + <summary>Updating oozie-site to remove redundant configurations</summary> + <type>oozie-site</type> + <transfer operation="DELETE" delete-key="*" preserve-edits="true"> + <keep-key>oozie.base.url</keep-key> + <keep-key>oozie.services.ext</keep-key> + <keep-key>oozie.db.schema.name</keep-key> + <keep-key>oozie.service.JPAService.jdbc.username</keep-key> + <keep-key>oozie.service.JPAService.jdbc.password</keep-key> + <keep-key>oozie.service.JPAService.jdbc.driver</keep-key> + <keep-key>oozie.service.JPAService.jdbc.url</keep-key> + <keep-key>oozie.service.AuthorizationService.security.enabled</keep-key> + <keep-key>oozie.authentication.type</keep-key> + <keep-key>oozie.authentication.simple.anonymous.allowed</keep-key> + <keep-key>oozie.authentication.kerberos.name.rules</keep-key> + <keep-key>oozie.credentials.credentialclasses</keep-key> + <keep-key>oozie.service.AuthorizationService.authorization.enabled</keep-key> + <keep-key>oozie.service.HadoopAccessorService.hadoop.configurations</keep-key> + <keep-key>oozie.service.HadoopAccessorService.kerberos.enabled</keep-key> + <keep-key>oozie.service.URIHandlerService.uri.handlers</keep-key> + </transfer> + </task> + </pre-upgrade> <upgrade> <task xsi:type="restart" /> </upgrade> @@ -590,6 +612,18 @@ <service name="FALCON"> <component name="FALCON_SERVER"> + <pre-upgrade> + <task xsi:type="configure"> + <type>oozie-site</type> + <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-job-submit-instances" to-key="oozie.service.ELService.ext.functions.coord-job-submit-instances"/> + <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-action-create-inst" to-key="oozie.service.ELService.ext.functions.coord-action-create-inst"/> + <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-action-create" to-key="oozie.service.ELService.ext.functions.coord-action-create"/> + <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-job-submit-data" to-key="oozie.service.ELService.ext.functions.coord-job-submit-data"/> + <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-action-start" to-key="oozie.service.ELService.ext.functions.coord-action-start"/> + <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-sla-submit" to-key="oozie.service.ELService.ext.functions.coord-sla-submit"/> + <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-sla-create" to-key="oozie.service.ELService.ext.functions.coord-sla-create"/> + </task> + </pre-upgrade> <upgrade> <task xsi:type="restart" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py index bb1a037..2bf2af7 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py @@ -255,6 +255,7 @@ class TestOozieServer(RMFTestCase): path = ['/usr/bin:/usr/bin'], ) self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-start.sh', + environment = {'OOZIE_CONFIG': '/etc/oozie/conf'}, not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', user = 'oozie', ) @@ -270,6 +271,7 @@ class TestOozieServer(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-stop.sh', + environment = {'OOZIE_CONFIG': '/etc/oozie/conf'}, only_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', user = 'oozie', ) @@ -312,6 +314,7 @@ class TestOozieServer(RMFTestCase): path = ['/usr/bin:/usr/bin'], ) self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-start.sh', + environment = {'OOZIE_CONFIG': '/etc/oozie/conf'}, not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', user = 'oozie', ) @@ -326,6 +329,7 @@ class TestOozieServer(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-stop.sh', + environment = {'OOZIE_CONFIG': '/etc/oozie/conf'}, only_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', user = 'oozie', ) http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py index f2e4e50..bab4ed1 100644 --- a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py +++ b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py @@ -21,7 +21,9 @@ import json from resource_management import * from stacks.utils.RMFTestCase import * from mock.mock import patch +from mock.mock import MagicMock +@patch("platform.linux_distribution", new = MagicMock(return_value="Linux")) class TestKnoxGateway(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "KNOX/0.5.0.2.2/package" STACK_VERSION = "2.2"
