AMBARI-21794. Post Ambari upgrade on a migration cluster Oozie failed to restart when LZO compression is enabled. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/712966db Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/712966db Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/712966db Branch: refs/heads/branch-2.6 Commit: 712966dbcc748a1799c18815572ae1ec9b2748b2 Parents: 16a93a7 Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Wed Aug 23 13:52:52 2017 -0700 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Wed Aug 23 13:52:52 2017 -0700 ---------------------------------------------------------------------- .../package/alerts/alert_check_oozie_server.py | 246 +++++++++ .../services/OOZIE/package/files/oozieSmoke2.sh | 84 +++ .../files/prepareOozieHdfsDirectories.sh | 42 ++ .../OOZIE/package/files/wrap_ooziedb.sh | 31 ++ .../scripts/check_oozie_server_status.py | 38 ++ .../services/OOZIE/package/scripts/oozie.py | 549 +++++++++++++++++++ .../OOZIE/package/scripts/oozie_client.py | 76 +++ .../OOZIE/package/scripts/oozie_server.py | 159 ++++++ .../package/scripts/oozie_server_upgrade.py | 247 +++++++++ .../OOZIE/package/scripts/oozie_service.py | 186 +++++++ .../services/OOZIE/package/scripts/params.py | 39 ++ .../OOZIE/package/scripts/params_linux.py | 387 +++++++++++++ .../OOZIE/package/scripts/params_windows.py | 34 ++ .../OOZIE/package/scripts/service_check.py | 133 +++++ .../OOZIE/package/scripts/status_params.py | 65 +++ .../OOZIE/package/templates/adminusers.txt.j2 | 28 + .../package/templates/oozie-log4j.properties.j2 | 93 ++++ .../OOZIE/package/templates/oozie.conf.j2 | 35 ++ .../package/templates/zkmigrator_jaas.conf.j2 | 26 + 19 files changed, 2498 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/712966db/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/alerts/alert_check_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/alerts/alert_check_oozie_server.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/alerts/alert_check_oozie_server.py new file mode 100644 index 0000000..54eef18 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/alerts/alert_check_oozie_server.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import os +import re + +from resource_management.core import global_lock +from resource_management.core.environment import Environment +from resource_management.core.resources import Execute +from resource_management.libraries.functions import format +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import get_klist_path +from resource_management.libraries.functions import stack_tools +from ambari_commons.os_check import OSConst, OSCheck +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from urlparse import urlparse + +STACK_ROOT_PATTERN = "{{ stack_root }}" +RESULT_CODE_OK = 'OK' +RESULT_CODE_CRITICAL = 'CRITICAL' +RESULT_CODE_UNKNOWN = 'UNKNOWN' + +if OSCheck.is_windows_family(): + OOZIE_ENV_HTTPS_RE = r"set\s+OOZIE_HTTPS_PORT=(\d+)" +else: + OOZIE_ENV_HTTPS_RE = r"export\s+OOZIE_HTTPS_PORT=(\d+)" + +# The configured Kerberos executable search paths, if any +KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY = '{{kerberos-env/executable_search_paths}}' + +OOZIE_URL_KEY = '{{oozie-site/oozie.base.url}}' +SECURITY_ENABLED = '{{cluster-env/security_enabled}}' +OOZIE_USER = '{{oozie-env/oozie_user}}' +OOZIE_CONF_DIR = "{0}/current/oozie-server/conf".format(STACK_ROOT_PATTERN) +OOZIE_CONF_DIR_LEGACY = '/etc/oozie/conf' +OOZIE_HTTPS_PORT = '{{oozie-site/oozie.https.port}}' +OOZIE_ENV_CONTENT = '{{oozie-env/content}}' + +USER_KEYTAB_KEY = '{{oozie-site/oozie.service.HadoopAccessorService.keytab.file}}' +USER_PRINCIPAL_KEY = '{{oozie-site/oozie.service.HadoopAccessorService.kerberos.principal}}' +USER_KEY = '{{oozie-env/oozie_user}}' + +# default keytab location +USER_KEYTAB_SCRIPT_PARAM_KEY = 'default.oozie.keytab' +USER_KEYTAB_DEFAULT = '/etc/security/keytabs/oozie.headless.keytab' + +# default user principal +USER_PRINCIPAL_SCRIPT_PARAM_KEY = 'default.oozie.principal' +USER_PRINCIPAL_DEFAULT = 'oo...@example.com' + +# default user +USER_DEFAULT = 'oozie' + +STACK_NAME_KEY = '{{cluster-env/stack_name}}' +STACK_ROOT_KEY = '{{cluster-env/stack_root}}' +STACK_ROOT_DEFAULT = '/usr/hdp' + +class KerberosPropertiesNotFound(Exception): pass + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def get_tokens(): + """ + Returns a tuple of tokens in the format {{site/property}} that will be used + to build the dictionary passed into execute + """ + return (OOZIE_URL_KEY,) + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def get_tokens(): + """ + Returns a tuple of tokens in the format {{site/property}} that will be used + to build the dictionary passed into execute + """ + return (OOZIE_URL_KEY, USER_PRINCIPAL_KEY, SECURITY_ENABLED, USER_KEYTAB_KEY, KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY, + USER_KEY, OOZIE_HTTPS_PORT, OOZIE_ENV_CONTENT, STACK_NAME_KEY, STACK_ROOT_KEY) + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def get_check_command(oozie_url, host_name, configurations): + from resource_management.libraries.functions import reload_windows_env + reload_windows_env() + oozie_home = os.environ['OOZIE_HOME'] + oozie_cmd = os.path.join(oozie_home, 'bin', 'oozie.cmd') + command = format("cmd /c {oozie_cmd} admin -oozie {oozie_url} -status") + return (command, None, None) + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def get_check_command(oozie_url, host_name, configurations, parameters, only_kinit): + kerberos_env = None + + user = USER_DEFAULT + if USER_KEY in configurations: + user = configurations[USER_KEY] + + if is_security_enabled(configurations): + # defaults + user_keytab = USER_KEYTAB_DEFAULT + user_principal = USER_PRINCIPAL_DEFAULT + + # check script params + if USER_PRINCIPAL_SCRIPT_PARAM_KEY in parameters: + user_principal = parameters[USER_PRINCIPAL_SCRIPT_PARAM_KEY] + user_principal = user_principal.replace('_HOST', host_name.lower()) + if USER_KEYTAB_SCRIPT_PARAM_KEY in parameters: + user_keytab = parameters[USER_KEYTAB_SCRIPT_PARAM_KEY] + + # check configurations last as they should always take precedence + if USER_PRINCIPAL_KEY in configurations: + user_principal = configurations[USER_PRINCIPAL_KEY] + user_principal = user_principal.replace('_HOST', host_name.lower()) + if USER_KEYTAB_KEY in configurations: + user_keytab = configurations[USER_KEYTAB_KEY] + + # Create the kerberos credentials cache (ccache) file and set it in the environment to use + # when executing curl + env = Environment.get_instance() + ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, os.sep, os.getpid()) + kerberos_env = {'KRB5CCNAME': ccache_file} + + # Get the configured Kerberos executable search paths, if any + kerberos_executable_search_paths = None + if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations: + kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY] + + klist_path_local = get_klist_path(kerberos_executable_search_paths) + kinit_path_local = get_kinit_path(kerberos_executable_search_paths) + kinit_part_command = format("{kinit_path_local} -l 5m20s -c {ccache_file} -kt {user_keytab} {user_principal}; ") + + # Determine if we need to kinit by testing to see if the relevant cache exists and has + # non-expired tickets. Tickets are marked to expire after 5 minutes to help reduce the number + # it kinits we do but recover quickly when keytabs are regenerated + + if only_kinit: + kinit_command = kinit_part_command + else: + kinit_command = "{0} -s {1} || ".format(klist_path_local, ccache_file) + kinit_part_command + + # prevent concurrent kinit + kinit_lock = global_lock.get_lock(global_lock.LOCK_TYPE_KERBEROS) + kinit_lock.acquire() + try: + Execute(kinit_command, environment=kerberos_env, user=user) + finally: + kinit_lock.release() + + # Configure stack root + stack_root = STACK_ROOT_DEFAULT + if STACK_NAME_KEY in configurations and STACK_ROOT_KEY in configurations: + stack_root = stack_tools.get_stack_root(configurations[STACK_NAME_KEY], configurations[STACK_ROOT_KEY]).lower() + + # oozie configuration directory using a symlink + oozie_config_directory = OOZIE_CONF_DIR.replace(STACK_ROOT_PATTERN, stack_root) + if not os.path.exists(oozie_config_directory): + oozie_config_directory = OOZIE_CONF_DIR_LEGACY + + command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format( + oozie_config_directory, oozie_url) + + return (command, kerberos_env, user) + +def execute(configurations={}, parameters={}, host_name=None): + """ + Returns a tuple containing the result code and a pre-formatted result label + + Keyword arguments: + configurations (dictionary): a mapping of configuration key to value + parameters (dictionary): a mapping of script parameter key to value + host_name (string): the name of this host where the alert is running + """ + + if configurations is None: + return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.']) + + if not OOZIE_URL_KEY in configurations: + return (RESULT_CODE_UNKNOWN, ['The Oozie URL is a required parameter.']) + + https_port = None + # try to get https port form oozie-env content + if OOZIE_ENV_CONTENT in configurations: + for line in configurations[OOZIE_ENV_CONTENT].splitlines(): + result = re.match(OOZIE_ENV_HTTPS_RE, line) + + if result is not None: + https_port = result.group(1) + # or from oozie-site.xml + if https_port is None and OOZIE_HTTPS_PORT in configurations: + https_port = configurations[OOZIE_HTTPS_PORT] + + oozie_url = configurations[OOZIE_URL_KEY] + + # construct proper url for https + if https_port is not None: + parsed_url = urlparse(oozie_url) + oozie_url = oozie_url.replace(parsed_url.scheme, "https") + if parsed_url.port is None: + oozie_url.replace(parsed_url.hostname, ":".join([parsed_url.hostname, str(https_port)])) + else: + oozie_url = oozie_url.replace(str(parsed_url.port), str(https_port)) + + # https will not work with localhost address, we need put fqdn + if https_port is None: + oozie_url = oozie_url.replace(urlparse(oozie_url).hostname, host_name) + + (code, msg) = get_check_result(oozie_url, host_name, configurations, parameters, False) + + # sometimes real lifetime for ticket is less than we have set(5m20s aS of now) + # so i've added this double check with rekinit command to be sure thaT it's not problem with ticket lifetime + if is_security_enabled(configurations) and code == RESULT_CODE_CRITICAL: + (code, msg) = get_check_result(oozie_url, host_name, configurations, parameters, True) + + return (code, msg) + + +def get_check_result(oozie_url, host_name, configurations, parameters, only_kinit): + try: + command, env, user = get_check_command(oozie_url, host_name, configurations, parameters, only_kinit) + # execute the command + Execute(command, environment=env, user=user) + + return (RESULT_CODE_OK, ["Successful connection to {0}".format(oozie_url)]) + except KerberosPropertiesNotFound, ex: + return (RESULT_CODE_UNKNOWN, [str(ex)]) + except Exception, ex: + return (RESULT_CODE_CRITICAL, [str(ex)]) + +def is_security_enabled(configurations): + security_enabled = False + if SECURITY_ENABLED in configurations: + security_enabled = str(configurations[SECURITY_ENABLED]).upper() == 'TRUE' + + return security_enabled http://git-wip-us.apache.org/repos/asf/ambari/blob/712966db/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/files/oozieSmoke2.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/files/oozieSmoke2.sh b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/files/oozieSmoke2.sh new file mode 100644 index 0000000..60716ae --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/files/oozieSmoke2.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +export os_family=$1 +export oozie_lib_dir=$2 +export oozie_conf_dir=$3 +export oozie_bin_dir=$4 +export oozie_server_url=$5 +export oozie_examples_dir=$6 +export hadoop_conf_dir=$7 +export hadoop_bin_dir=$8 +export smoke_test_user=$9 +export job_name=${10} +export security_enabled=${11} +export smoke_user_keytab=${12} +export kinit_path_local=${13} +export smokeuser_principal=${14} + +function checkOozieJobStatus { + local job_id=$1 + local num_of_tries=$2 + #default num_of_tries to 10 if not present + num_of_tries=${num_of_tries:-10} + local i=0 + local rc=1 + local cmd="source ${oozie_conf_dir}/oozie-env.sh ; ${oozie_bin_dir}/oozie job -oozie ${OOZIE_SERVER} -info $job_id" + /var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd" + while [ $i -lt $num_of_tries ] ; do + cmd_output=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd"` + (IFS='';echo $cmd_output) + act_status=$(IFS='';echo $cmd_output | grep ^Status | cut -d':' -f2 | sed 's| ||g') + echo "workflow_status=$act_status" + if [ "RUNNING" == "$act_status" ]; then + #increment the counter and get the status again after waiting for 15 secs + sleep 15 + (( i++ )) + elif [ "SUCCEEDED" == "$act_status" ]; then + rc=0; + break; + else + rc=1 + break; + fi + done + return $rc +} + +export OOZIE_EXIT_CODE=0 +export OOZIE_SERVER=$oozie_server_url + +cd $oozie_examples_dir + +if [[ $security_enabled == "True" ]]; then + kinitcmd="${kinit_path_local} -kt ${smoke_user_keytab} ${smokeuser_principal}; " +else + kinitcmd="" +fi + +cmd="${kinitcmd}source ${oozie_conf_dir}/oozie-env.sh ; ${oozie_bin_dir}/oozie -Doozie.auth.token.cache=false job -oozie $OOZIE_SERVER -config $oozie_examples_dir/examples/apps/${job_name}/job.properties -run" +echo $cmd +job_info=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd" | grep "job:"` +job_id="`echo $job_info | cut -d':' -f2`" +checkOozieJobStatus "$job_id" 15 +OOZIE_EXIT_CODE="$?" +exit $OOZIE_EXIT_CODE http://git-wip-us.apache.org/repos/asf/ambari/blob/712966db/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/files/prepareOozieHdfsDirectories.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/files/prepareOozieHdfsDirectories.sh b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/files/prepareOozieHdfsDirectories.sh new file mode 100644 index 0000000..f2bee2d --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/files/prepareOozieHdfsDirectories.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +export oozie_conf_dir=$1 +export oozie_examples_dir=$2 +export hadoop_conf_dir=$3 +export JOBTRACKER=$4 +export NAMENODE=$5 +export QUEUE=$6 +export JOB_NAME=$7 + +cd $oozie_examples_dir + +/var/lib/ambari-agent/ambari-sudo.sh tar -zxf oozie-examples.tar.gz +/var/lib/ambari-agent/ambari-sudo.sh chmod -R o+rx examples + +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:8020|nameNode=$NAMENODE|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:9000|nameNode=$NAMENODE|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8021|jobTracker=$JOBTRACKER|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:9001|jobTracker=$JOBTRACKER|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8032|jobTracker=$JOBTRACKER|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|queueName=default|queueName=$QUEUE|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|oozie.wf.application.path=hdfs://localhost:9000|oozie.wf.application.path=$NAMENODE|g" examples/apps/$JOB_NAME/job.properties http://git-wip-us.apache.org/repos/asf/ambari/blob/712966db/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/files/wrap_ooziedb.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/files/wrap_ooziedb.sh b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/files/wrap_ooziedb.sh new file mode 100644 index 0000000..36576b5 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/files/wrap_ooziedb.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +OUT=`cd /var/tmp/oozie && /usr/lib/oozie/bin/ooziedb.sh "$@" 2>&1` +EC=$? +echo $OUT +GRVAR=`echo ${OUT} | grep -o "java.lang.Exception: DB schema exists"` +if [ ${EC} -ne 0 ] && [ -n "$GRVAR" ] +then + exit 0 +else + exit $EC +fi http://git-wip-us.apache.org/repos/asf/ambari/blob/712966db/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/check_oozie_server_status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/check_oozie_server_status.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/check_oozie_server_status.py new file mode 100644 index 0000000..7c69779 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/check_oozie_server_status.py @@ -0,0 +1,38 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst + + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def check_oozie_server_status(): + import status_params + from resource_management.libraries.functions.windows_service_utils import check_windows_service_status + + check_windows_service_status(status_params.oozie_server_win_service_name) + + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def check_oozie_server_status(): + import status_params + from resource_management.libraries.functions.check_process_status import check_process_status + + check_process_status(status_params.pid_file) + http://git-wip-us.apache.org/repos/asf/ambari/blob/712966db/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py new file mode 100644 index 0000000..aa5bc30 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py @@ -0,0 +1,549 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +# Python Imports +import os +import re + +# Resource Management Imports +from resource_management.core.resources.service import ServiceConfig +from resource_management.core.resources.system import Directory, Execute, File +from resource_management.core.source import DownloadSource +from resource_management.core.source import InlineTemplate +from resource_management.core.source import Template +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.oozie_prepare_war import prepare_war +from resource_management.libraries.functions.copy_tarball import get_current_version +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.security_commons import update_credential_provider_path +from resource_management.core.resources.packaging import Package +from resource_management.core.shell import as_user, as_sudo, call +from resource_management.core.exceptions import Fail + +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook +from ambari_commons.constants import SERVICE, UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING +from resource_management.libraries.functions.constants import Direction + +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst +from ambari_commons.inet_utils import download_file + +from resource_management.core import Logger + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def oozie(is_server=False, upgrade_type=None): + import params + + from status_params import oozie_server_win_service_name + + XmlConfig("oozie-site.xml", + conf_dir=params.oozie_conf_dir, + configurations=params.config['configurations']['oozie-site'], + owner=params.oozie_user, + mode='f', + configuration_attributes=params.config['configuration_attributes']['oozie-site'] + ) + + File(os.path.join(params.oozie_conf_dir, "oozie-env.cmd"), + owner=params.oozie_user, + content=InlineTemplate(params.oozie_env_cmd_template) + ) + + Directory(params.oozie_tmp_dir, + owner=params.oozie_user, + create_parents = True, + ) + + if is_server: + # Manually overriding service logon user & password set by the installation package + ServiceConfig(oozie_server_win_service_name, + action="change_user", + username = params.oozie_user, + password = Script.get_password(params.oozie_user)) + + download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), + os.path.join(params.oozie_root, "extra_libs", "sqljdbc4.jar") + ) + webapps_sqljdbc_path = os.path.join(params.oozie_home, "oozie-server", "webapps", "oozie", "WEB-INF", "lib", "sqljdbc4.jar") + if os.path.isfile(webapps_sqljdbc_path): + download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), + webapps_sqljdbc_path + ) + download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), + os.path.join(params.oozie_home, "share", "lib", "oozie", "sqljdbc4.jar") + ) + download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), + os.path.join(params.oozie_home, "temp", "WEB-INF", "lib", "sqljdbc4.jar") + ) + +# TODO: see if see can remove this +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def oozie(is_server=False, upgrade_type=None): + import params + + if is_server: + params.HdfsResource(params.oozie_hdfs_user_dir, + type="directory", + action="create_on_execute", + owner=params.oozie_user, + mode=params.oozie_hdfs_user_mode + ) + params.HdfsResource(None, action="execute") + Directory(params.conf_dir, + create_parents = True, + owner = params.oozie_user, + group = params.user_group + ) + + params.oozie_site = update_credential_provider_path(params.oozie_site, + 'oozie-site', + os.path.join(params.conf_dir, 'oozie-site.jceks'), + params.oozie_user, + params.user_group + ) + + XmlConfig("oozie-site.xml", + conf_dir = params.conf_dir, + configurations = params.oozie_site, + configuration_attributes=params.config['configuration_attributes']['oozie-site'], + owner = params.oozie_user, + group = params.user_group, + mode = 0664 + ) + File(format("{conf_dir}/oozie-env.sh"), + owner=params.oozie_user, + content=InlineTemplate(params.oozie_env_sh_template), + group=params.user_group, + ) + + # On some OS this folder could be not exists, so we will create it before pushing there files + Directory(params.limits_conf_dir, + create_parents=True, + owner='root', + group='root' + ) + + File(os.path.join(params.limits_conf_dir, 'oozie.conf'), + owner='root', + group='root', + mode=0644, + content=Template("oozie.conf.j2") + ) + + if (params.log4j_props != None): + File(format("{params.conf_dir}/oozie-log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.oozie_user, + content=InlineTemplate(params.log4j_props) + ) + elif (os.path.exists(format("{params.conf_dir}/oozie-log4j.properties"))): + File(format("{params.conf_dir}/oozie-log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.oozie_user + ) + + if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_ADMIN_USER, params.stack_version_formatted): + File(format("{params.conf_dir}/adminusers.txt"), + mode=0644, + group=params.user_group, + owner=params.oozie_user, + content=Template('adminusers.txt.j2', oozie_admin_users=params.oozie_admin_users) + ) + else: + File ( format("{params.conf_dir}/adminusers.txt"), + owner = params.oozie_user, + group = params.user_group + ) + + if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \ + params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \ + params.jdbc_driver_name == "org.postgresql.Driver" or \ + params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": + File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), + content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), + ) + pass + + oozie_ownership() + + if is_server: + oozie_server_specific(upgrade_type) + +def oozie_ownership(): + import params + + File ( format("{conf_dir}/hadoop-config.xml"), + owner = params.oozie_user, + group = params.user_group + ) + + File ( format("{conf_dir}/oozie-default.xml"), + owner = params.oozie_user, + group = params.user_group + ) + + Directory ( format("{conf_dir}/action-conf"), + owner = params.oozie_user, + group = params.user_group + ) + + File ( format("{conf_dir}/action-conf/hive.xml"), + owner = params.oozie_user, + group = params.user_group + ) + +def get_oozie_ext_zip_source_paths(upgrade_type, params): + """ + Get an ordered list of Oozie ext zip file paths from the source stack. + :param upgrade_type: Upgrade type will be None if not in the middle of a stack upgrade. + :param params: Expected to contain fields for ext_js_path, upgrade_direction, source_stack_name, and ext_js_file + :return: Source paths to use for Oozie extension zip file + """ + # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip + paths = [] + source_ext_js_path = params.ext_js_path + # Preferred location used by HDP and BigInsights 4.2.5 + if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE: + source_ext_js_path = "/usr/share/" + params.source_stack_name.upper() + "-oozie/" + params.ext_js_file + paths.append(source_ext_js_path) + + # Alternate location used by BigInsights 4.2.0 when migrating to another stack. + paths.append("/var/lib/oozie/" + params.ext_js_file) + + return paths + +def oozie_server_specific(upgrade_type): + import params + + no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user) + + File(params.pid_file, + action="delete", + not_if=no_op_test + ) + + oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir] + Directory( oozie_server_directories, + owner = params.oozie_user, + group = params.user_group, + mode = 0755, + create_parents = True, + cd_access="a", + ) + + Directory(params.oozie_libext_dir, + create_parents = True, + ) + + hashcode_file = format("{oozie_home}/.hashcode") + skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share") + + untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home) + + Execute( untar_sharelib, # time-expensive + not_if = format("{no_op_test} || {skip_recreate_sharelib}"), + sudo = True, + ) + + configure_cmds = [] + # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path + source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params) + + # Copy the first oozie ext-2.2.zip file that is found. + # This uses a list to handle the cases when migrating from some versions of BigInsights to HDP. + if source_ext_zip_paths is not None: + for source_ext_zip_path in source_ext_zip_paths: + if os.path.isfile(source_ext_zip_path): + configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir)) + configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) + + Execute(configure_cmds, + not_if=no_op_test, + sudo=True, + ) + break + + + Directory(params.oozie_webapps_conf_dir, + owner = params.oozie_user, + group = params.user_group, + recursive_ownership = True, + recursion_follow_links = True, + ) + + # download the database JAR + download_database_library_if_needed() + + #falcon el extension + if params.has_falcon_host: + Execute(format('{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'), + not_if = no_op_test) + + Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'), + not_if = no_op_test) + + if params.lzo_enabled and len(params.all_lzo_packages) > 0: + Package(params.all_lzo_packages, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, + retry_count=params.agent_stack_retry_count) + Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), + not_if = no_op_test, + ) + + prepare_war(params) + + File(hashcode_file, + mode = 0644, + ) + + if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS, params.stack_version_formatted): + # Create hive-site and tez-site configs for oozie + Directory(params.hive_conf_dir, + create_parents = True, + owner = params.oozie_user, + group = params.user_group + ) + if 'hive-site' in params.config['configurations']: + hive_site_config = update_credential_provider_path(params.config['configurations']['hive-site'], + 'hive-site', + os.path.join(params.hive_conf_dir, 'hive-site.jceks'), + params.oozie_user, + params.user_group + ) + XmlConfig("hive-site.xml", + conf_dir=params.hive_conf_dir, + configurations=hive_site_config, + configuration_attributes=params.config['configuration_attributes']['hive-site'], + owner=params.oozie_user, + group=params.user_group, + mode=0644 + ) + if 'tez-site' in params.config['configurations']: + XmlConfig( "tez-site.xml", + conf_dir = params.hive_conf_dir, + configurations = params.config['configurations']['tez-site'], + configuration_attributes=params.config['configuration_attributes']['tez-site'], + owner = params.oozie_user, + group = params.user_group, + mode = 0664 + ) + + # If Atlas is also installed, need to generate Atlas Hive hook (hive-atlas-application.properties file) in directory + # {stack_root}/{current_version}/atlas/hook/hive/ + # Because this is a .properties file instead of an xml file, it will not be read automatically by Oozie. + # However, should still save the file on this host so that can upload it to the Oozie Sharelib in DFS. + if has_atlas_in_cluster(): + atlas_hook_filepath = os.path.join(params.hive_conf_dir, params.atlas_hook_filename) + Logger.info("Has atlas in cluster, will save Atlas Hive hook into location %s" % str(atlas_hook_filepath)) + setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.oozie_user, params.user_group) + + Directory(params.oozie_server_dir, + owner = params.oozie_user, + group = params.user_group, + recursive_ownership = True, + ) + if params.security_enabled: + File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'), + owner=params.oozie_user, + group=params.user_group, + content=Template("zkmigrator_jaas.conf.j2") + ) + +def __parse_sharelib_from_output(output): + """ + Return the parent directory of the first path from the output of the "oozie admin -shareliblist command $comp" + Output will match pattern like: + + Potential errors + [Available ShareLib] + hive + hdfs://server:8020/user/oozie/share/lib/lib_20160811235630/hive/file1.jar + hdfs://server:8020/user/oozie/share/lib/lib_20160811235630/hive/file2.jar + """ + if output is not None: + pattern = re.compile(r"\[Available ShareLib\]\n\S*?\n(.*share.*)", re.IGNORECASE) + m = pattern.search(output) + if m and len(m.groups()) == 1: + jar_path = m.group(1) + # Remove leading/trailing spaces and get the containing directory + sharelib_dir = os.path.dirname(jar_path.strip()) + return sharelib_dir + return None + +def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=None): + """ + If the Atlas Hive Hook direcotry is present, Atlas is installed, and this is the first Oozie Server, + then copy the entire contents of that directory to the Oozie Sharelib in DFS, e.g., + /usr/$stack/$current_version/atlas/hook/hive/ -> hdfs:///user/oozie/share/lib/lib_$timetamp/hive + + :param upgrade_type: If in the middle of a stack upgrade, the type as UPGRADE_TYPE_ROLLING or UPGRADE_TYPE_NON_ROLLING + :param upgrade_direction: If in the middle of a stack upgrade, the direction as Direction.UPGRADE or Direction.DOWNGRADE. + """ + import params + + # Calculate the effective version since this code can also be called during EU/RU in the upgrade direction. + effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(params.version) + if not check_stack_feature(StackFeature.ATLAS_HOOK_SUPPORT, effective_version): + return + + # Important that oozie_server_hostnames is sorted by name so that this only runs on a single Oozie server. + if not (len(params.oozie_server_hostnames) > 0 and params.hostname == params.oozie_server_hostnames[0]): + Logger.debug("Will not attempt to copy Atlas Hive hook to DFS since this is not the first Oozie Server " + "sorted by hostname.") + return + + if not has_atlas_in_cluster(): + Logger.debug("Will not attempt to copy Atlas Hve hook to DFS since Atlas is not installed on the cluster.") + return + + if upgrade_type is not None and upgrade_direction == Direction.DOWNGRADE: + Logger.debug("Will not attempt to copy Atlas Hve hook to DFS since in the middle of Rolling/Express upgrade " + "and performing a Downgrade.") + return + + current_version = get_current_version() + atlas_hive_hook_dir = format("{stack_root}/{current_version}/atlas/hook/hive/") + if not os.path.exists(atlas_hive_hook_dir): + Logger.error(format("ERROR. Atlas is installed in cluster but this Oozie server doesn't " + "contain directory {atlas_hive_hook_dir}")) + return + + atlas_hive_hook_impl_dir = os.path.join(atlas_hive_hook_dir, "atlas-hive-plugin-impl") + + num_files = len([name for name in os.listdir(atlas_hive_hook_impl_dir) if os.path.exists(os.path.join(atlas_hive_hook_impl_dir, name))]) + Logger.info("Found %d files/directories inside Atlas Hive hook impl directory %s"% (num_files, atlas_hive_hook_impl_dir)) + + # This can return over 100 files, so take the first 5 lines after "Available ShareLib" + # Use -oozie http(s):localhost:{oozie_server_admin_port}/oozie as oozie-env does not export OOZIE_URL + command = format(r'source {conf_dir}/oozie-env.sh ; oozie admin -oozie {oozie_base_url} -shareliblist hive | grep "\[Available ShareLib\]" -A 5') + + try: + code, out = call(command, user=params.oozie_user, tries=10, try_sleep=5, logoutput=True) + if code == 0 and out is not None: + hive_sharelib_dir = __parse_sharelib_from_output(out) + + if hive_sharelib_dir is None: + raise Fail("Could not parse Hive sharelib from output.") + + Logger.info("Parsed Hive sharelib = %s and will attempt to copy/replace %d files to it from %s" % + (hive_sharelib_dir, num_files, atlas_hive_hook_impl_dir)) + + params.HdfsResource(hive_sharelib_dir, + type="directory", + action="create_on_execute", + source=atlas_hive_hook_impl_dir, + user=params.hdfs_user, + owner=params.oozie_user, + group=params.hdfs_user, + mode=0755, + recursive_chown=True, + recursive_chmod=True, + replace_existing_files=True + ) + + Logger.info("Copying Atlas Hive hook properties file to Oozie Sharelib in DFS.") + atlas_hook_filepath_source = os.path.join(params.hive_conf_dir, params.atlas_hook_filename) + atlas_hook_file_path_dest_in_dfs = os.path.join(hive_sharelib_dir, params.atlas_hook_filename) + params.HdfsResource(atlas_hook_file_path_dest_in_dfs, + type="file", + source=atlas_hook_filepath_source, + action="create_on_execute", + owner=params.oozie_user, + group=params.hdfs_user, + mode=0755, + replace_existing_files=True + ) + params.HdfsResource(None, action="execute") + + # Update the sharelib after making any changes + # Use -oozie http(s):localhost:{oozie_server_admin_port}/oozie as oozie-env does not export OOZIE_URL + command = format("source {conf_dir}/oozie-env.sh ; oozie admin -oozie {oozie_base_url} -sharelibupdate") + code, out = call(command, user=params.oozie_user, tries=5, try_sleep=5, logoutput=True) + if code == 0 and out is not None: + Logger.info("Successfully updated the Oozie ShareLib") + else: + raise Exception("Could not update the Oozie ShareLib after uploading the Atlas Hive hook directory to DFS. " + "Code: %s" % str(code)) + else: + raise Exception("Code is non-zero or output is empty. Code: %s" % str(code)) + except Fail, e: + Logger.error("Failed to get Hive sharelib directory in DFS. %s" % str(e)) + + +def download_database_library_if_needed(target_directory = None): + """ + Downloads the library to use when connecting to the Oozie database, if + necessary. The library will be downloaded to 'params.target' unless + otherwise specified. + :param target_directory: the location where the database library will be + downloaded to. + :return: + """ + import params + jdbc_drivers = ["com.mysql.jdbc.Driver", + "com.microsoft.sqlserver.jdbc.SQLServerDriver", + "oracle.jdbc.driver.OracleDriver","sap.jdbc4.sqlanywhere.IDriver"] + + # check to see if the JDBC driver name is in the list of ones that need to + # be downloaded + if params.jdbc_driver_name not in jdbc_drivers or not params.jdbc_driver_jar: + return + + if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar): + File(params.previous_jdbc_jar, action='delete') + + # if the target directory is not specified + if target_directory is None: + target_jar_with_directory = params.target + else: + # create the full path using the supplied target directory and the JDBC JAR + target_jar_with_directory = target_directory + os.path.sep + params.jdbc_driver_jar + + if not os.path.exists(target_jar_with_directory): + File(params.downloaded_custom_connector, + content = DownloadSource(params.driver_curl_source)) + + if params.sqla_db_used: + untar_sqla_type2_driver = ('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir) + + Execute(untar_sqla_type2_driver, sudo = True) + + Execute(format("yes | {sudo} cp {jars_path_in_archive} {oozie_libext_dir}")) + + Directory(params.jdbc_libs_dir, + create_parents = True) + + Execute(format("yes | {sudo} cp {libs_path_in_archive} {jdbc_libs_dir}")) + + Execute(format("{sudo} chown -R {oozie_user}:{user_group} {oozie_libext_dir}/*")) + + else: + Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target_jar_with_directory), + path=["/bin", "/usr/bin/"], + sudo = True) + + File(target_jar_with_directory, owner = params.oozie_user, + group = params.user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/712966db/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_client.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_client.py new file mode 100644 index 0000000..9bbca9f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_client.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from oozie import oozie +from oozie_service import oozie_service +from resource_management.core.exceptions import ClientComponentHasNoStatus + +class OozieClient(Script): + + def get_component_name(self): + return "oozie-client" + + def install(self, env): + self.install_packages(env) + self.configure(env) + + + def configure(self, env): + import params + env.set_params(params) + + oozie(is_server=False) + + def status(self, env): + raise ClientComponentHasNoStatus() + + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + # this function should not execute if the version can't be determined or + # the stack does not support rolling upgrade + if not (params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version)): + return + + Logger.info("Executing Oozie Client Stack Upgrade pre-restart") + conf_select.select(params.stack_name, "oozie", params.version) + stack_select.select("oozie-client", params.version) + + # We substitute some configs (oozie.authentication.kerberos.principal) before generation (see oozie.py and params.py). + # This function returns changed configs (it's used for config generation before config download) + def generate_configs_get_xml_file_content(self, filename, dictionary): + if dictionary == 'oozie-site': + import params + config = self.get_config() + return {'configurations': params.oozie_site, + 'configuration_attributes': config['configuration_attributes'][dictionary]} + else: + return super(OozieClient, self).generate_configs_get_xml_file_content(filename, dictionary) + +if __name__ == "__main__": + OozieClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/712966db/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_server.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_server.py new file mode 100644 index 0000000..b3a8643 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_server.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core import Logger +from resource_management.libraries.script import Script +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions import default +from resource_management.libraries.functions.constants import Direction +from resource_management.libraries.functions.security_commons import build_expectations +from resource_management.libraries.functions.security_commons import cached_kinit_executor +from resource_management.libraries.functions.security_commons import get_params_from_filesystem +from resource_management.libraries.functions.security_commons import validate_security_config_properties +from resource_management.libraries.functions.security_commons import FILE_TYPE_XML + +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING + +from oozie import oozie +from oozie_service import oozie_service +from oozie_server_upgrade import OozieUpgrade + +from check_oozie_server_status import check_oozie_server_status +from resource_management.core.resources.zkmigrator import ZkMigrator + +class OozieServer(Script): + + def get_component_name(self): + return "oozie-server" + + def install(self, env): + self.install_packages(env) + + def configure(self, env, upgrade_type=None): + import params + + # The configure command doesn't actually receive the upgrade_type from Script.py, so get it from the config dictionary + if upgrade_type is None: + upgrade_type = Script.get_upgrade_type(default("/commandParams/upgrade_type", "")) + + if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None: + Logger.info(format("Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}")) + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + # In order for the "<stack-root>/current/oozie-<client/server>" point to the new version of + # oozie, we need to create the symlinks both for server and client. + # This is required as both need to be pointing to new installed oozie version. + + # Sets the symlink : eg: <stack-root>/current/oozie-client -> <stack-root>/a.b.c.d-<version>/oozie + stack_select.select("oozie-client", params.version) + # Sets the symlink : eg: <stack-root>/current/oozie-server -> <stack-root>/a.b.c.d-<version>/oozie + stack_select.select("oozie-server", params.version) + + if params.version and check_stack_feature(StackFeature.CONFIG_VERSIONING, params.version): + conf_select.select(params.stack_name, "oozie", params.version) + + env.set_params(params) + oozie(is_server=True, upgrade_type=upgrade_type) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + + self.configure(env) + + # preparing the WAR file must run after configure since configure writes out + # oozie-env.sh which is needed to have the right environment directories setup! + if upgrade_type is not None: + OozieUpgrade.prepare_warfile() + + oozie_service(action='start', upgrade_type=upgrade_type) + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + oozie_service(action='stop', upgrade_type=upgrade_type) + + + def status(self, env): + import status_params + env.set_params(status_params) + check_oozie_server_status() + + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class OozieServerDefault(OozieServer): + + def pre_upgrade_restart(self, env, upgrade_type=None): + """ + Performs the tasks that should be done before an upgrade of oozie. This includes: + - backing up configurations + - running <stack-selector-tool> and <conf-selector-tool> + - restoring configurations + - preparing the libext directory + :param env: + :return: + """ + import params + env.set_params(params) + + # this function should not execute if the version can't be determined or + # the stack does not support rolling upgrade + if not (params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version)): + return + + Logger.info("Executing Oozie Server Stack Upgrade pre-restart") + + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + conf_select.select(params.stack_name, "oozie", params.version) + stack_select.select("oozie-server", params.version) + + OozieUpgrade.prepare_libext_directory(upgrade_type=upgrade_type) + + def disable_security(self, env): + import params + if not params.stack_supports_zk_security: + Logger.info("Stack doesn't support zookeeper security") + return + if not params.zk_connection_string: + Logger.info("No zookeeper connection string. Skipping reverting ACL") + return + zkmigrator = ZkMigrator(params.zk_connection_string, params.java_exec, params.java64_home, params.jaas_file, params.oozie_user) + zkmigrator.set_acls(params.zk_namespace if params.zk_namespace.startswith('/') else '/' + params.zk_namespace, 'world:anyone:crdwa') + + def get_log_folder(self): + import params + return params.oozie_log_dir + + def get_user(self): + import params + return params.oozie_user + + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class OozieServerWindows(OozieServer): + pass + +if __name__ == "__main__": + OozieServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/712966db/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_server_upgrade.py new file mode 100644 index 0000000..eb57c22 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_server_upgrade.py @@ -0,0 +1,247 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import glob +import os +import shutil + +from resource_management.core.logger import Logger +from resource_management.core.exceptions import Fail +from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import File +from resource_management.libraries.functions import Direction +from resource_management.libraries.functions import format +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.oozie_prepare_war import prepare_war +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature + +import oozie + +BACKUP_TEMP_DIR = "oozie-upgrade-backup" +BACKUP_CONF_ARCHIVE = "oozie-conf-backup.tar" + +class OozieUpgrade(Script): + + @staticmethod + def prepare_libext_directory(upgrade_type=None): + """ + Performs the following actions on libext: + - creates <stack-root>/current/oozie/libext and recursively + - set 777 permissions on it and its parents. + - downloads JDBC driver JAR if needed + - copies Falcon JAR for the Oozie WAR if needed + """ + import params + + # some stack versions don't need the lzo compression libraries + target_version_needs_compression_libraries = params.version and check_stack_feature(StackFeature.LZO, params.version) + + # ensure the directory exists + Directory(params.oozie_libext_dir, mode = 0777) + + # get all hadooplzo* JAR files + # <stack-selector-tool> set hadoop-client has not run yet, therefore we cannot use + # <stack-root>/current/hadoop-client ; we must use params.version directly + # however, this only works when upgrading beyond 2.2.0.0; don't do this + # for downgrade to 2.2.0.0 since hadoop-lzo will not be present + # This can also be called during a Downgrade. + # When a version is Installed, it is responsible for downloading the hadoop-lzo packages + # if lzo is enabled. + if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): + hadoop_lzo_pattern = 'hadoop-lzo*.jar' + hadoop_client_new_lib_dir = format("{stack_root}/{version}/hadoop/lib") + + files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) + if not files: + raise Fail("There are no files at {0} matching {1}".format( + hadoop_client_new_lib_dir, hadoop_lzo_pattern)) + + # copy files into libext + files_copied = False + for file in files: + if os.path.isfile(file): + Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_dir)) + shutil.copy2(file, params.oozie_libext_dir) + files_copied = True + + if not files_copied: + raise Fail("There are no files at {0} matching {1}".format( + hadoop_client_new_lib_dir, hadoop_lzo_pattern)) + + # something like <stack-root>/current/oozie-server/libext/ext-2.2.zip + oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, params.ext_js_file) + + # Copy ext ZIP to libext dir + # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path + source_ext_zip_paths = oozie.get_oozie_ext_zip_source_paths(upgrade_type, params) + + found_at_least_one_oozie_ext_file = False + + # Copy the first oozie ext-2.2.zip file that is found. + # This uses a list to handle the cases when migrating from some versions of BigInsights to HDP. + if source_ext_zip_paths is not None: + for source_ext_zip_path in source_ext_zip_paths: + if os.path.isfile(source_ext_zip_path): + found_at_least_one_oozie_ext_file = True + Logger.info("Copying {0} to {1}".format(source_ext_zip_path, params.oozie_libext_dir)) + Execute(("cp", source_ext_zip_path, params.oozie_libext_dir), sudo=True) + Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True) + File(oozie_ext_zip_target_path, + mode=0644 + ) + break + + if not found_at_least_one_oozie_ext_file: + raise Fail("Unable to find any Oozie source extension files from the following paths {0}".format(source_ext_zip_paths)) + + # Redownload jdbc driver to a new current location + oozie.download_database_library_if_needed() + + # get the upgrade version in the event that it's needed + upgrade_stack = stack_select._get_upgrade_stack() + if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None: + raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.") + + stack_version = upgrade_stack[1] + + # copy the Falcon JAR if needed; falcon has not upgraded yet, so we must + # use the versioned falcon directory + if params.has_falcon_host: + versioned_falcon_jar_directory = "{0}/{1}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(params.stack_root, stack_version) + Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir)) + + Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}')) + Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar')) + + + @staticmethod + def prepare_warfile(): + """ + Invokes the 'prepare-war' command in Oozie in order to create the WAR. + The prepare-war command uses the input WAR from ${OOZIE_HOME}/oozie.war and + outputs the prepared WAR to ${CATALINA_BASE}/webapps/oozie.war - because of this, + both of these environment variables must point to the upgraded oozie-server path and + not oozie-client since it was not yet updated. + + This method will also perform a kinit if necessary. + :return: + """ + import params + + # get the kerberos token if necessary to execute commands as oozie + if params.security_enabled: + oozie_principal_with_host = params.oozie_principal.replace("_HOST", params.hostname) + command = format("{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}") + Execute(command, user=params.oozie_user, logoutput=True) + + prepare_war(params) + + + def upgrade_oozie_database_and_sharelib(self, env): + """ + Performs the creation and upload of the sharelib and the upgrade of the + database. This method will also perform a kinit if necessary. + It is run before the upgrade of oozie begins exactly once as part of the + upgrade orchestration. + + Since this runs before the upgrade has occurred, it should not use any + "current" directories since they will still be pointing to the older + version of Oozie. Instead, it should use versioned directories to ensure + that the commands running are from the oozie version about to be upgraded to. + :return: + """ + import params + env.set_params(params) + + Logger.info("Will upgrade the Oozie database") + + # get the kerberos token if necessary to execute commands as oozie + if params.security_enabled: + oozie_principal_with_host = params.oozie_principal.replace("_HOST", params.hostname) + command = format("{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}") + Execute(command, user=params.oozie_user, logoutput=True) + + upgrade_stack = stack_select._get_upgrade_stack() + if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None: + raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.") + + stack_version = upgrade_stack[1] + + # upgrade oozie DB + Logger.info(format('Upgrading the Oozie database, using version {stack_version}')) + + # the database upgrade requires the db driver JAR, but since we have + # not yet run <stack-selector-tool> to upgrade the current points, we have to use + # the versioned libext directory as the location[[-vufdtffr, + versioned_libext_dir = "{0}/{1}/oozie/libext".format(params.stack_root, stack_version) + oozie.download_database_library_if_needed(target_directory=versioned_libext_dir) + + database_upgrade_command = "{0}/{1}/oozie/bin/ooziedb.sh upgrade -run".format(params.stack_root, stack_version) + Execute(database_upgrade_command, user=params.oozie_user, logoutput=True) + + # install new sharelib to HDFS + self.create_sharelib(env) + + + def create_sharelib(self, env): + """ + Performs the creation and upload of the sharelib. + This method will also perform a kinit if necessary. + It is run before the upgrade of oozie begins exactly once as part of the + upgrade orchestration. + + Since this runs before the upgrade has occurred, it should not use any + "current" directories since they will still be pointing to the older + version of Oozie. Instead, it should use versioned directories to ensure + that the commands running are from the oozie version about to be upgraded to. + :param env: + :return: + """ + import params + env.set_params(params) + + Logger.info('Creating a new sharelib and uploading it to HDFS...') + + # ensure the oozie directory exists for the sharelib + params.HdfsResource(format("{oozie_hdfs_user_dir}/share"), + action = "create_on_execute", + type = "directory", + owner = "oozie", + group = "hadoop", + mode = 0755, + recursive_chmod = True) + + params.HdfsResource(None, action = "execute") + + upgrade_stack = stack_select._get_upgrade_stack() + if upgrade_stack is None or upgrade_stack[1] is None: + raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.") + + stack_version = upgrade_stack[1] + + # install new sharelib to HDFS + sharelib_command = "{0}/{1}/oozie/bin/oozie-setup.sh sharelib create -fs {2}".format( + params.stack_root, stack_version, params.fs_root) + + Execute(sharelib_command, user=params.oozie_user, logoutput=True) + +if __name__ == "__main__": + OozieUpgrade().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/712966db/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_service.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_service.py new file mode 100644 index 0000000..7b15a63 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_service.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +# Python Imports +import os + +# Local Imports +from oozie import copy_atlas_hive_hook_to_dfs_share_lib + +# Resource Managemente Imports +from resource_management.core import shell, sudo +from resource_management import * +from resource_management.core.shell import as_user +from resource_management.core.logger import Logger +from resource_management.libraries.functions.show_logs import show_logs +from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst + +from resource_management.core import Logger + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def oozie_service(action='start', upgrade_type=None): + import params + + if action == 'start': + cmd = format("cmd /C \"cd /d {oozie_tmp_dir} && {oozie_home}\\bin\\ooziedb.cmd create -sqlfile oozie.sql -run\"") + Execute(cmd, user=params.oozie_user, ignore_failures=True) + Service(params.oozie_server_win_service_name, action="start") + elif action == 'stop': + Service(params.oozie_server_win_service_name, action="stop") + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def oozie_service(action = 'start', upgrade_type=None): + """ + Starts or stops the Oozie service + :param action: 'start' or 'stop' + :param upgrade_type: type of upgrade, either "rolling" or "non_rolling" + skipped since a variation of them was performed during the rolling upgrade + :return: + """ + import params + + environment={'OOZIE_CONFIG': params.conf_dir} + + if params.security_enabled: + if params.oozie_principal is None: + oozie_principal_with_host = 'missing_principal' + else: + oozie_principal_with_host = params.oozie_principal.replace("_HOST", params.hostname) + kinit_if_needed = format("{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host};") + else: + kinit_if_needed = "" + + no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user) + + if action == 'start': + start_cmd = format("cd {oozie_tmp_dir} && {oozie_home}/bin/oozie-start.sh") + path_to_jdbc = params.target + + if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \ + params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \ + params.jdbc_driver_name == "org.postgresql.Driver" or \ + params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": + + if not params.jdbc_driver_jar: + path_to_jdbc = format("{oozie_libext_dir}/") + \ + params.default_connectors_map[params.jdbc_driver_name] if params.jdbc_driver_name in params.default_connectors_map else None + if not os.path.isfile(path_to_jdbc): + path_to_jdbc = format("{oozie_libext_dir}/") + "*" + error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.jdbc_driver_name] + \ + " in oozie lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'" + Logger.error(error_message) + + db_connection_check_command = format("{java_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{oozie_jdbc_connection_url}' {oozie_metastore_user_name} {oozie_metastore_user_passwd!p} {jdbc_driver_name}") + else: + db_connection_check_command = None + + if upgrade_type is None: + if not os.path.isfile(path_to_jdbc) and params.jdbc_driver_name == "org.postgresql.Driver": + print format("ERROR: jdbc file {target} is unavailable. Please, follow next steps:\n" \ + "1) Download postgresql-9.0-801.jdbc4.jar.\n2) Create needed directory: mkdir -p {oozie_home}/libserver/\n" \ + "3) Copy postgresql-9.0-801.jdbc4.jar to newly created dir: cp /path/to/jdbc/postgresql-9.0-801.jdbc4.jar " \ + "{oozie_home}/libserver/\n4) Copy postgresql-9.0-801.jdbc4.jar to libext: cp " \ + "/path/to/jdbc/postgresql-9.0-801.jdbc4.jar {oozie_home}/libext/\n") + exit(1) + + if db_connection_check_command: + sudo.chmod(params.check_db_connection_jar, 0755) + Execute( db_connection_check_command, + tries=5, + try_sleep=10, + user=params.oozie_user, + ) + + Execute( format("cd {oozie_tmp_dir} && {oozie_home}/bin/ooziedb.sh create -sqlfile oozie.sql -run"), + user = params.oozie_user, not_if = no_op_test, + ignore_failures = True + ) + + if params.security_enabled: + Execute(kinit_if_needed, + user = params.oozie_user, + ) + + if params.sysprep_skip_copy_oozie_share_lib_to_hdfs: + Logger.info("Skipping creation of oozie sharelib as host is sys prepped") + # Copy current hive-site to hdfs:/user/oozie/share/lib/spark/ + params.HdfsResource(format("{hdfs_share_dir}/lib/spark/hive-site.xml"), + action="create_on_execute", + type = 'file', + mode=0444, + owner=params.oozie_user, + group=params.user_group, + source=format("{hive_conf_dir}/hive-site.xml"), + ) + params.HdfsResource(None, action="execute") + + hdfs_share_dir_exists = True # skip time-expensive hadoop fs -ls check + elif WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs): + # check with webhdfs is much faster than executing hadoop fs -ls. + util = WebHDFSUtil(params.hdfs_site, params.oozie_user, params.security_enabled) + list_status = util.run_command(params.hdfs_share_dir, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False) + hdfs_share_dir_exists = ('FileStatus' in list_status) + else: + # have to do time expensive hadoop fs -ls check. + hdfs_share_dir_exists = shell.call(format("{kinit_if_needed} hadoop --config {hadoop_conf_dir} dfs -ls {hdfs_share_dir} | awk 'BEGIN {{count=0;}} /share/ {{count++}} END {{if (count > 0) {{exit 0}} else {{exit 1}}}}'"), + user=params.oozie_user)[0] + + if not hdfs_share_dir_exists: + Execute( params.put_shared_lib_to_hdfs_cmd, + user = params.oozie_user, + path = params.execute_path + ) + params.HdfsResource(format("{oozie_hdfs_user_dir}/share"), + type="directory", + action="create_on_execute", + mode=0755, + recursive_chmod=True, + ) + params.HdfsResource(None, action="execute") + + + try: + # start oozie + Execute( start_cmd, environment=environment, user = params.oozie_user, + not_if = no_op_test ) + + copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type, params.upgrade_direction) + except: + show_logs(params.oozie_log_dir, params.oozie_user) + raise + + elif action == 'stop': + Directory(params.oozie_tmp_dir, + owner=params.oozie_user, + create_parents = True, + ) + + stop_cmd = format("cd {oozie_tmp_dir} && {oozie_home}/bin/oozied.sh stop 60 -force") + + try: + # stop oozie + Execute(stop_cmd, environment=environment, only_if = no_op_test, + user = params.oozie_user) + except: + show_logs(params.oozie_log_dir, params.oozie_user) + raise + + File(params.pid_file, action = "delete") http://git-wip-us.apache.org/repos/asf/ambari/blob/712966db/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params.py new file mode 100644 index 0000000..f39d632 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from ambari_commons import OSCheck +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.expect import expect +from resource_management.libraries.functions.copy_tarball import get_sysprep_skip_copy_tarballs_hdfs + +if OSCheck.is_windows_family(): + from params_windows import * +else: + from params_linux import * + +java_home = config['hostLevelParams']['java_home'] +java_version = expect("/hostLevelParams/java_version", int) + + +host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) + +# By default, copy the tarballs to HDFS. If the cluster is sysprepped, then set based on the config. +sysprep_skip_copy_oozie_share_lib_to_hdfs = False +if host_sys_prepped: + sysprep_skip_copy_oozie_share_lib_to_hdfs = default("/configurations/cluster-env/sysprep_skip_copy_oozie_share_lib_to_hdfs", False)