Repository: ambari Updated Branches: refs/heads/branch-2.5 6bfc894e7 -> 6870b5584
http://git-wip-us.apache.org/repos/asf/ambari/blob/6870b558/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py new file mode 100644 index 0000000..0e6179b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management import * +from ambari_commons.constants import AMBARI_SUDO_BINARY +from resource_management.libraries.functions import format +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import get_port_from_url +from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages +from resource_management.libraries.functions.expect import expect +from resource_management.libraries.functions.get_architecture import get_architecture +from resource_management.libraries.functions.stack_features import get_stack_feature_version +from resource_management.libraries.functions.stack_tools import get_stack_name +from resource_management.libraries.functions.version import get_major_version + +from resource_management.core.utils import PasswordString +from ambari_commons.credential_store_helper import get_password_from_credential_store +from urlparse import urlparse + +import status_params +import os + +# server configurations +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() +sudo = AMBARI_SUDO_BINARY + +architecture = get_architecture() + + +# Needed since this writes out the Atlas Hive Hook config file. +cluster_name = config['clusterName'] + +hostname = config["hostname"] + +# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +version = default("/commandParams/version", None) +stack_name = status_params.stack_name +stack_name_uppercase = stack_name.upper() +upgrade_direction = default("/commandParams/upgrade_direction", None) +agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability'] +agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) + +stack_root = status_params.stack_root + +# The source stack will be present during a cross-stack upgrade. +# E.g., BigInsights-4.2.5 or HDP-2.6 +source_stack = default("/commandParams/source_stack", None) +# This variable name is important, do not change +source_stack_name = get_stack_name(source_stack) + +stack_version_unformatted = status_params.stack_version_unformatted +stack_version_formatted = status_params.stack_version_formatted +major_stack_version = get_major_version(stack_version_formatted) +version_for_stack_feature_checks = get_stack_feature_version(config) + +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hadoop_lib_home = stack_select.get_hadoop_dir("lib") + +#hadoop params +if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE,stack_version_formatted): + stack_version = None + upgrade_stack = stack_select._get_upgrade_stack() + if upgrade_stack is not None and len(upgrade_stack) == 2 and upgrade_stack[1] is not None: + stack_version = upgrade_stack[1] + + # oozie-server or oozie-client, depending on role + oozie_root = status_params.component_directory + + # using the correct oozie root dir, format the correct location + oozie_lib_dir = format("{stack_root}/current/{oozie_root}") + oozie_setup_sh = format("{stack_root}/current/{oozie_root}/bin/oozie-setup.sh") + oozie_webapps_dir = format("{stack_root}/current/{oozie_root}/oozie-server/webapps") + oozie_webapps_conf_dir = format("{stack_root}/current/{oozie_root}/oozie-server/conf") + oozie_libext_dir = format("{stack_root}/current/{oozie_root}/libext") + oozie_server_dir = format("{stack_root}/current/{oozie_root}/oozie-server") + oozie_shared_lib = format("{stack_root}/current/{oozie_root}/share") + oozie_home = format("{stack_root}/current/{oozie_root}") + oozie_bin_dir = format("{stack_root}/current/{oozie_root}/bin") + oozie_examples_regex = format("{stack_root}/current/{oozie_root}/doc") + + # set the falcon home for copying JARs; if in an upgrade, then use the version of falcon that + # matches the version of oozie + falcon_home = format("{stack_root}/current/falcon-client") + if stack_version is not None: + falcon_home = '{0}/{1}/falcon'.format(stack_root, stack_version) + + conf_dir = format("{stack_root}/current/{oozie_root}/conf") + hive_conf_dir = format("{conf_dir}/action-conf/hive") + +else: + oozie_lib_dir = "/var/lib/oozie" + oozie_setup_sh = "/usr/lib/oozie/bin/oozie-setup.sh" + oozie_webapps_dir = "/var/lib/oozie/oozie-server/webapps/" + oozie_webapps_conf_dir = "/var/lib/oozie/oozie-server/conf" + oozie_libext_dir = "/usr/lib/oozie/libext" + oozie_server_dir = "/var/lib/oozie/oozie-server" + oozie_shared_lib = "/usr/lib/oozie/share" + oozie_home = "/usr/lib/oozie" + oozie_bin_dir = "/usr/bin" + falcon_home = '/usr/lib/falcon' + conf_dir = "/etc/oozie/conf" + hive_conf_dir = "/etc/oozie/conf/action-conf/hive" + oozie_examples_regex = "/usr/share/doc/oozie-*" + +execute_path = oozie_bin_dir + os.pathsep + hadoop_bin_dir + +oozie_user = config['configurations']['oozie-env']['oozie_user'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +smoke_hdfs_user_mode = 0770 +service_check_queue_name = default('/configurations/yarn-env/service_check.queue.name', 'default') + +# This config actually contains {oozie_user} +oozie_admin_users = format(config['configurations']['oozie-env']['oozie_admin_users']) + +user_group = config['configurations']['cluster-env']['user_group'] +jdk_location = config['hostLevelParams']['jdk_location'] +check_db_connection_jar_name = "DBConnectionVerification.jar" +check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}") +oozie_tmp_dir = default("configurations/oozie-env/oozie_tmp_dir", "/var/tmp/oozie") +oozie_hdfs_user_dir = format("/user/{oozie_user}") +oozie_pid_dir = status_params.oozie_pid_dir +pid_file = status_params.pid_file +hadoop_jar_location = "/usr/lib/hadoop/" +java_share_dir = "/usr/share/java" +java64_home = config['hostLevelParams']['java_home'] +java_exec = format("{java64_home}/bin/java") + +# This variable name is important, do not change +ext_js_file = "ext-2.2.zip" + +# During a cross-stack migration, the source location will be different +# This variable name is important, do not change +ext_js_path = format("/usr/share/{stack_name_uppercase}-oozie/{ext_js_file}") + +security_enabled = config['configurations']['cluster-env']['security_enabled'] +oozie_heapsize = config['configurations']['oozie-env']['oozie_heapsize'] +oozie_permsize = config['configurations']['oozie-env']['oozie_permsize'] + +limits_conf_dir = "/etc/security/limits.d" + +oozie_user_nofile_limit = config['configurations']['oozie-env']['oozie_user_nofile_limit'] +oozie_user_nproc_limit = config['configurations']['oozie-env']['oozie_user_nproc_limit'] + +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +oozie_service_keytab = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.keytab.file'] +oozie_principal = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.kerberos.principal'] +http_principal = config['configurations']['oozie-site']['oozie.authentication.kerberos.principal'] +oozie_site = config['configurations']['oozie-site'] +# Need this for yarn.nodemanager.recovery.dir in yarn-site +yarn_log_dir_prefix = config['configurations']['yarn-env']['yarn_log_dir_prefix'] +yarn_resourcemanager_address = config['configurations']['yarn-site']['yarn.resourcemanager.address'] +zk_namespace = default('/configurations/oozie-site/oozie.zookeeper.namespace', 'oozie') +zk_connection_string = default('/configurations/oozie-site/oozie.zookeeper.connection.string', None) +jaas_file = os.path.join(conf_dir, 'zkmigrator_jaas.conf') +stack_supports_zk_security = check_stack_feature(StackFeature.SECURE_ZOOKEEPER, version_for_stack_feature_checks) + +credential_store_enabled = False +if 'credentialStoreEnabled' in config: + credential_store_enabled = config['credentialStoreEnabled'] + +if security_enabled: + oozie_site = dict(config['configurations']['oozie-site']) + oozie_principal_with_host = oozie_principal.replace('_HOST', hostname) + + # If a user-supplied oozie.ha.authentication.kerberos.principal property exists in oozie-site, + # use it to replace the existing oozie.authentication.kerberos.principal value. This is to ensure + # that any special principal name needed for HA is used rather than the Ambari-generated value + if "oozie.ha.authentication.kerberos.principal" in oozie_site: + oozie_site['oozie.authentication.kerberos.principal'] = oozie_site['oozie.ha.authentication.kerberos.principal'] + http_principal = oozie_site['oozie.authentication.kerberos.principal'] + + # If a user-supplied oozie.ha.authentication.kerberos.keytab property exists in oozie-site, + # use it to replace the existing oozie.authentication.kerberos.keytab value. This is to ensure + # that any special keytab file needed for HA is used rather than the Ambari-generated value + if "oozie.ha.authentication.kerberos.keytab" in oozie_site: + oozie_site['oozie.authentication.kerberos.keytab'] = oozie_site['oozie.ha.authentication.kerberos.keytab'] + + if stack_version_formatted and check_stack_feature(StackFeature.OOZIE_HOST_KERBEROS, stack_version_formatted): + #older versions of oozie have problems when using _HOST in principal + oozie_site['oozie.service.HadoopAccessorService.kerberos.principal'] = oozie_principal_with_host + oozie_site['oozie.authentication.kerberos.principal'] = http_principal.replace('_HOST', hostname) + +smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +oozie_keytab = default("/configurations/oozie-env/oozie_keytab", oozie_service_keytab) +oozie_env_sh_template = config['configurations']['oozie-env']['content'] + +oracle_driver_jar_name = "ojdbc6.jar" + +oozie_metastore_user_name = config['configurations']['oozie-site']['oozie.service.JPAService.jdbc.username'] + +if credential_store_enabled: + if 'hadoop.security.credential.provider.path' in config['configurations']['oozie-site']: + cs_lib_path = config['configurations']['oozie-site']['credentialStoreClassPath'] + java_home = config['hostLevelParams']['java_home'] + alias = 'oozie.service.JPAService.jdbc.password' + provider_path = config['configurations']['oozie-site']['hadoop.security.credential.provider.path'] + oozie_metastore_user_passwd = PasswordString(get_password_from_credential_store(alias, provider_path, cs_lib_path, java_home, jdk_location)) + else: + raise Exception("hadoop.security.credential.provider.path property should be set") +else: + oozie_metastore_user_passwd = default("/configurations/oozie-site/oozie.service.JPAService.jdbc.password","") + +oozie_jdbc_connection_url = default("/configurations/oozie-site/oozie.service.JPAService.jdbc.url", "") +oozie_log_dir = config['configurations']['oozie-env']['oozie_log_dir'] +oozie_data_dir = config['configurations']['oozie-env']['oozie_data_dir'] +oozie_server_port = get_port_from_url(config['configurations']['oozie-site']['oozie.base.url']) +oozie_server_admin_port = config['configurations']['oozie-env']['oozie_admin_port'] +if 'export OOZIE_HTTPS_PORT' in oozie_env_sh_template or 'oozie.https.port' in config['configurations']['oozie-site'] or 'oozie.https.keystore.file' in config['configurations']['oozie-site'] or 'oozie.https.keystore.pass' in config['configurations']['oozie-site']: + oozie_secure = '-secure' +else: + oozie_secure = '' + +https_port = None +# try to get https port form oozie-env content +for line in oozie_env_sh_template.splitlines(): + result = re.match(r"export\s+OOZIE_HTTPS_PORT=(\d+)", line) + if result is not None: + https_port = result.group(1) +# or from oozie-site.xml +if https_port is None and 'oozie.https.port' in config['configurations']['oozie-site']: + https_port = config['configurations']['oozie-site']['oozie.https.port'] + +oozie_base_url = config['configurations']['oozie-site']['oozie.base.url'] + +service_check_job_name = default("/configurations/oozie-env/service_check_job_name", "no-op") + +# construct proper url for https +if https_port is not None: + parsed_url = urlparse(oozie_base_url) + oozie_base_url = oozie_base_url.replace(parsed_url.scheme, "https") + if parsed_url.port is None: + oozie_base_url.replace(parsed_url.hostname, ":".join([parsed_url.hostname, str(https_port)])) + else: + oozie_base_url = oozie_base_url.replace(str(parsed_url.port), str(https_port)) + +oozie_setup_sh_current = oozie_setup_sh + +hdfs_site = config['configurations']['hdfs-site'] +fs_root = config['configurations']['core-site']['fs.defaultFS'] + +if stack_version_formatted and check_stack_feature(StackFeature.OOZIE_SETUP_SHARED_LIB, stack_version_formatted): + put_shared_lib_to_hdfs_cmd = format("{oozie_setup_sh} sharelib create -fs {fs_root} -locallib {oozie_shared_lib}") + # for older +else: + put_shared_lib_to_hdfs_cmd = format("hadoop --config {hadoop_conf_dir} dfs -put {oozie_shared_lib} {oozie_hdfs_user_dir}") + +default_connectors_map = { "com.microsoft.sqlserver.jdbc.SQLServerDriver":"sqljdbc4.jar", + "com.mysql.jdbc.Driver":"mysql-connector-java.jar", + "org.postgresql.Driver":"postgresql-jdbc.jar", + "oracle.jdbc.driver.OracleDriver":"ojdbc.jar", + "sap.jdbc4.sqlanywhere.IDriver":"sajdbc4.jar"} + +jdbc_driver_name = default("/configurations/oozie-site/oozie.service.JPAService.jdbc.driver", "") +# NOT SURE THAT IT'S A GOOD IDEA TO USE PATH TO CLASS IN DRIVER, MAYBE IT WILL BE BETTER TO USE DB TYPE. +# BECAUSE PATH TO CLASSES COULD BE CHANGED +sqla_db_used = False +previous_jdbc_jar_name = None +if jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver": + jdbc_driver_jar = default("/hostLevelParams/custom_mssql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) +elif jdbc_driver_name == "com.mysql.jdbc.Driver": + jdbc_driver_jar = default("/hostLevelParams/custom_mysql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) +elif jdbc_driver_name == "org.postgresql.Driver": + jdbc_driver_jar = format("{oozie_home}/libserver/postgresql-9.0-801.jdbc4.jar") #oozie using it's own postgres jdbc + previous_jdbc_jar_name = None +elif jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": + jdbc_driver_jar = default("/hostLevelParams/custom_oracle_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) +elif jdbc_driver_name == "sap.jdbc4.sqlanywhere.IDriver": + jdbc_driver_jar = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) + sqla_db_used = True +else: + jdbc_driver_jar = "" + jdbc_symlink_name = "" + previous_jdbc_jar_name = None + +default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) +driver_curl_source = format("{jdk_location}/{jdbc_driver_jar}") +downloaded_custom_connector = format("{tmp_dir}/{jdbc_driver_jar}") +if jdbc_driver_name == "org.postgresql.Driver": + target = jdbc_driver_jar + previous_jdbc_jar = None +else: + target = format("{oozie_libext_dir}/{jdbc_driver_jar}") + previous_jdbc_jar = format("{oozie_libext_dir}/{previous_jdbc_jar_name}") + +#constants for type2 jdbc +jdbc_libs_dir = format("{oozie_libext_dir}/native/lib64") +lib_dir_available = os.path.exists(jdbc_libs_dir) + +if sqla_db_used: + jars_path_in_archive = format("{tmp_dir}/sqla-client-jdbc/java/*") + libs_path_in_archive = format("{tmp_dir}/sqla-client-jdbc/native/lib64/*") + downloaded_custom_connector = format("{tmp_dir}/{jdbc_driver_jar}") + +hdfs_share_dir = format("{oozie_hdfs_user_dir}/share") +ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] +falcon_host = default("/clusterHostInfo/falcon_server_hosts", []) +has_falcon_host = not len(falcon_host) == 0 + +oozie_server_hostnames = default("/clusterHostInfo/oozie_server", []) +oozie_server_hostnames = sorted(oozie_server_hostnames) + +oozie_log_maxhistory = default('configurations/oozie-log4j/oozie_log_maxhistory',720) + +#oozie-log4j.properties +if (('oozie-log4j' in config['configurations']) and ('content' in config['configurations']['oozie-log4j'])): + log4j_props = config['configurations']['oozie-log4j']['content'] +else: + log4j_props = None + +oozie_hdfs_user_mode = 0775 +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] + +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] + +dfs_type = default("/commandParams/dfs_type", "") + + +######################################################## +############# Atlas related params ##################### +######################################################## +#region Atlas Hooks needed by Hive on Oozie +hive_atlas_application_properties = default('/configurations/hive-atlas-application.properties', {}) + +if has_atlas_in_cluster(): + atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') +#endregion + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs, + immutable_paths = get_not_managed_resources(), + dfs_type = dfs_type +) + +is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'] + +# The logic for LZO also exists in HDFS' params.py +io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) +lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() + +all_lzo_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native"] http://git-wip-us.apache.org/repos/asf/ambari/blob/6870b558/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_windows.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_windows.py new file mode 100644 index 0000000..1f939d4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_windows.py @@ -0,0 +1,34 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script.script import Script +import os +from status_params import * + +config = Script.get_config() + +hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"] +stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], "..")) +oozie_root = os.environ['OOZIE_ROOT'] +oozie_home = os.environ['OOZIE_HOME'] +oozie_conf_dir = os.path.join(oozie_home,'conf') +oozie_user = hadoop_user +oozie_tmp_dir = "c:\\hadoop\\temp\\oozie" + +oozie_env_cmd_template = config['configurations']['oozie-env']['content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/6870b558/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/service_check.py new file mode 100644 index 0000000..03030ca --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/service_check.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os +import glob + +from resource_management.core.resources.system import Execute +from resource_management.core.resources import File +from resource_management.core.source import StaticFile +from resource_management.core.system import System +from resource_management.libraries.functions import format +from resource_management.libraries.script import Script +from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons import OSConst + +from resource_management.core.logger import Logger + +class OozieServiceCheck(Script): + pass + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class OozieServiceCheckDefault(OozieServiceCheck): + + def service_check(self, env): + import params + env.set_params(params) + + # on HDP1 this file is different + prepare_hdfs_file_name = 'prepareOozieHdfsDirectories.sh' + smoke_test_file_name = 'oozieSmoke2.sh' + + OozieServiceCheckDefault.oozie_smoke_shell_file(smoke_test_file_name, prepare_hdfs_file_name) + + @staticmethod + def oozie_smoke_shell_file(file_name, prepare_hdfs_file_name): + import params + + File(format("{tmp_dir}/{file_name}"), + content=StaticFile(file_name), + mode=0755 + ) + File(format("{tmp_dir}/{prepare_hdfs_file_name}"), + content=StaticFile(prepare_hdfs_file_name), + mode=0755 + ) + + os_family = System.get_instance().os_family + oozie_examples_dir = glob.glob(params.oozie_examples_regex)[0] + + Execute((format("{tmp_dir}/{prepare_hdfs_file_name}"), params.conf_dir, oozie_examples_dir, params.hadoop_conf_dir, params.yarn_resourcemanager_address, params.fs_root, params.service_check_queue_name, params.service_check_job_name), + tries=3, + try_sleep=5, + logoutput=True + ) + + params.HdfsResource(format("/user/{smokeuser}"), + type="directory", + action="create_on_execute", + owner=params.smokeuser, + mode=params.smoke_hdfs_user_mode, + ) + + examples_dir = format('/user/{smokeuser}/examples') + params.HdfsResource(examples_dir, + action = "delete_on_execute", + type = "directory" + ) + params.HdfsResource(examples_dir, + action = "create_on_execute", + type = "directory", + source = format("{oozie_examples_dir}/examples"), + owner = params.smokeuser, + group = params.user_group + ) + + input_data_dir = format('/user/{smokeuser}/input-data') + params.HdfsResource(input_data_dir, + action = "delete_on_execute", + type = "directory" + ) + params.HdfsResource(input_data_dir, + action = "create_on_execute", + type = "directory", + source = format("{oozie_examples_dir}/examples/input-data"), + owner = params.smokeuser, + group = params.user_group + ) + params.HdfsResource(None, action="execute") + + if params.security_enabled: + sh_cmd = format( + "{tmp_dir}/{file_name} {os_family} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {oozie_base_url} {oozie_examples_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {service_check_job_name} {security_enabled} {smokeuser_keytab} {kinit_path_local} {smokeuser_principal}") + else: + sh_cmd = format( + "{tmp_dir}/{file_name} {os_family} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {oozie_base_url} {oozie_examples_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {service_check_job_name} {security_enabled}") + + Execute(sh_cmd, + path=params.execute_path, + tries=3, + try_sleep=5, + logoutput=True + ) + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class OozieServiceCheckWindows(OozieServiceCheck): + + def service_check(self, env): + import params + + env.set_params(params) + smoke_cmd = os.path.join(params.stack_root, "Run-SmokeTests.cmd") + service = "OOZIE" + Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True) + +if __name__ == "__main__": + OozieServiceCheck().execute() + http://git-wip-us.apache.org/repos/asf/ambari/blob/6870b558/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/status_params.py new file mode 100644 index 0000000..ce990cf --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/status_params.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from ambari_commons.os_check import OSCheck +from resource_management.libraries.functions import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.version import format_stack_version + +# a map of the Ambari role to the component name +# for use with <stack-root>/current/<component> +SERVER_ROLE_DIRECTORY_MAP = { + 'OOZIE_SERVER' : 'oozie-server', + 'OOZIE_CLIENT' : 'oozie-client', + 'OOZIE_SERVICE_CHECK' : 'oozie-client', + 'ru_execute_tasks' : 'oozie-server' +} + +component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "OOZIE_CLIENT") + +config = Script.get_config() +stack_root = Script.get_stack_root() + +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) + +if OSCheck.is_windows_family(): + # windows service mapping + oozie_server_win_service_name = "oozieservice" +else: + oozie_pid_dir = config['configurations']['oozie-env']['oozie_pid_dir'] + pid_file = format("{oozie_pid_dir}/oozie.pid") + + security_enabled = config['configurations']['cluster-env']['security_enabled'] + kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) + + conf_dir = "/etc/oozie/conf" + if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): + conf_dir = format("{stack_root}/current/{component_directory}/conf") + + tmp_dir = Script.get_tmp_dir() + oozie_user = config['configurations']['oozie-env']['oozie_user'] + hostname = config["hostname"] + +stack_name = default("/hostLevelParams/stack_name", None) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6870b558/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/adminusers.txt.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/adminusers.txt.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/adminusers.txt.j2 new file mode 100644 index 0000000..2a0f7b2 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/adminusers.txt.j2 @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Users should be set using following rules: +# +# One user name per line +# Empty lines and lines starting with '#' are ignored + +{% if oozie_admin_users %} +{% for oozie_admin_user in oozie_admin_users.split(',') %} +{{oozie_admin_user|trim}} +{% endfor %} +{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/6870b558/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/oozie-log4j.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/oozie-log4j.properties.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/oozie-log4j.properties.j2 new file mode 100644 index 0000000..e39428f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/oozie-log4j.properties.j2 @@ -0,0 +1,93 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# + +# If the Java System property 'oozie.log.dir' is not defined at Oozie start up time +# XLogService sets its value to '${oozie.home}/logs' + +log4j.appender.oozie=org.apache.log4j.DailyRollingFileAppender +log4j.appender.oozie.File=${oozie.log.dir}/oozie.log +log4j.appender.oozie.Append=true +log4j.appender.oozie.layout=org.apache.log4j.PatternLayout +log4j.appender.oozie.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n +log4j.appender.oozie.RollingPolicy.FileNamePattern=${log4j.appender.oozie.File}-%d{yyyy-MM-dd} +log4j.appender.oozie.DatePattern='.'yyyy-MM-dd + +log4j.appender.oozieops=org.apache.log4j.DailyRollingFileAppender +log4j.appender.oozieops.DatePattern='.'yyyy-MM-dd +log4j.appender.oozieops.File=${oozie.log.dir}/oozie-ops.log +log4j.appender.oozieops.Append=true +log4j.appender.oozieops.layout=org.apache.log4j.PatternLayout +log4j.appender.oozieops.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + +log4j.appender.oozieinstrumentation=org.apache.log4j.DailyRollingFileAppender +log4j.appender.oozieinstrumentation.DatePattern='.'yyyy-MM-dd +log4j.appender.oozieinstrumentation.File=${oozie.log.dir}/oozie-instrumentation.log +log4j.appender.oozieinstrumentation.Append=true +log4j.appender.oozieinstrumentation.layout=org.apache.log4j.PatternLayout +log4j.appender.oozieinstrumentation.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + +log4j.appender.oozieaudit=org.apache.log4j.DailyRollingFileAppender +log4j.appender.oozieaudit.DatePattern='.'yyyy-MM-dd +log4j.appender.oozieaudit.File=${oozie.log.dir}/oozie-audit.log +log4j.appender.oozieaudit.Append=true +log4j.appender.oozieaudit.layout=org.apache.log4j.PatternLayout +log4j.appender.oozieaudit.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + +log4j.appender.openjpa=org.apache.log4j.DailyRollingFileAppender +log4j.appender.openjpa.DatePattern='.'yyyy-MM-dd +log4j.appender.openjpa.File=${oozie.log.dir}/oozie-jpa.log +log4j.appender.openjpa.Append=true +log4j.appender.openjpa.layout=org.apache.log4j.PatternLayout +log4j.appender.openjpa.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + +log4j.logger.openjpa=INFO, openjpa +log4j.logger.oozieops=INFO, oozieops +log4j.logger.oozieinstrumentation=ALL, oozieinstrumentation +log4j.logger.oozieaudit=ALL, oozieaudit +log4j.logger.org.apache.oozie=INFO, oozie +log4j.logger.org.apache.hadoop=WARN, oozie +log4j.logger.org.mortbay=WARN, oozie +log4j.logger.org.hsqldb=WARN, oozie +log4j.logger.org.apache.hadoop.security.authentication.server=INFO, oozie http://git-wip-us.apache.org/repos/asf/ambari/blob/6870b558/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/oozie.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/oozie.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/oozie.conf.j2 new file mode 100644 index 0000000..1f99e49 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/oozie.conf.j2 @@ -0,0 +1,35 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{oozie_user}} - nofile {{oozie_user_nofile_limit}} +{{oozie_user}} - nproc {{oozie_user_nproc_limit}} http://git-wip-us.apache.org/repos/asf/ambari/blob/6870b558/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/zkmigrator_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/zkmigrator_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/zkmigrator_jaas.conf.j2 new file mode 100644 index 0000000..fbc0ce5 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/templates/zkmigrator_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + useTicketCache=false + keyTab="{{oozie_keytab}}" + principal="{{oozie_principal_with_host}}"; +}; \ No newline at end of file
