AMBARI-19835 HDP 3.0 support for Slider with configs, kerberos, widgets, metrics, quicklinks, and themes (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/528ac5c3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/528ac5c3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/528ac5c3 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 528ac5c33d0549f18c5d4518478d274ce1551a15 Parents: 0d2f2c2 Author: Dmytro Sen <d...@apache.org> Authored: Fri Mar 10 18:46:49 2017 +0200 Committer: Dmytro Sen <d...@apache.org> Committed: Fri Mar 10 18:46:49 2017 +0200 ---------------------------------------------------------------------- .../0.91.0.3.0/configuration/slider-client.xml | 56 ++++++++++ .../0.91.0.3.0/configuration/slider-env.xml | 46 ++++++++ .../0.91.0.3.0/configuration/slider-log4j.xml | 92 ++++++++++++++++ .../SLIDER/0.91.0.3.0/kerberos.json | 17 +++ .../SLIDER/0.91.0.3.0/metainfo.xml | 109 +++++++++++++++++++ .../package/files/hbaseSmokeVerify.sh | 34 ++++++ .../0.91.0.3.0/package/scripts/__init__.py | 19 ++++ .../SLIDER/0.91.0.3.0/package/scripts/params.py | 54 +++++++++ .../0.91.0.3.0/package/scripts/params_linux.py | 80 ++++++++++++++ .../package/scripts/params_windows.py | 45 ++++++++ .../0.91.0.3.0/package/scripts/service_check.py | 63 +++++++++++ .../SLIDER/0.91.0.3.0/package/scripts/slider.py | 96 ++++++++++++++++ .../0.91.0.3.0/package/scripts/slider_client.py | 71 ++++++++++++ .../package/templates/storm-slider-env.sh.j2 | 38 +++++++ .../stacks/HDP/3.0/services/SLIDER/metainfo.xml | 51 +++++++++ 15 files changed, 871 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/configuration/slider-client.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/configuration/slider-client.xml b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/configuration/slider-client.xml new file mode 100644 index 0000000..adeba58 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/configuration/slider-client.xml @@ -0,0 +1,56 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="true"> + <!-- + <property> + <name>slider.security.protocol.acl</name> + <value>*</value> + <description>When security is enabled, set appropriate acl. Default value means allow everyone.</description> + </property> + --> + <!-- + The recommended approach is to configure slider-env.sh and set HADOOP_CONF_DIR. + Otherwise, appropriate configurations from hdfs-site, yarn-site, can be dropped in this file + for Slider client to work. The following list is not an exhaustive list but the minimal config + needed to interact with a non-secure cluster. + --> + <!-- + <property> + <name>yarn.resourcemanager.address</name> + <value>localhost:8050</value> + <description>The address of the applications manager interface in the RM.</description> + </property> + + <property> + <name>yarn.resourcemanager.scheduler.address</name> + <value>localhost:8030</value> + <description>The address of the scheduler interface.</description> + </property> + + <property> + <name>fs.defaultFS</name> + <value>hdfs://localhost:8020</value> + <description>The name of the default file system. Either the + literal string "local" or a host:port for HDFS.</description> + </property> + --> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/configuration/slider-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/configuration/slider-env.xml b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/configuration/slider-env.xml new file mode 100644 index 0000000..ce06a18 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/configuration/slider-env.xml @@ -0,0 +1,46 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <!-- slider-env.sh --> + <property> + <name>content</name> + <display-name>slider-env template</display-name> + <description>This is the jinja template for slider-env.sh file</description> + <value> +# Set Slider-specific environment variables here. + +# The only required environment variable is JAVA_HOME. All others are +# optional. When running a distributed configuration it is best to +# set JAVA_HOME in this file, so that it is correctly defined on +# remote nodes. + +# The java implementation to use. Required. +export JAVA_HOME={{java64_home}} +# The hadoop conf directory. Optional as slider-client.xml can be edited to add properties. +export HADOOP_CONF_DIR={{hadoop_conf_dir}} + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/configuration/slider-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/configuration/slider-log4j.xml b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/configuration/slider-log4j.xml new file mode 100644 index 0000000..cd9b2a4 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/configuration/slider-log4j.xml @@ -0,0 +1,92 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>content</name> + <display-name>slider-log4j template</display-name> + <description>Custom log4j.properties</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Define some default values that can be overridden by system properties +log4j.rootLogger=INFO,stdout +log4j.threshhold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + +# log layout skips stack-trace creation operations by avoiding line numbers and method +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +# debug edition is much more expensive +#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n + + +log4j.appender.subprocess=org.apache.log4j.ConsoleAppender +log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout +log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n +#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess + +# for debugging Slider +#log4j.logger.org.apache.slider=DEBUG +#log4j.logger.org.apache.slider=DEBUG + +# uncomment to debug service lifecycle issues +#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG +#log4j.logger.org.apache.hadoop.yarn.service=DEBUG + +# uncomment for YARN operations +#log4j.logger.org.apache.hadoop.yarn.client=DEBUG + +# uncomment this to debug security problems +#log4j.logger.org.apache.hadoop.security=DEBUG + +#crank back on some noise +log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +log4j.logger.org.apache.hadoop.hdfs=WARN + + +log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN +log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN +log4j.logger.org.apache.zookeeper=WARN + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/kerberos.json b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/kerberos.json new file mode 100644 index 0000000..0ebeb7d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/kerberos.json @@ -0,0 +1,17 @@ +{ + "services": [ + { + "name": "SLIDER", + "components": [ + { + "name": "SLIDER", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + } + ] + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/metainfo.xml new file mode 100644 index 0000000..168a2df --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/metainfo.xml @@ -0,0 +1,109 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>SLIDER</name> + <displayName>Slider</displayName> + <comment>A framework for deploying, managing and monitoring existing distributed applications on YARN.</comment> + <version>0.91.0.3.0</version> + <components> + <component> + <name>SLIDER</name> + <displayName>Slider Client</displayName> + <category>CLIENT</category> + <cardinality>0+</cardinality> + <versionAdvertised>true</versionAdvertised> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>YARN/YARN_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/slider_client.py</script> + <scriptType>PYTHON</scriptType> + <timeout>1200</timeout> + </commandScript> + <configFiles> + <configFile> + <type>xml</type> + <fileName>slider-client.xml</fileName> + <dictionaryName>slider-client</dictionaryName> + </configFile> + <configFile> + <type>xml</type> + <fileName>core-site.xml</fileName> + <dictionaryName>core-site</dictionaryName> + </configFile> + <configFile> + <type>xml</type> + <fileName>hdfs-site.xml</fileName> + <dictionaryName>hdfs-site</dictionaryName> + </configFile> + <configFile> + <type>xml</type> + <fileName>yarn-site.xml</fileName> + <dictionaryName>yarn-site</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>slider-env.sh</fileName> + <dictionaryName>slider-env</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>log4j.properties</fileName> + <dictionaryName>slider-log4j</dictionaryName> + </configFile> + </configFiles> + </component> + </components> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>YARN</service> + <service>HDFS</service> + <service>ZOOKEEPER</service> + </requiredServices> + + <configuration-dependencies> + <config-type>slider-log4j</config-type> + <config-type>slider-client</config-type> + <config-type>slider-env</config-type> + </configuration-dependencies> + + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/files/hbaseSmokeVerify.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/files/hbaseSmokeVerify.sh b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/files/hbaseSmokeVerify.sh new file mode 100644 index 0000000..5c320c0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/files/hbaseSmokeVerify.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +conf_dir=$1 +data=$2 +hbase_cmd=$3 +echo "scan 'ambarismoketest'" | $hbase_cmd --config $conf_dir shell > /tmp/hbase_chk_verify +cat /tmp/hbase_chk_verify +echo "Looking for $data" +grep -q $data /tmp/hbase_chk_verify +if [ "$?" -ne 0 ] +then + exit 1 +fi + +grep -q '1 row(s)' /tmp/hbase_chk_verify http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/__init__.py b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/__init__.py new file mode 100644 index 0000000..5561e10 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/params.py new file mode 100644 index 0000000..c03329c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/params.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from ambari_commons.os_check import OSCheck +from resource_management.libraries.functions import format +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.copy_tarball import get_sysprep_skip_copy_tarballs_hdfs + +if OSCheck.is_windows_family(): + from params_windows import * +else: + from params_linux import * + +# server configurations +config = Script.get_config() + +stack_name = default("/hostLevelParams/stack_name", None) + +# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade +version = default("/commandParams/version", None) +sysprep_skip_copy_tarballs_hdfs = get_sysprep_skip_copy_tarballs_hdfs() + +#hadoop params +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() + +smokeuser = config['configurations']['cluster-env']['smokeuser'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +slider_env_sh_template = config['configurations']['slider-env']['content'] + +java64_home = config['hostLevelParams']['java_home'] +log4j_props = config['configurations']['slider-log4j']['content'] +slider_cmd = format("{slider_bin_dir}/slider") http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/params_linux.py new file mode 100644 index 0000000..350be45 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/params_linux.py @@ -0,0 +1,80 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.libraries.resources import HdfsResource +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources + +# server configurations +config = Script.get_config() +stack_root = Script.get_stack_root() + +slider_home_dir = format('{stack_root}/current/slider-client') +stack_version_unformatted = str(config['hostLevelParams']['stack_version']) +stack_version_formatted = format_stack_version(stack_version_unformatted) + +#hadoop params +slider_bin_dir = format('{slider_home_dir}/bin') + +slider_conf_dir = format("{slider_home_dir}/conf") +storm_slider_conf_dir = format('{stack_root}/current/storm-slider-client/conf') + +slider_lib_dir = format('{slider_home_dir}/lib') +slider_tar_gz = format('{slider_lib_dir}/slider.tar.gz') + +user_group = config['configurations']['cluster-env']['user_group'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] + + +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +security_enabled = config['configurations']['cluster-env']['security_enabled'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] + +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() + +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs, + immutable_paths = get_not_managed_resources() +) http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/params_windows.py new file mode 100644 index 0000000..fd2f358 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/params_windows.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script.script import Script +import os + +# server configurations +config = Script.get_config() + +stack_root = None +slider_home = None +slider_bin_dir = None +slider_conf_dir = None +storm_slider_conf_dir = None +try: + stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],"..")) + slider_home = os.environ['SLIDER_HOME'] + slider_bin_dir = os.path.join(slider_home, 'bin') + slider_conf_dir = os.path.join(slider_home, 'conf') + storm_slider_conf_dir = os.path.join(os.environ['STORM_HOME'], 'conf') +except: + pass + +slider_home_dir = slider_home + +hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"] +slider_user = hadoop_user +hdfs_user = hadoop_user http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/service_check.py new file mode 100644 index 0000000..8582b5e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/service_check.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs +from resource_management.libraries.functions.constants import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.format import format +from resource_management.core.resources.system import Execute +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst + +class SliderServiceCheck(Script): + + @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) + def service_check(self, env): + import params + env.set_params(params) + smoke_cmd = os.path.join(params.stack_root, "Run-SmokeTests.cmd") + service = "SLIDER" + Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True, user=params.hdfs_user) + + @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) + def service_check(self, env): + import params + env.set_params(params) + + if params.stack_version_formatted and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted): + copy_to_hdfs("slider", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs) + + smokeuser_kinit_cmd = format( + "{kinit_path_local} -kt {smokeuser_keytab} {smokeuser_principal};") if params.security_enabled else "" + + servicecheckcmd = format("{smokeuser_kinit_cmd} {slider_cmd} list") + + Execute(servicecheckcmd, + tries=3, + try_sleep=5, + user=params.smokeuser, + logoutput=True + ) + + +if __name__ == "__main__": + SliderServiceCheck().execute() + http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/slider.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/slider.py b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/slider.py new file mode 100644 index 0000000..f7d7fbf --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/slider.py @@ -0,0 +1,96 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +import os +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.constants import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.core.resources.system import Directory, File +from resource_management.core.source import Template, InlineTemplate +from resource_management.libraries.functions.format import format +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def slider(): + import params + + slider_client_config = params.config['configurations']['slider-client'] if 'configurations' in params.config and 'slider-client' in params.config['configurations'] else {} + + XmlConfig("slider-client.xml", + conf_dir=params.slider_conf_dir, + configurations=slider_client_config + ) + + if (params.log4j_props != None): + File(os.path.join(params.slider_conf_dir, "log4j.properties"), + content=params.log4j_props + ) + elif (os.path.exists(os.path.join(params.slider_conf_dir, "log4j.properties"))): + File(os.path.join(params.slider_conf_dir, "log4j.properties")) + + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def slider(): + import params + + Directory(params.slider_conf_dir, + create_parents = True + ) + + slider_client_config = params.config['configurations']['slider-client'] if 'configurations' in params.config and 'slider-client' in params.config['configurations'] else {} + + XmlConfig("slider-client.xml", + conf_dir=params.slider_conf_dir, + configurations=slider_client_config, + mode=0644 + ) + + File(format("{slider_conf_dir}/slider-env.sh"), + mode=0755, + content=InlineTemplate(params.slider_env_sh_template) + ) + + # check to see if the current/storm_slider_client symlink is broken if it is then the storm slider client is not installed + storm_slider_client_dir = os.path.join(params.storm_slider_conf_dir, "..") + if (os.path.exists(storm_slider_client_dir) or not os.path.islink(storm_slider_client_dir)): + Directory(params.storm_slider_conf_dir, + create_parents = True + ) + + File(format("{storm_slider_conf_dir}/storm-slider-env.sh"), + mode=0755, + content=Template('storm-slider-env.sh.j2') + ) + + if (params.log4j_props != None): + File(format("{params.slider_conf_dir}/log4j.properties"), + mode=0644, + content=params.log4j_props + ) + elif (os.path.exists(format("{params.slider_conf_dir}/log4j.properties"))): + File(format("{params.slider_conf_dir}/log4j.properties"), + mode=0644 + ) + if params.stack_version_formatted and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted): + File(params.slider_tar_gz, + owner=params.hdfs_user, + group=params.user_group, + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/slider_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/slider_client.py b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/slider_client.py new file mode 100644 index 0000000..cf9f830 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/scripts/slider_client.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions import conf_select, stack_select +from resource_management.libraries.functions.constants import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from slider import slider +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from resource_management.core.exceptions import ClientComponentHasNoStatus + +class SliderClient(Script): + def status(self, env): + raise ClientComponentHasNoStatus() + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class SliderClientLinux(SliderClient): + def get_component_name(self): + return "slider-client" + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + conf_select.select(params.stack_name, "slider", params.version) + stack_select.select("slider-client", params.version) + + # also set all of the hadoop clients since slider client is upgraded as + # part of the final "CLIENTS" group and we need to ensure that + # hadoop-client is also set + conf_select.select(params.stack_name, "hadoop", params.version) + stack_select.select("hadoop-client", params.version) + + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + slider() + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class SliderClientWindows(SliderClient): + def install(self, env): + import params + if params.slider_home is None: + self.install_packages(env) + self.configure(env) + +if __name__ == "__main__": + SliderClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/templates/storm-slider-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/templates/storm-slider-env.sh.j2 b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/templates/storm-slider-env.sh.j2 new file mode 100644 index 0000000..8022a4b --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/package/templates/storm-slider-env.sh.j2 @@ -0,0 +1,38 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +#!/usr/bin/env python +# -*- coding: utf-8 -*- +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +export JAVA_HOME={{java64_home}} +export SLIDER_HOME={{slider_home_dir}} http://git-wip-us.apache.org/repos/asf/ambari/blob/528ac5c3/ambari-server/src/main/resources/stacks/HDP/3.0/services/SLIDER/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/SLIDER/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/SLIDER/metainfo.xml new file mode 100644 index 0000000..23f558d --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/SLIDER/metainfo.xml @@ -0,0 +1,51 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>SLIDER</name> + <version>0.91.0.3.0</version> + <extends>common-services/SLIDER/0.91.0.3.0</extends> + <osSpecifics> + <osSpecific> + <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily> + <packages> + <package> + <name>slider_${stack_version}</name> + </package> + <!--<package>--> + <!--<name>storm_${stack_version}-slider-client</name>--> + <!--</package>--> + </packages> + </osSpecific> + <osSpecific> + <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily> + <packages> + <package> + <name>slider-${stack_version}</name> + </package> + <!--<package>--> + <!--<name>storm-${stack_version}-slider-client</name>--> + <!--</package>--> + </packages> + </osSpecific> + </osSpecifics> + </service> + </services> +</metainfo>
