This is an automated email from the ASF dual-hosted git repository. jialiang pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push: new 9743c79440 AMBARI-26090: Add Livy as Individual Service to Ambari Bigtop Stack (#3790) 9743c79440 is described below commit 9743c794407deff76c5a657d8fb6331d0def8fa7 Author: William Horn <william.horn....@gmail.com> AuthorDate: Sat Jul 27 00:20:40 2024 -0400 AMBARI-26090: Add Livy as Individual Service to Ambari Bigtop Stack (#3790) * feat: Add Livy service to Ambari Bigtop stack (#1) feat: Add Livy service to Ambari Bigtop stack Co-authored-by: Carmen Blaine <carmenbla...@gmail.com> * Remove Hive as a dependency from Livy * removed SPARK STS as start up dependency for Livy --------- Co-authored-by: Paul Yakovlev <yvpa...@gmail.com> Co-authored-by: Carmen Blaine <carmenbla...@gmail.com> Co-authored-by: carmenblaine3 <87210471+carmenblai...@users.noreply.github.com> --- .../BIGTOP/3.2.0/properties/stack_packages.json | 26 ++++ .../stacks/BIGTOP/3.2.0/services/LIVY/alerts.json | 30 ++++ .../LIVY/configuration/livy-client-conf.xml | 30 ++++ .../services/LIVY/configuration/livy-conf.xml | 129 +++++++++++++++++ .../3.2.0/services/LIVY/configuration/livy-env.xml | 108 +++++++++++++++ .../LIVY/configuration/livy-log4j-properties.xml | 41 ++++++ .../LIVY/configuration/livy-spark-blacklist.xml | 52 +++++++ .../BIGTOP/3.2.0/services/LIVY/kerberos.json | 52 +++++++ .../stacks/BIGTOP/3.2.0/services/LIVY/metainfo.xml | 124 +++++++++++++++++ .../LIVY/package/scripts/alerts/alert_livy_port.py | 154 +++++++++++++++++++++ .../services/LIVY/package/scripts/livy_server.py | 149 ++++++++++++++++++++ .../services/LIVY/package/scripts/livy_service.py | 45 ++++++ .../{SPARK => LIVY}/package/scripts/params.py | 60 +++++++- .../services/LIVY/package/scripts/service_check.py | 56 ++++++++ .../services/LIVY/package/scripts/setup_livy.py | 102 ++++++++++++++ .../services/LIVY/package/scripts/status_params.py | 45 ++++++ .../package/templates/input.config-livy.json.j2} | 27 +--- .../3.2.0/services/LIVY/role_command_order.json | 8 ++ .../3.2.0/services/LIVY/themes/directories.json | 89 ++++++++++++ .../3.2.0/services/SPARK/package/scripts/params.py | 13 +- .../services/SPARK/package/scripts/setup_spark.py | 15 +- .../package/templates/input.config-spark.json.j2 | 3 +- .../stacks/BIGTOP/3.3.0/services/LIVY/metainfo.xml | 27 ++++ 23 files changed, 1340 insertions(+), 45 deletions(-) diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json index d3fc8d65c4..a050d4c5ef 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json @@ -271,6 +271,32 @@ ] } }, + "LIVY": { + "LIVY_CLIENT": { + "STACK-SELECT-PACKAGE": "livy-client", + "INSTALL": [ + "livy-client" + ], + "PATCH": [ + "livy-client" + ], + "STANDARD": [ + "livy-client" + ] + }, + "LIVY_SERVER": { + "STACK-SELECT-PACKAGE": "livy-server", + "INSTALL": [ + "livy-server" + ], + "PATCH": [ + "livy-server" + ], + "STANDARD": [ + "livy-server" + ] + } + }, "FLINK": { "FLINK_CLIENT": { "STACK-SELECT-PACKAGE": "flink-client", diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/alerts.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/alerts.json new file mode 100644 index 0000000000..2e753dcd30 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/alerts.json @@ -0,0 +1,30 @@ +{ + "SPARK": { + "service": [], + "LIVY_SERVER": [ + { + "name": "livy_server_status", + "label": "Spark Livy Server", + "description": "This host-level alert is triggered if the Livy2 Server cannot be determined to be up.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "SCRIPT", + "path": "BIGTOP/3.2.0/services/LIVY/package/scripts/alerts/alert_livy_port.py", + "parameters": [ + { + "name": "check.command.timeout", + "display_name": "Command Timeout", + "value": 60.0, + "type": "NUMERIC", + "description": "The maximum time before check command will be killed by timeout", + "units": "seconds", + "threshold": "CRITICAL" + } + ] + } + } + ] + } + } + \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-client-conf.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-client-conf.xml new file mode 100644 index 0000000000..d761f412a8 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-client-conf.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="true"> + <property> + <name>livy.rsc.launcher.address</name> + <value> </value> + <description> + Address for the RSC driver to connect back with it's connection info. + </description> + <on-ambari-upgrade add="true"/> + </property> +</configuration> diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-conf.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-conf.xml new file mode 100644 index 0000000000..a0521c360f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-conf.xml @@ -0,0 +1,129 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="true"> + <property> + <name>livy.environment</name> + <value>production</value> + <description> + Specifies Livy's environment. May either be "production" or "development". In "development" + mode, Livy will enable debugging options, such as reporting possible routes on a 404. + defaults to development + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.server.port</name> + <value>8999</value> + <description> + What port to start the server on. Defaults to 8999. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.server.session.timeout</name> + <value>3600000</value> + <description> + Time in milliseconds on how long Livy will wait before timing out an idle session. + Default is one hour. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.impersonation.enabled</name> + <value>true</value> + <description> + If livy should use proxy users when submitting a job. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.impersonation.enabled</name> + <value>true</value> + <description> + If livy should use proxy users when submitting a job. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.server.csrf_protection.enabled</name> + <value>true</value> + <description> + Whether to enable csrf protection for livy's rest api. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.spark.master</name> + <value>yarn</value> + <description> + spark.master property for spark engine + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.spark.deploy-mode</name> + <value>cluster</value> + <description> + spark.deploy-mode property for spark engine + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.repl.enableHiveContext</name> + <value>true</value> + <description> + Whether to enable HiveContext in livy interpreter + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.server.recovery.mode</name> + <value>recovery</value> + <description> + Recovery mode for livy, either be "off" or "recovery". + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.server.recovery.state-store</name> + <value>filesystem</value> + <description> + Where Livy should store state for recovery. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.server.recovery.state-store.url</name> + <value>/livy-recovery</value> + <description> + Where Livy should store state for recovery. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.server.access-control.enabled</name> + <value>true</value> + <description> + Property to configure Livy user access. + </description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-env.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-env.xml new file mode 100644 index 0000000000..0b27bed3ff --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-env.xml @@ -0,0 +1,108 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <property> + <name>livy_user</name> + <display-name>Livy User</display-name> + <value>livy</value> + <property-type>USER</property-type> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + <user-groups> + <property> + <type>cluster-env</type> + <name>user_group</name> + </property> + <property> + <type>livy-env</type> + <name>livy_group</name> + </property> + </user-groups> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy_group</name> + <display-name>Livy Group</display-name> + <value>livy</value> + <property-type>GROUP</property-type> + <description>livy group</description> + <value-attributes> + <type>user</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy_log_dir</name> + <display-name>Livy Log directory</display-name> + <value>/var/log/livy</value> + <description>Livy Log Dir</description> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy_pid_dir</name> + <display-name>Livy PID directory</display-name> + <value>/var/run/livy</value> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>spark_home</name> + <value>/usr/bigtop/current/spark-client</value> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <!-- livy-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for livy-env.sh file</description> + <value> + #!/usr/bin/env bash + + # - SPARK_HOME Spark which you would like to use in livy + # - SPARK_CONF_DIR Directory containing the Spark configuration to use. + # - HADOOP_CONF_DIR Directory containing the Hadoop / YARN configuration to use. + # - LIVY_LOG_DIR Where log files are stored. (Default: ${LIVY_HOME}/logs) + # - LIVY_PID_DIR Where the pid file is stored. (Default: /tmp) + # - LIVY_SERVER_JAVA_OPTS Java Opts for running livy server (You can set jvm related setting here, like jvm memory/gc algorithm and etc.) + export SPARK_HOME=/usr/bigtop/current/spark-client + export SPARK_CONF_DIR=/etc/spark/conf + export JAVA_HOME={{java_home}} + export HADOOP_CONF_DIR=/etc/hadoop/conf + export LIVY_LOG_DIR={{livy_log_dir}} + export LIVY_PID_DIR={{livy_pid_dir}} + export LIVY_SERVER_JAVA_OPTS="-Xmx2g" + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-log4j-properties.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-log4j-properties.xml new file mode 100644 index 0000000000..fc15c7f6b9 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-log4j-properties.xml @@ -0,0 +1,41 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>content</name> + <description>Livy-log4j-Properties</description> + <value> + # Set everything to be logged to the console + log4j.rootCategory=INFO, console + log4j.appender.console=org.apache.log4j.ConsoleAppender + log4j.appender.console.target=System.err + log4j.appender.console.layout=org.apache.log4j.PatternLayout + log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n + + log4j.logger.org.eclipse.jetty=WARN + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-spark-blacklist.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-spark-blacklist.xml new file mode 100644 index 0000000000..84c899c1e0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-spark-blacklist.xml @@ -0,0 +1,52 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>content</name> + <description>spark-blacklist.properties</description> + <value> + # + # Configuration override / blacklist. Defines a list of properties that users are not allowed + # to override when starting Spark sessions. + # + # This file takes a list of property names (one per line). Empty lines and lines starting with "#" + # are ignored. + # + + # Disallow overriding the master and the deploy mode. + spark.master + spark.submit.deployMode + + # Disallow overriding the location of Spark cached jars. + spark.yarn.jar + spark.yarn.jars + spark.yarn.archive + + # Don't allow users to override the RSC timeout. + livy.rsc.server.idle_timeout + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/kerberos.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/kerberos.json new file mode 100644 index 0000000000..5a6ca1590c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/kerberos.json @@ -0,0 +1,52 @@ +{ + "services": [ + { + "name": "LIVY_SERVER", + "identities": [ + { + "name": "hdfs", + "reference": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "livyuser", + "principal": { + "value": "${livy-env/livy_user}/_HOST@${realm}", + "type": "service", + "configuration": "livy-conf/livy.server.launch.kerberos.principal", + "local_username": "${livy-env/livy_user}" + }, + "keytab": { + "file": "${keytab_dir}/livy.service.keytab", + "owner": { + "name": "${livy-env/livy_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "livy-conf/livy.server.launch.kerberos.keytab" + } + }, + { + "name": "livy_spnego", + "reference": "/spnego", + "principal": { + "configuration": "livy-conf/livy.server.auth.kerberos.principal" + }, + "keytab": { + "configuration": "livy-conf/livy.server.auth.kerberos.keytab" + } + } + ], + "configurations": [ + { + "livy-conf": { + "livy.server.auth.type": "kerberos", + "livy.impersonation.enabled": "true" + } + } + ] + } + ] + } \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/metainfo.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/metainfo.xml new file mode 100644 index 0000000000..fece912b2c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/metainfo.xml @@ -0,0 +1,124 @@ +<?xml version="1.0"?> +<!--Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>LIVY</name> + <displayName>Livy</displayName> + <comment>Apache Livy is a service that enables easy interaction with a Spark cluster over a REST interface. </comment> + <version>0.7.1-1</version> + <components> + <component> + <name>LIVY_SERVER</name> + <displayName>Livy Server</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> + <dependencies> + <dependency> + <name>SPARK/SPARK_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>YARN/YARN_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/livy_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + <logs> + <log> + <logId>livy_server</logId> + <primary>true</primary> + </log> + </logs> + </component> + </components> + + <configuration-dependencies> + <config-type>core-site</config-type> + <config-type>spark-defaults</config-type> + <config-type>spark-env</config-type> + <config-type>spark-log4j-properties</config-type> + <config-type>spark-metrics-properties</config-type> + <config-type>spark-hive-site-override</config-type> + <config-type>spark-thrift-fairscheduler</config-type> + <config-type>livy-client-conf</config-type> + <config-type>livy-conf</config-type> + <config-type>livy-env</config-type> + <config-type>livy-log4j-properties</config-type> + <config-type>livy-spark-blacklist</config-type> + </configuration-dependencies> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>HDFS</service> + <service>YARN</service> + <service>SPARK</service> + </requiredServices> + + <osSpecifics> + <osSpecific> + <osFamily>redhat7</osFamily> + <packages> + <package> + <name>spark_${stack_version}-core</name> + </package> + <package> + <name>spark_${stack_version}-python</name> + </package> + <package> + <name>livy_${stack_version}</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <themes> + <theme> + <fileName>directories.json</fileName> + <default>true</default> + </theme> + </themes> + + </service> + </services> +</metainfo> diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/alerts/alert_livy_port.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/alerts/alert_livy_port.py new file mode 100644 index 0000000000..50d3791a20 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/alerts/alert_livy_port.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import time +import logging +import traceback +import socket +from resource_management import * +from resource_management.libraries.functions import format +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from resource_management.libraries.script.script import Script +from resource_management.core.resources import Execute +from resource_management.core.logger import Logger +from resource_management.core import global_lock +from resource_management.libraries.functions import get_kinit_path + + +OK_MESSAGE = "TCP OK - {0:.3f}s response on port {1}" +CRITICAL_MESSAGE = "Connection failed on host {0}:{1} ({2})" + +logger = logging.getLogger('ambari_alerts') + +LIVY_SERVER_HOST_KEY = '{{livy-conf/livy.server.host}}' + +LIVY_SERVER_PORT_KEY = '{{livy-conf/livy.server.port}}' + +LIVYUSER_DEFAULT = 'livy' + +CHECK_COMMAND_TIMEOUT_KEY = 'check.command.timeout' +CHECK_COMMAND_TIMEOUT_DEFAULT = 60.0 + +SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}' +SMOKEUSER_KEYTAB_KEY = '{{cluster-env/smokeuser_keytab}}' +SMOKEUSER_PRINCIPAL_KEY = '{{cluster-env/smokeuser_principal_name}}' +SMOKEUSER_KEY = '{{cluster-env/smokeuser}}' +LIVY_SSL_ENABLED_KEY = '{{livy-conf/livy.keystore}}' + +# The configured Kerberos executable search paths, if any +KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY = '{{kerberos-env/executable_search_paths}}' + + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def get_tokens(): + """ + Returns a tuple of tokens in the format {{site/property}} that will be used + to build the dictionary passed into execute + """ + return (LIVY_SERVER_HOST_KEY,LIVY_SERVER_PORT_KEY,LIVYUSER_DEFAULT,SECURITY_ENABLED_KEY,SMOKEUSER_KEYTAB_KEY,SMOKEUSER_PRINCIPAL_KEY,SMOKEUSER_KEY,LIVY_SSL_ENABLED_KEY) + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def execute(configurations={}, parameters={}, host_name=None): + """ + Returns a tuple containing the result code and a pre-formatted result label + + Keyword arguments: + configurations (dictionary): a mapping of configuration key to value + parameters (dictionary): a mapping of script parameter key to value + host_name (string): the name of this host where the alert is running + """ + + if configurations is None: + return ('UNKNOWN', ['There were no configurations supplied to the script.']) + + LIVY_PORT_DEFAULT = 8999 + + port = LIVY_PORT_DEFAULT + if LIVY_SERVER_PORT_KEY in configurations: + port = int(configurations[LIVY_SERVER_PORT_KEY]) + + if LIVY_SERVER_HOST_KEY in configurations: + host_name = str(configurations[LIVY_SERVER_HOST_KEY]) + + if host_name is None: + host_name = socket.getfqdn() + + livyuser = configurations[SMOKEUSER_KEY] + + security_enabled = False + if SECURITY_ENABLED_KEY in configurations: + security_enabled = str(configurations[SECURITY_ENABLED_KEY]).upper() == 'TRUE' + + smokeuser_kerberos_keytab = None + if SMOKEUSER_KEYTAB_KEY in configurations: + smokeuser_kerberos_keytab = configurations[SMOKEUSER_KEYTAB_KEY] + + if host_name is None: + host_name = socket.getfqdn() + + smokeuser_principal = None + if SMOKEUSER_PRINCIPAL_KEY in configurations: + smokeuser_principal = configurations[SMOKEUSER_PRINCIPAL_KEY] + smokeuser_principal = smokeuser_principal.replace('_HOST',host_name.lower()) + + # Get the configured Kerberos executable search paths, if any + if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations: + kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY] + else: + kerberos_executable_search_paths = None + + kinit_path_local = get_kinit_path(kerberos_executable_search_paths) + + if security_enabled: + kinitcmd = format("{kinit_path_local} -kt {smokeuser_kerberos_keytab} {smokeuser_principal}; ") + # prevent concurrent kinit + kinit_lock = global_lock.get_lock(global_lock.LOCK_TYPE_KERBEROS) + kinit_lock.acquire() + try: + Execute(kinitcmd, user=livyuser) + finally: + kinit_lock.release() + + http_scheme = 'https' if LIVY_SSL_ENABLED_KEY in configurations else 'http' + result_code = None + try: + start_time = time.time() + try: + livy_livyserver_host = str(host_name) + + livy_cmd = format("curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k {http_scheme}://{livy_livyserver_host}:{port}/sessions | grep 200 ") + + Execute(livy_cmd, + tries=3, + try_sleep=1, + logoutput=True, + user=livyuser + ) + + total_time = time.time() - start_time + result_code = 'OK' + label = OK_MESSAGE.format(total_time, port) + except: + result_code = 'CRITICAL' + label = CRITICAL_MESSAGE.format(host_name, port, traceback.format_exc()) + except: + label = traceback.format_exc() + result_code = 'UNKNOWN' + + return (result_code, [label]) diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/livy_server.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/livy_server.py new file mode 100644 index 0000000000..a123a5bf66 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/livy_server.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.constants import StackFeature +from resource_management.core.exceptions import Fail +from resource_management.core.resources.system import Execute +from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil +from resource_management.libraries.providers.hdfs_resource import HdfsResourceProvider +from resource_management import is_empty +from resource_management import shell +from resource_management.libraries.functions.decorator import retry +from resource_management.core.logger import Logger +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import namenode_ha_utils + +from livy_service import livy_service +from setup_livy import setup_livy + +class LivyServer(Script): + + def install(self, env): + import params + env.set_params(params) + + self.install_packages(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + import params + env.set_params(params) + + setup_livy(env, 'server', upgrade_type=upgrade_type, action = 'config') + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + + if params.has_ats and params.has_livyserver: + Logger.info("Verifying DFS directories where ATS stores time line data for active and completed applications.") + self.wait_for_dfs_directories_created([params.entity_groupfs_store_dir, params.entity_groupfs_active_dir]) + + self.configure(env) + livy_service('server', upgrade_type=upgrade_type, action='start') + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + livy_service('server', upgrade_type=upgrade_type, action='stop') + + def status(self, env): + import status_params + env.set_params(status_params) + + check_process_status(status_params.livy_server_pid_file) + + # TODO move out and compose with similar method in resourcemanager.py + def wait_for_dfs_directories_created(self, dirs): + import params + + ignored_dfs_dirs = HdfsResourceProvider.get_ignored_resources_list(params.hdfs_resource_ignore_file) + + if params.security_enabled: + Execute(format("{kinit_path_local} -kt {livy_kerberos_keytab} {livy_principal}"), + user=params.livy_user + ) + Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"), + user=params.hdfs_user + ) + + for dir_path in dirs: + self.wait_for_dfs_directory_created(dir_path, ignored_dfs_dirs) + + def get_pid_files(self): + import status_params + return [status_params.livy_server_pid_file] + + + @retry(times=8, sleep_time=20, backoff_factor=1, err_class=Fail) + def wait_for_dfs_directory_created(self, dir_path, ignored_dfs_dirs): + import params + + if not is_empty(dir_path): + dir_path = HdfsResourceProvider.parse_path(dir_path) + + if dir_path in ignored_dfs_dirs: + Logger.info("Skipping DFS directory '" + dir_path + "' as it's marked to be ignored.") + return + + Logger.info("Verifying if DFS directory '" + dir_path + "' exists.") + + dir_exists = None + + nameservices = namenode_ha_utils.get_nameservices(params.hdfs_site) + nameservice = None if not nameservices else nameservices[-1] + + if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.dfs_type): + # check with webhdfs is much faster than executing hdfs dfs -test + util = WebHDFSUtil(params.hdfs_site, nameservice, params.hdfs_user, params.security_enabled) + list_status = util.run_command(dir_path, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False) + dir_exists = ('FileStatus' in list_status) + else: + # have to do time expensive hdfs dfs -d check. + dfs_ret_code = shell.call(format("hdfs --config {hadoop_conf_dir} dfs -test -d " + dir_path), user=params.livy_user)[0] + dir_exists = not dfs_ret_code #dfs -test -d returns 0 in case the dir exists + + if not dir_exists: + raise Fail("DFS directory '" + dir_path + "' does not exist !") + else: + Logger.info("DFS directory '" + dir_path + "' exists.") + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + + env.set_params(params) + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + Logger.info("Executing Livy Server Stack Upgrade pre-restart") + stack_select.select_packages(params.version) + + def get_log_folder(self): + import params + return params.livy_log_dir + + def get_user(self): + import params + return params.livy_user +if __name__ == "__main__": + LivyServer().execute() + diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/livy_service.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/livy_service.py new file mode 100644 index 0000000000..55aa1b2902 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/livy_service.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from resource_management.libraries.functions import format +from resource_management.core.resources.system import File, Execute +from resource_management.libraries.functions import get_user_call_output + + +def livy_service(name, upgrade_type=None, action=None): + import params + + # use the livy user to get the PID (it is protected on non-root systems) + livy_server_pid = get_user_call_output.get_user_call_output(format("cat {livy_server_pid_file}"), + user=params.livy_user, is_checked_call=False)[1] + livy_server_pid = livy_server_pid.replace("\n", " ") + process_id_exists_command = format("ls {livy_server_pid_file} >/dev/null 2>&1 && ps -p {livy_server_pid} >/dev/null 2>&1") + + if action == 'start': + Execute(format('{livy_server_start}'), + user=params.livy_user, + environment={'JAVA_HOME': params.java_home}, + not_if=process_id_exists_command) + elif action == 'stop': + Execute(format('{livy_server_stop}'), + user=params.livy_user, + only_if=process_id_exists_command, + timeout=10, + on_timeout=format("! ( {process_id_exists_command} ) || {sudo} -H -E kill -9 {livy_server_pid}"), + environment={'JAVA_HOME': params.java_home}) diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/params.py similarity index 81% copy from ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py copy to ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/params.py index 222810f5b3..a38ef7b786 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/params.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 """ Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file @@ -45,7 +45,9 @@ sudo = AMBARI_SUDO_BINARY SERVER_ROLE_DIRECTORY_MAP = { 'SPARK_JOBHISTORYSERVER' : 'spark-historyserver', 'SPARK_CLIENT' : 'spark-client', - 'SPARK_THRIFTSERVER' : 'spark-thriftserver' + 'SPARK_THRIFTSERVER' : 'spark-thriftserver', + 'LIVY_SERVER' : 'livy-server', + 'LIVY_CLIENT' : 'livy-client' } HIVE_SERVER_ROLE_DIRECTORY_MAP = { @@ -241,7 +243,56 @@ has_ats = len(ats_host) > 0 dfs_type = default("/clusterLevelParams/dfs_type", "") - +# livy configs +has_livyserver = False + +if stack_version_formatted and "livy-env" in config['configurations']: + livy_component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "LIVY_SERVER") + livy_conf = format("{stack_root}/current/{livy_component_directory}/conf") + livy_log_dir = config['configurations']['livy-env']['livy_log_dir'] + livy_pid_dir = status_params.livy_pid_dir + livy_home = format("{stack_root}/current/{livy_component_directory}") + livy_user = status_params.livy_user + livy_group = status_params.livy_group + user_group = status_params.user_group + livy_hdfs_user_dir = format("/user/{livy_user}") + livy_server_pid_file = status_params.livy_server_pid_file + livy_recovery_dir = default("/configurations/livy-conf/livy.server.recovery.state-store.url", "/livy-recovery") + livy_recovery_store = default("/configurations/livy-conf/livy.server.recovery.state-store", "filesystem") + + livy_server_start = format("{livy_home}/bin/livy-server start") + livy_server_stop = format("{livy_home}/bin/livy-server stop") + livy_logs_dir = format("{livy_home}/logs") + + livy_env_sh = config['configurations']['livy-env']['content'] + livy_log4j_properties = config['configurations']['livy-log4j-properties']['content'] + livy_spark_blacklist_properties = config['configurations']['livy-spark-blacklist']['content'] + + if 'livy.server.kerberos.keytab' in config['configurations']['livy-conf']: + livy_kerberos_keytab = config['configurations']['livy-conf']['livy.server.kerberos.keytab'] + else: + livy_kerberos_keytab = config['configurations']['livy-conf']['livy.server.launch.kerberos.keytab'] + if 'livy.server.kerberos.principal' in config['configurations']['livy-conf']: + livy_kerberos_principal = config['configurations']['livy-conf']['livy.server.kerberos.principal'] + else: + livy_kerberos_principal = config['configurations']['livy-conf']['livy.server.launch.kerberos.principal'] + + livy_livyserver_hosts = default("/clusterHostInfo/livy_server_hosts", []) + livy_http_scheme = 'https' if 'livy.keystore' in config['configurations']['livy-conf'] else 'http' + + # ats 1.5 properties + entity_groupfs_active_dir = config['configurations']['yarn-site']['yarn.timeline-service.entity-group-fs-store.active-dir'] + entity_groupfs_active_dir_mode = 0o1777 + entity_groupfs_store_dir = config['configurations']['yarn-site']['yarn.timeline-service.entity-group-fs-store.done-dir'] + entity_groupfs_store_dir_mode = 0o700 + is_webhdfs_enabled = hdfs_site['dfs.webhdfs.enabled'] + + if len(livy_livyserver_hosts) > 0: + has_livyserver = True + if security_enabled: + livy_principal = livy_kerberos_principal.replace('_HOST', config['agentLevelParams']['hostname'].lower()) + + livy_livyserver_port = default('configurations/livy-conf/livy.server.port',8999) import functools #create partial functions with common arguments for every HdfsResource call @@ -260,5 +311,4 @@ HdfsResource = functools.partial( default_fs = default_fs, immutable_paths = get_not_managed_resources(), dfs_type = dfs_type -) - +) \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/service_check.py new file mode 100644 index 0000000000..cfadd034e8 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/service_check.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agree in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import subprocess +import time +import os + +from resource_management.core.exceptions import Fail +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.format import format +from resource_management.core.resources.system import Execute +from resource_management.core.logger import Logger + +CHECK_COMMAND_TIMEOUT_DEFAULT = 60.0 + +class LivyServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + if params.has_livyserver: + live_livyserver_host = "" + for livyserver_host in params.livy_livyserver_hosts: + try: + Execute(format("curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k {livy_http_scheme}://{livyserver_host}:{livy_livyserver_port}/sessions | grep 200"), + tries=3, + try_sleep=1, + logoutput=True, + user=params.smoke_user + ) + live_livyserver_host = livyserver_host + break + except: + pass + if len(params.livy_livyserver_hosts) > 0 and live_livyserver_host == "": + raise Fail("Connection to all Livy servers failed") + + +if __name__ == "__main__": + LivyServiceCheck().execute() + diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/setup_livy.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/setup_livy.py new file mode 100644 index 0000000000..b169cdd6a6 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/setup_livy.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.generate_logfeeder_input_config import generate_logfeeder_input_config +from resource_management import Directory, File, PropertiesFile, Template, InlineTemplate, format + + +def setup_livy(env, type, upgrade_type = None, action = None): + import params + + Directory([params.livy_pid_dir, params.livy_log_dir], + owner=params.livy_user, + group=params.user_group, + mode=0o775, + cd_access='a', + create_parents=True + ) + if type == 'server' and action == 'config': + params.HdfsResource(params.livy_hdfs_user_dir, + type="directory", + action="create_on_execute", + owner=params.livy_user, + mode=0o775 + ) + params.HdfsResource(None, action="execute") + + if params.livy_recovery_store == 'filesystem': + params.HdfsResource(params.livy_recovery_dir, + type="directory", + action="create_on_execute", + owner=params.livy_user, + mode=0o700 + ) + params.HdfsResource(None, action="execute") + + generate_logfeeder_input_config('livy', Template("input.config-livy.json.j2", extra_imports=[default])) + + # create livy-env.sh in etc/conf dir + File(os.path.join(params.livy_conf, 'livy-env.sh'), + owner=params.livy_user, + group=params.livy_group, + content=InlineTemplate(params.livy_env_sh), + mode=0o644, + ) + + # create livy-client.conf in etc/conf dir + PropertiesFile(format("{livy_conf}/livy-client.conf"), + properties = params.config['configurations']['livy-client-conf'], + key_value_delimiter = " ", + owner=params.livy_user, + group=params.livy_group, + ) + + # create livy.conf in etc/conf dir + PropertiesFile(format("{livy_conf}/livy.conf"), + properties = params.config['configurations']['livy-conf'], + key_value_delimiter = " ", + owner=params.livy_user, + group=params.livy_group, + ) + + # create log4j.properties in etc/conf dir + File(os.path.join(params.livy_conf, 'log4j.properties'), + owner=params.livy_user, + group=params.livy_group, + content=params.livy_log4j_properties, + mode=0o644, + ) + + # create spark-blacklist.properties in etc/conf dir + File(os.path.join(params.livy_conf, 'spark-blacklist.conf'), + owner=params.livy_user, + group=params.livy_group, + content=params.livy_spark_blacklist_properties, + mode=0o644, + ) + + Directory(params.livy_logs_dir, + owner=params.livy_user, + group=params.livy_group, + mode=0o755, + ) + diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/status_params.py new file mode 100644 index 0000000000..da308ac05d --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/status_params.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.functions.format import format +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.default import default + +config = Script.get_config() + +spark_user = config['configurations']['spark-env']['spark_user'] +spark_group = config['configurations']['spark-env']['spark_group'] +user_group = config['configurations']['cluster-env']['user_group'] + +if 'hive-env' in config['configurations']: + hive_user = config['configurations']['hive-env']['hive_user'] +else: + hive_user = "hive" + +spark_pid_dir = config['configurations']['spark-env']['spark_pid_dir'] +spark_history_server_pid_file = format("{spark_pid_dir}/spark-{spark_user}-org.apache.spark.deploy.history.HistoryServer-1.pid") +spark_thrift_server_pid_file = format("{spark_pid_dir}/spark-{spark_user}-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid") +stack_name = default("/clusterLevelParams/stack_name", None) + +if "livy-env" in config['configurations']: + livy_user = config['configurations']['livy-env']['livy_user'] + livy_group = config['configurations']['livy-env']['livy_group'] + livy_pid_dir = config['configurations']['livy-env']['livy_pid_dir'] + livy_server_pid_file = format("{livy_pid_dir}/livy-{livy_user}-server.pid") \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/templates/input.config-livy.json.j2 similarity index 53% copy from ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2 copy to ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/templates/input.config-livy.json.j2 index 089897b978..96df20410e 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2 +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/templates/input.config-livy.json.j2 @@ -18,14 +18,9 @@ { "input":[ { - "type":"spark_jobhistory_server", + "type":"livy_server", "rowtype":"service", - "path":"{{default('/configurations/spark-env/spark_log_dir', '/var/log/spark')}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out" - }, - { - "type":"spark_thriftserver", - "rowtype":"service", - "path":"{{default('/configurations/spark-env/spark_log_dir', '/var/log/spark')}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out" + "path":"{{default('/configurations/livy-env/livy_log_dir', '/var/log/livy')}}/livy-{{default('/configurations/livy-env/livy_user', 'livy')}}-server.out" } ], "filter":[ @@ -34,25 +29,17 @@ "conditions":{ "fields":{ "type":[ - "spark_jobhistory_server", - "spark_thriftserver", - "livy2_server" + "livy_server" ] } }, - "log4j_format":"", - "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})", - "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}", + "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\((%{JAVAFILE:file})?:(%{JAVAMETHOD:method})?\\((%{INT:line_number})?\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", "post_map_values":{ "logtime":{ "map_date":{ - "target_date_pattern":"yy/MM/dd HH:mm:ss" - } - }, - "level":{ - "map_field_value":{ - "pre_value":"WARNING", - "post_value":"WARN" + "target_date_pattern":"yyyy-MM-dd'T'HH:mm:ss,SSS" } } } diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/role_command_order.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/role_command_order.json new file mode 100644 index 0000000000..55a24334a3 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/role_command_order.json @@ -0,0 +1,8 @@ +{ + "general_deps" : { + "_comment" : "dependencies for LIVY", + "LIVY_SERVER-START" : ["SPARK_JOBHISTORYSERVER-START"], + "LIVY_SERVICE_CHECK-SERVICE_CHECK" : ["LIVY_SERVER-START"] + } +} + diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/themes/directories.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/themes/directories.json new file mode 100644 index 0000000000..08eec0ec3f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/themes/directories.json @@ -0,0 +1,89 @@ +{ + "name": "directories", + "description": "Directories theme for LIVY service", + "configuration": { + "layouts": [ + { + "name": "directories", + "tabs": [ + { + "name": "directories", + "display-name": "Directories", + "layout": { + "tab-columns": "1", + "tab-rows": "4", + "sections": [ + { + "name": "subsection-log-dirs", + "display-name": "LOG DIRS", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-log-dirs", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "subsection-pid-dirs", + "display-name": "PID DIRS", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-pid-dirs", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + } + ] + } + ], + "placement": { + "configuration-layout": "default", + "configs": [ + { + "config": "livy-env/livy_log_dir", + "subsection-name": "subsection-log-dirs" + }, + { + "config": "livy-env/livy_pid_dir", + "subsection-name": "subsection-pid-dirs" + } + ] + }, + "widgets": [ + { + "config": "livy-env/livy_log_dir", + "widget": { + "type": "text-field" + } + }, + { + "config": "livy-env/livy_pid_dir", + "widget": { + "type": "text-field" + } + } + ] + } + } + \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py index 222810f5b3..655f8c5142 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py @@ -206,16 +206,9 @@ elif spark_transport_mode.lower() == 'http': spark_thrift_endpoint = default("configurations/spark-hive-site-override/hive.server2.http.endpoint", "cliservice") # thrift server support -spark_thrift_sparkconf = None -spark_thrift_cmd_opts_properties = '' -spark_thrift_fairscheduler_content = None - -if has_spark_thriftserver and 'spark-defaults' in config['configurations']: - spark_thrift_sparkconf = config['configurations']['spark-defaults'] - spark_thrift_cmd_opts_properties = config['configurations']['spark-env']['spark_thrift_cmd_opts'] - - if 'spark-thrift-fairscheduler' in config['configurations'] and 'fairscheduler_content' in config['configurations']['spark-thrift-fairscheduler']: - spark_thrift_fairscheduler_content = config['configurations']['spark-thrift-fairscheduler']['fairscheduler_content'] +spark_thrift_sparkconf = config['configurations']['spark-defaults'] +spark_thrift_cmd_opts_properties = config['configurations']['spark-env']['spark_thrift_cmd_opts'] +spark_thrift_fairscheduler_content = config['configurations']['spark-thrift-fairscheduler']['fairscheduler_content'] if is_hive_installed: # update default metastore client properties (async wait for metastore component) it is useful in case of diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/setup_spark.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/setup_spark.py index 7d205be57f..3097f122b9 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/setup_spark.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/setup_spark.py @@ -141,14 +141,13 @@ def setup_spark(env, type, upgrade_type = None, action = None): group=params.spark_group, mode=0o644) - if params.spark_thrift_fairscheduler_content: - # create spark-thrift-fairscheduler.xml - File(os.path.join(params.spark_conf_dir,"spark-thrift-fairscheduler.xml"), - owner=params.spark_user, - group=params.spark_group, - mode=0o755, - content=InlineTemplate(params.spark_thrift_fairscheduler_content) - ) + # create spark-thrift-fairscheduler.xml + File(os.path.join(params.spark_conf_dir,"spark-thrift-fairscheduler.xml"), + owner=params.spark_user, + group=params.spark_group, + mode=0o755, + content=InlineTemplate(params.spark_thrift_fairscheduler_content) + ) if type == "client": Logger.info('Spark client config.') diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2 index 089897b978..06a280b7a6 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2 +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2 @@ -35,8 +35,7 @@ "fields":{ "type":[ "spark_jobhistory_server", - "spark_thriftserver", - "livy2_server" + "spark_thriftserver" ] } }, diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.3.0/services/LIVY/metainfo.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.3.0/services/LIVY/metainfo.xml new file mode 100644 index 0000000000..61f32e92f9 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.3.0/services/LIVY/metainfo.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<!--Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>LIVY</name> + <version>0.8.0-1</version> + </service> + </services> +</metainfo> \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@ambari.apache.org For additional commands, e-mail: commits-h...@ambari.apache.org