This is an automated email from the ASF dual-hosted git repository.
jialiang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9743c79440 AMBARI-26090: Add Livy as Individual Service to Ambari
Bigtop Stack (#3790)
9743c79440 is described below
commit 9743c794407deff76c5a657d8fb6331d0def8fa7
Author: William Horn <[email protected]>
AuthorDate: Sat Jul 27 00:20:40 2024 -0400
AMBARI-26090: Add Livy as Individual Service to Ambari Bigtop Stack (#3790)
* feat: Add Livy service to Ambari Bigtop stack (#1)
feat: Add Livy service to Ambari Bigtop stack
Co-authored-by: Carmen Blaine <[email protected]>
* Remove Hive as a dependency from Livy
* removed SPARK STS as start up dependency for Livy
---------
Co-authored-by: Paul Yakovlev <[email protected]>
Co-authored-by: Carmen Blaine <[email protected]>
Co-authored-by: carmenblaine3
<[email protected]>
---
.../BIGTOP/3.2.0/properties/stack_packages.json | 26 ++++
.../stacks/BIGTOP/3.2.0/services/LIVY/alerts.json | 30 ++++
.../LIVY/configuration/livy-client-conf.xml | 30 ++++
.../services/LIVY/configuration/livy-conf.xml | 129 +++++++++++++++++
.../3.2.0/services/LIVY/configuration/livy-env.xml | 108 +++++++++++++++
.../LIVY/configuration/livy-log4j-properties.xml | 41 ++++++
.../LIVY/configuration/livy-spark-blacklist.xml | 52 +++++++
.../BIGTOP/3.2.0/services/LIVY/kerberos.json | 52 +++++++
.../stacks/BIGTOP/3.2.0/services/LIVY/metainfo.xml | 124 +++++++++++++++++
.../LIVY/package/scripts/alerts/alert_livy_port.py | 154 +++++++++++++++++++++
.../services/LIVY/package/scripts/livy_server.py | 149 ++++++++++++++++++++
.../services/LIVY/package/scripts/livy_service.py | 45 ++++++
.../{SPARK => LIVY}/package/scripts/params.py | 60 +++++++-
.../services/LIVY/package/scripts/service_check.py | 56 ++++++++
.../services/LIVY/package/scripts/setup_livy.py | 102 ++++++++++++++
.../services/LIVY/package/scripts/status_params.py | 45 ++++++
.../package/templates/input.config-livy.json.j2} | 27 +---
.../3.2.0/services/LIVY/role_command_order.json | 8 ++
.../3.2.0/services/LIVY/themes/directories.json | 89 ++++++++++++
.../3.2.0/services/SPARK/package/scripts/params.py | 13 +-
.../services/SPARK/package/scripts/setup_spark.py | 15 +-
.../package/templates/input.config-spark.json.j2 | 3 +-
.../stacks/BIGTOP/3.3.0/services/LIVY/metainfo.xml | 27 ++++
23 files changed, 1340 insertions(+), 45 deletions(-)
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
index d3fc8d65c4..a050d4c5ef 100644
---
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
@@ -271,6 +271,32 @@
]
}
},
+ "LIVY": {
+ "LIVY_CLIENT": {
+ "STACK-SELECT-PACKAGE": "livy-client",
+ "INSTALL": [
+ "livy-client"
+ ],
+ "PATCH": [
+ "livy-client"
+ ],
+ "STANDARD": [
+ "livy-client"
+ ]
+ },
+ "LIVY_SERVER": {
+ "STACK-SELECT-PACKAGE": "livy-server",
+ "INSTALL": [
+ "livy-server"
+ ],
+ "PATCH": [
+ "livy-server"
+ ],
+ "STANDARD": [
+ "livy-server"
+ ]
+ }
+ },
"FLINK": {
"FLINK_CLIENT": {
"STACK-SELECT-PACKAGE": "flink-client",
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/alerts.json
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/alerts.json
new file mode 100644
index 0000000000..2e753dcd30
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/alerts.json
@@ -0,0 +1,30 @@
+{
+ "SPARK": {
+ "service": [],
+ "LIVY_SERVER": [
+ {
+ "name": "livy_server_status",
+ "label": "Spark Livy Server",
+ "description": "This host-level alert is triggered if the Livy2
Server cannot be determined to be up.",
+ "interval": 1,
+ "scope": "ANY",
+ "source": {
+ "type": "SCRIPT",
+ "path":
"BIGTOP/3.2.0/services/LIVY/package/scripts/alerts/alert_livy_port.py",
+ "parameters": [
+ {
+ "name": "check.command.timeout",
+ "display_name": "Command Timeout",
+ "value": 60.0,
+ "type": "NUMERIC",
+ "description": "The maximum time before check command will be
killed by timeout",
+ "units": "seconds",
+ "threshold": "CRITICAL"
+ }
+ ]
+ }
+ }
+ ]
+ }
+ }
+
\ No newline at end of file
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-client-conf.xml
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-client-conf.xml
new file mode 100644
index 0000000000..d761f412a8
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-client-conf.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+ <property>
+ <name>livy.rsc.launcher.address</name>
+ <value> </value>
+ <description>
+ Address for the RSC driver to connect back with it's connection
info.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-conf.xml
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-conf.xml
new file mode 100644
index 0000000000..a0521c360f
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-conf.xml
@@ -0,0 +1,129 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+ <property>
+ <name>livy.environment</name>
+ <value>production</value>
+ <description>
+ Specifies Livy's environment. May either be "production" or
"development". In "development"
+ mode, Livy will enable debugging options, such as reporting
possible routes on a 404.
+ defaults to development
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.server.port</name>
+ <value>8999</value>
+ <description>
+ What port to start the server on. Defaults to 8999.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.server.session.timeout</name>
+ <value>3600000</value>
+ <description>
+ Time in milliseconds on how long Livy will wait before timing out
an idle session.
+ Default is one hour.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.impersonation.enabled</name>
+ <value>true</value>
+ <description>
+ If livy should use proxy users when submitting a job.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.impersonation.enabled</name>
+ <value>true</value>
+ <description>
+ If livy should use proxy users when submitting a job.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.server.csrf_protection.enabled</name>
+ <value>true</value>
+ <description>
+ Whether to enable csrf protection for livy's rest api.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.spark.master</name>
+ <value>yarn</value>
+ <description>
+ spark.master property for spark engine
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.spark.deploy-mode</name>
+ <value>cluster</value>
+ <description>
+ spark.deploy-mode property for spark engine
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.repl.enableHiveContext</name>
+ <value>true</value>
+ <description>
+ Whether to enable HiveContext in livy interpreter
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.server.recovery.mode</name>
+ <value>recovery</value>
+ <description>
+ Recovery mode for livy, either be "off" or "recovery".
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.server.recovery.state-store</name>
+ <value>filesystem</value>
+ <description>
+ Where Livy should store state for recovery.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.server.recovery.state-store.url</name>
+ <value>/livy-recovery</value>
+ <description>
+ Where Livy should store state for recovery.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.server.access-control.enabled</name>
+ <value>true</value>
+ <description>
+ Property to configure Livy user access.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-env.xml
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-env.xml
new file mode 100644
index 0000000000..0b27bed3ff
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-env.xml
@@ -0,0 +1,108 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+ <property>
+ <name>livy_user</name>
+ <display-name>Livy User</display-name>
+ <value>livy</value>
+ <property-type>USER</property-type>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ <user-groups>
+ <property>
+ <type>cluster-env</type>
+ <name>user_group</name>
+ </property>
+ <property>
+ <type>livy-env</type>
+ <name>livy_group</name>
+ </property>
+ </user-groups>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy_group</name>
+ <display-name>Livy Group</display-name>
+ <value>livy</value>
+ <property-type>GROUP</property-type>
+ <description>livy group</description>
+ <value-attributes>
+ <type>user</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy_log_dir</name>
+ <display-name>Livy Log directory</display-name>
+ <value>/var/log/livy</value>
+ <description>Livy Log Dir</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy_pid_dir</name>
+ <display-name>Livy PID directory</display-name>
+ <value>/var/run/livy</value>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>spark_home</name>
+ <value>/usr/bigtop/current/spark-client</value>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <!-- livy-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for livy-env.sh
file</description>
+ <value>
+ #!/usr/bin/env bash
+
+ # - SPARK_HOME Spark which you would like to use in livy
+ # - SPARK_CONF_DIR Directory containing the Spark configuration
to use.
+ # - HADOOP_CONF_DIR Directory containing the Hadoop / YARN
configuration to use.
+ # - LIVY_LOG_DIR Where log files are stored. (Default:
${LIVY_HOME}/logs)
+ # - LIVY_PID_DIR Where the pid file is stored. (Default: /tmp)
+ # - LIVY_SERVER_JAVA_OPTS Java Opts for running livy server (You
can set jvm related setting here, like jvm memory/gc algorithm and etc.)
+ export SPARK_HOME=/usr/bigtop/current/spark-client
+ export SPARK_CONF_DIR=/etc/spark/conf
+ export JAVA_HOME={{java_home}}
+ export HADOOP_CONF_DIR=/etc/hadoop/conf
+ export LIVY_LOG_DIR={{livy_log_dir}}
+ export LIVY_PID_DIR={{livy_pid_dir}}
+ export LIVY_SERVER_JAVA_OPTS="-Xmx2g"
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-log4j-properties.xml
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-log4j-properties.xml
new file mode 100644
index 0000000000..fc15c7f6b9
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-log4j-properties.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+ <property>
+ <name>content</name>
+ <description>Livy-log4j-Properties</description>
+ <value>
+ # Set everything to be logged to the console
+ log4j.rootCategory=INFO, console
+ log4j.appender.console=org.apache.log4j.ConsoleAppender
+ log4j.appender.console.target=System.err
+ log4j.appender.console.layout=org.apache.log4j.PatternLayout
+ log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd
HH:mm:ss} %p %c{1}: %m%n
+
+ log4j.logger.org.eclipse.jetty=WARN
+ </value>
+ <value-attributes>
+ <type>content</type>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-spark-blacklist.xml
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-spark-blacklist.xml
new file mode 100644
index 0000000000..84c899c1e0
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/configuration/livy-spark-blacklist.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+ <property>
+ <name>content</name>
+ <description>spark-blacklist.properties</description>
+ <value>
+ #
+ # Configuration override / blacklist. Defines a list of properties
that users are not allowed
+ # to override when starting Spark sessions.
+ #
+ # This file takes a list of property names (one per line). Empty
lines and lines starting with "#"
+ # are ignored.
+ #
+
+ # Disallow overriding the master and the deploy mode.
+ spark.master
+ spark.submit.deployMode
+
+ # Disallow overriding the location of Spark cached jars.
+ spark.yarn.jar
+ spark.yarn.jars
+ spark.yarn.archive
+
+ # Don't allow users to override the RSC timeout.
+ livy.rsc.server.idle_timeout
+ </value>
+ <value-attributes>
+ <type>content</type>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/kerberos.json
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/kerberos.json
new file mode 100644
index 0000000000..5a6ca1590c
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/kerberos.json
@@ -0,0 +1,52 @@
+{
+ "services": [
+ {
+ "name": "LIVY_SERVER",
+ "identities": [
+ {
+ "name": "hdfs",
+ "reference": "/HDFS/NAMENODE/hdfs"
+ },
+ {
+ "name": "livyuser",
+ "principal": {
+ "value": "${livy-env/livy_user}/_HOST@${realm}",
+ "type": "service",
+ "configuration":
"livy-conf/livy.server.launch.kerberos.principal",
+ "local_username": "${livy-env/livy_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/livy.service.keytab",
+ "owner": {
+ "name": "${livy-env/livy_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "livy-conf/livy.server.launch.kerberos.keytab"
+ }
+ },
+ {
+ "name": "livy_spnego",
+ "reference": "/spnego",
+ "principal": {
+ "configuration": "livy-conf/livy.server.auth.kerberos.principal"
+ },
+ "keytab": {
+ "configuration": "livy-conf/livy.server.auth.kerberos.keytab"
+ }
+ }
+ ],
+ "configurations": [
+ {
+ "livy-conf": {
+ "livy.server.auth.type": "kerberos",
+ "livy.impersonation.enabled": "true"
+ }
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/metainfo.xml
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/metainfo.xml
new file mode 100644
index 0000000000..fece912b2c
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/metainfo.xml
@@ -0,0 +1,124 @@
+<?xml version="1.0"?>
+<!--Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>LIVY</name>
+ <displayName>Livy</displayName>
+ <comment>Apache Livy is a service that enables easy interaction with a
Spark cluster over a REST interface. </comment>
+ <version>0.7.1-1</version>
+ <components>
+ <component>
+ <name>LIVY_SERVER</name>
+ <displayName>Livy Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>SPARK/SPARK_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>YARN/YARN_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/livy_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ <logs>
+ <log>
+ <logId>livy_server</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
+ </component>
+ </components>
+
+ <configuration-dependencies>
+ <config-type>core-site</config-type>
+ <config-type>spark-defaults</config-type>
+ <config-type>spark-env</config-type>
+ <config-type>spark-log4j-properties</config-type>
+ <config-type>spark-metrics-properties</config-type>
+ <config-type>spark-hive-site-override</config-type>
+ <config-type>spark-thrift-fairscheduler</config-type>
+ <config-type>livy-client-conf</config-type>
+ <config-type>livy-conf</config-type>
+ <config-type>livy-env</config-type>
+ <config-type>livy-log4j-properties</config-type>
+ <config-type>livy-spark-blacklist</config-type>
+ </configuration-dependencies>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>HDFS</service>
+ <service>YARN</service>
+ <service>SPARK</service>
+ </requiredServices>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>redhat7</osFamily>
+ <packages>
+ <package>
+ <name>spark_${stack_version}-core</name>
+ </package>
+ <package>
+ <name>spark_${stack_version}-python</name>
+ </package>
+ <package>
+ <name>livy_${stack_version}</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <themes>
+ <theme>
+ <fileName>directories.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+
+ </service>
+ </services>
+</metainfo>
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/alerts/alert_livy_port.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/alerts/alert_livy_port.py
new file mode 100644
index 0000000000..50d3791a20
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/alerts/alert_livy_port.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import time
+import logging
+import traceback
+import socket
+from resource_management import *
+from resource_management.libraries.functions import format
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from resource_management.libraries.script.script import Script
+from resource_management.core.resources import Execute
+from resource_management.core.logger import Logger
+from resource_management.core import global_lock
+from resource_management.libraries.functions import get_kinit_path
+
+
+OK_MESSAGE = "TCP OK - {0:.3f}s response on port {1}"
+CRITICAL_MESSAGE = "Connection failed on host {0}:{1} ({2})"
+
+logger = logging.getLogger('ambari_alerts')
+
+LIVY_SERVER_HOST_KEY = '{{livy-conf/livy.server.host}}'
+
+LIVY_SERVER_PORT_KEY = '{{livy-conf/livy.server.port}}'
+
+LIVYUSER_DEFAULT = 'livy'
+
+CHECK_COMMAND_TIMEOUT_KEY = 'check.command.timeout'
+CHECK_COMMAND_TIMEOUT_DEFAULT = 60.0
+
+SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}'
+SMOKEUSER_KEYTAB_KEY = '{{cluster-env/smokeuser_keytab}}'
+SMOKEUSER_PRINCIPAL_KEY = '{{cluster-env/smokeuser_principal_name}}'
+SMOKEUSER_KEY = '{{cluster-env/smokeuser}}'
+LIVY_SSL_ENABLED_KEY = '{{livy-conf/livy.keystore}}'
+
+# The configured Kerberos executable search paths, if any
+KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY =
'{{kerberos-env/executable_search_paths}}'
+
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def get_tokens():
+ """
+ Returns a tuple of tokens in the format {{site/property}} that will be used
+ to build the dictionary passed into execute
+ """
+ return
(LIVY_SERVER_HOST_KEY,LIVY_SERVER_PORT_KEY,LIVYUSER_DEFAULT,SECURITY_ENABLED_KEY,SMOKEUSER_KEYTAB_KEY,SMOKEUSER_PRINCIPAL_KEY,SMOKEUSER_KEY,LIVY_SSL_ENABLED_KEY)
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def execute(configurations={}, parameters={}, host_name=None):
+ """
+ Returns a tuple containing the result code and a pre-formatted result label
+
+ Keyword arguments:
+ configurations (dictionary): a mapping of configuration key to value
+ parameters (dictionary): a mapping of script parameter key to value
+ host_name (string): the name of this host where the alert is running
+ """
+
+ if configurations is None:
+ return ('UNKNOWN', ['There were no configurations supplied to the
script.'])
+
+ LIVY_PORT_DEFAULT = 8999
+
+ port = LIVY_PORT_DEFAULT
+ if LIVY_SERVER_PORT_KEY in configurations:
+ port = int(configurations[LIVY_SERVER_PORT_KEY])
+
+ if LIVY_SERVER_HOST_KEY in configurations:
+ host_name = str(configurations[LIVY_SERVER_HOST_KEY])
+
+ if host_name is None:
+ host_name = socket.getfqdn()
+
+ livyuser = configurations[SMOKEUSER_KEY]
+
+ security_enabled = False
+ if SECURITY_ENABLED_KEY in configurations:
+ security_enabled = str(configurations[SECURITY_ENABLED_KEY]).upper()
== 'TRUE'
+
+ smokeuser_kerberos_keytab = None
+ if SMOKEUSER_KEYTAB_KEY in configurations:
+ smokeuser_kerberos_keytab = configurations[SMOKEUSER_KEYTAB_KEY]
+
+ if host_name is None:
+ host_name = socket.getfqdn()
+
+ smokeuser_principal = None
+ if SMOKEUSER_PRINCIPAL_KEY in configurations:
+ smokeuser_principal = configurations[SMOKEUSER_PRINCIPAL_KEY]
+ smokeuser_principal =
smokeuser_principal.replace('_HOST',host_name.lower())
+
+ # Get the configured Kerberos executable search paths, if any
+ if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
+ kerberos_executable_search_paths =
configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
+ else:
+ kerberos_executable_search_paths = None
+
+ kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
+
+ if security_enabled:
+ kinitcmd = format("{kinit_path_local} -kt {smokeuser_kerberos_keytab}
{smokeuser_principal}; ")
+ # prevent concurrent kinit
+ kinit_lock = global_lock.get_lock(global_lock.LOCK_TYPE_KERBEROS)
+ kinit_lock.acquire()
+ try:
+ Execute(kinitcmd, user=livyuser)
+ finally:
+ kinit_lock.release()
+
+ http_scheme = 'https' if LIVY_SSL_ENABLED_KEY in configurations else 'http'
+ result_code = None
+ try:
+ start_time = time.time()
+ try:
+ livy_livyserver_host = str(host_name)
+
+ livy_cmd = format("curl -s -o /dev/null -w'%{{http_code}}'
--negotiate -u: -k {http_scheme}://{livy_livyserver_host}:{port}/sessions |
grep 200 ")
+
+ Execute(livy_cmd,
+ tries=3,
+ try_sleep=1,
+ logoutput=True,
+ user=livyuser
+ )
+
+ total_time = time.time() - start_time
+ result_code = 'OK'
+ label = OK_MESSAGE.format(total_time, port)
+ except:
+ result_code = 'CRITICAL'
+ label = CRITICAL_MESSAGE.format(host_name, port,
traceback.format_exc())
+ except:
+ label = traceback.format_exc()
+ result_code = 'UNKNOWN'
+
+ return (result_code, [label])
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/livy_server.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/livy_server.py
new file mode 100644
index 0000000000..a123a5bf66
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/livy_server.py
@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.check_process_status import
check_process_status
+from resource_management.libraries.functions.stack_features import
check_stack_feature
+from resource_management.libraries.functions.constants import StackFeature
+from resource_management.core.exceptions import Fail
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil
+from resource_management.libraries.providers.hdfs_resource import
HdfsResourceProvider
+from resource_management import is_empty
+from resource_management import shell
+from resource_management.libraries.functions.decorator import retry
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import namenode_ha_utils
+
+from livy_service import livy_service
+from setup_livy import setup_livy
+
+class LivyServer(Script):
+
+ def install(self, env):
+ import params
+ env.set_params(params)
+
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ import params
+ env.set_params(params)
+
+ setup_livy(env, 'server', upgrade_type=upgrade_type, action = 'config')
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ if params.has_ats and params.has_livyserver:
+ Logger.info("Verifying DFS directories where ATS stores time line data
for active and completed applications.")
+ self.wait_for_dfs_directories_created([params.entity_groupfs_store_dir,
params.entity_groupfs_active_dir])
+
+ self.configure(env)
+ livy_service('server', upgrade_type=upgrade_type, action='start')
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ livy_service('server', upgrade_type=upgrade_type, action='stop')
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+
+ check_process_status(status_params.livy_server_pid_file)
+
+ # TODO move out and compose with similar method in resourcemanager.py
+ def wait_for_dfs_directories_created(self, dirs):
+ import params
+
+ ignored_dfs_dirs =
HdfsResourceProvider.get_ignored_resources_list(params.hdfs_resource_ignore_file)
+
+ if params.security_enabled:
+ Execute(format("{kinit_path_local} -kt {livy_kerberos_keytab}
{livy_principal}"),
+ user=params.livy_user
+ )
+ Execute(format("{kinit_path_local} -kt {hdfs_user_keytab}
{hdfs_principal_name}"),
+ user=params.hdfs_user
+ )
+
+ for dir_path in dirs:
+ self.wait_for_dfs_directory_created(dir_path, ignored_dfs_dirs)
+
+ def get_pid_files(self):
+ import status_params
+ return [status_params.livy_server_pid_file]
+
+
+ @retry(times=8, sleep_time=20, backoff_factor=1, err_class=Fail)
+ def wait_for_dfs_directory_created(self, dir_path, ignored_dfs_dirs):
+ import params
+
+ if not is_empty(dir_path):
+ dir_path = HdfsResourceProvider.parse_path(dir_path)
+
+ if dir_path in ignored_dfs_dirs:
+ Logger.info("Skipping DFS directory '" + dir_path + "' as it's marked
to be ignored.")
+ return
+
+ Logger.info("Verifying if DFS directory '" + dir_path + "' exists.")
+
+ dir_exists = None
+
+ nameservices = namenode_ha_utils.get_nameservices(params.hdfs_site)
+ nameservice = None if not nameservices else nameservices[-1]
+
+ if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled,
params.dfs_type):
+ # check with webhdfs is much faster than executing hdfs dfs -test
+ util = WebHDFSUtil(params.hdfs_site, nameservice, params.hdfs_user,
params.security_enabled)
+ list_status = util.run_command(dir_path, 'GETFILESTATUS',
method='GET', ignore_status_codes=['404'], assertable_result=False)
+ dir_exists = ('FileStatus' in list_status)
+ else:
+ # have to do time expensive hdfs dfs -d check.
+ dfs_ret_code = shell.call(format("hdfs --config {hadoop_conf_dir} dfs
-test -d " + dir_path), user=params.livy_user)[0]
+ dir_exists = not dfs_ret_code #dfs -test -d returns 0 in case the dir
exists
+
+ if not dir_exists:
+ raise Fail("DFS directory '" + dir_path + "' does not exist !")
+ else:
+ Logger.info("DFS directory '" + dir_path + "' exists.")
+
+ def pre_upgrade_restart(self, env, upgrade_type=None):
+ import params
+
+ env.set_params(params)
+ if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE,
params.version):
+ Logger.info("Executing Livy Server Stack Upgrade pre-restart")
+ stack_select.select_packages(params.version)
+
+ def get_log_folder(self):
+ import params
+ return params.livy_log_dir
+
+ def get_user(self):
+ import params
+ return params.livy_user
+if __name__ == "__main__":
+ LivyServer().execute()
+
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/livy_service.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/livy_service.py
new file mode 100644
index 0000000000..55aa1b2902
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/livy_service.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.libraries.functions import format
+from resource_management.core.resources.system import File, Execute
+from resource_management.libraries.functions import get_user_call_output
+
+
+def livy_service(name, upgrade_type=None, action=None):
+ import params
+
+ # use the livy user to get the PID (it is protected on non-root systems)
+ livy_server_pid = get_user_call_output.get_user_call_output(format("cat
{livy_server_pid_file}"),
+
user=params.livy_user, is_checked_call=False)[1]
+ livy_server_pid = livy_server_pid.replace("\n", " ")
+ process_id_exists_command = format("ls {livy_server_pid_file} >/dev/null
2>&1 && ps -p {livy_server_pid} >/dev/null 2>&1")
+
+ if action == 'start':
+ Execute(format('{livy_server_start}'),
+ user=params.livy_user,
+ environment={'JAVA_HOME': params.java_home},
+ not_if=process_id_exists_command)
+ elif action == 'stop':
+ Execute(format('{livy_server_stop}'),
+ user=params.livy_user,
+ only_if=process_id_exists_command,
+ timeout=10,
+ on_timeout=format("! ( {process_id_exists_command} ) || {sudo} -H
-E kill -9 {livy_server_pid}"),
+ environment={'JAVA_HOME': params.java_home})
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/params.py
similarity index 81%
copy from
ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py
copy to
ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/params.py
index 222810f5b3..a38ef7b786 100644
---
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/params.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -45,7 +45,9 @@ sudo = AMBARI_SUDO_BINARY
SERVER_ROLE_DIRECTORY_MAP = {
'SPARK_JOBHISTORYSERVER' : 'spark-historyserver',
'SPARK_CLIENT' : 'spark-client',
- 'SPARK_THRIFTSERVER' : 'spark-thriftserver'
+ 'SPARK_THRIFTSERVER' : 'spark-thriftserver',
+ 'LIVY_SERVER' : 'livy-server',
+ 'LIVY_CLIENT' : 'livy-client'
}
HIVE_SERVER_ROLE_DIRECTORY_MAP = {
@@ -241,7 +243,56 @@ has_ats = len(ats_host) > 0
dfs_type = default("/clusterLevelParams/dfs_type", "")
-
+# livy configs
+has_livyserver = False
+
+if stack_version_formatted and "livy-env" in config['configurations']:
+ livy_component_directory =
Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "LIVY_SERVER")
+ livy_conf = format("{stack_root}/current/{livy_component_directory}/conf")
+ livy_log_dir = config['configurations']['livy-env']['livy_log_dir']
+ livy_pid_dir = status_params.livy_pid_dir
+ livy_home = format("{stack_root}/current/{livy_component_directory}")
+ livy_user = status_params.livy_user
+ livy_group = status_params.livy_group
+ user_group = status_params.user_group
+ livy_hdfs_user_dir = format("/user/{livy_user}")
+ livy_server_pid_file = status_params.livy_server_pid_file
+ livy_recovery_dir =
default("/configurations/livy-conf/livy.server.recovery.state-store.url",
"/livy-recovery")
+ livy_recovery_store =
default("/configurations/livy-conf/livy.server.recovery.state-store",
"filesystem")
+
+ livy_server_start = format("{livy_home}/bin/livy-server start")
+ livy_server_stop = format("{livy_home}/bin/livy-server stop")
+ livy_logs_dir = format("{livy_home}/logs")
+
+ livy_env_sh = config['configurations']['livy-env']['content']
+ livy_log4j_properties =
config['configurations']['livy-log4j-properties']['content']
+ livy_spark_blacklist_properties =
config['configurations']['livy-spark-blacklist']['content']
+
+ if 'livy.server.kerberos.keytab' in config['configurations']['livy-conf']:
+ livy_kerberos_keytab =
config['configurations']['livy-conf']['livy.server.kerberos.keytab']
+ else:
+ livy_kerberos_keytab =
config['configurations']['livy-conf']['livy.server.launch.kerberos.keytab']
+ if 'livy.server.kerberos.principal' in config['configurations']['livy-conf']:
+ livy_kerberos_principal =
config['configurations']['livy-conf']['livy.server.kerberos.principal']
+ else:
+ livy_kerberos_principal =
config['configurations']['livy-conf']['livy.server.launch.kerberos.principal']
+
+ livy_livyserver_hosts = default("/clusterHostInfo/livy_server_hosts", [])
+ livy_http_scheme = 'https' if 'livy.keystore' in
config['configurations']['livy-conf'] else 'http'
+
+ # ats 1.5 properties
+ entity_groupfs_active_dir =
config['configurations']['yarn-site']['yarn.timeline-service.entity-group-fs-store.active-dir']
+ entity_groupfs_active_dir_mode = 0o1777
+ entity_groupfs_store_dir =
config['configurations']['yarn-site']['yarn.timeline-service.entity-group-fs-store.done-dir']
+ entity_groupfs_store_dir_mode = 0o700
+ is_webhdfs_enabled = hdfs_site['dfs.webhdfs.enabled']
+
+ if len(livy_livyserver_hosts) > 0:
+ has_livyserver = True
+ if security_enabled:
+ livy_principal = livy_kerberos_principal.replace('_HOST',
config['agentLevelParams']['hostname'].lower())
+
+ livy_livyserver_port =
default('configurations/livy-conf/livy.server.port',8999)
import functools
#create partial functions with common arguments for every HdfsResource call
@@ -260,5 +311,4 @@ HdfsResource = functools.partial(
default_fs = default_fs,
immutable_paths = get_not_managed_resources(),
dfs_type = dfs_type
-)
-
+)
\ No newline at end of file
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/service_check.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/service_check.py
new file mode 100644
index 0000000000..cfadd034e8
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/service_check.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agree in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import subprocess
+import time
+import os
+
+from resource_management.core.exceptions import Fail
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.format import format
+from resource_management.core.resources.system import Execute
+from resource_management.core.logger import Logger
+
+CHECK_COMMAND_TIMEOUT_DEFAULT = 60.0
+
+class LivyServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ if params.has_livyserver:
+ live_livyserver_host = ""
+ for livyserver_host in params.livy_livyserver_hosts:
+ try:
+ Execute(format("curl -s -o /dev/null -w'%{{http_code}}' --negotiate
-u: -k {livy_http_scheme}://{livyserver_host}:{livy_livyserver_port}/sessions |
grep 200"),
+ tries=3,
+ try_sleep=1,
+ logoutput=True,
+ user=params.smoke_user
+ )
+ live_livyserver_host = livyserver_host
+ break
+ except:
+ pass
+ if len(params.livy_livyserver_hosts) > 0 and live_livyserver_host == "":
+ raise Fail("Connection to all Livy servers failed")
+
+
+if __name__ == "__main__":
+ LivyServiceCheck().execute()
+
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/setup_livy.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/setup_livy.py
new file mode 100644
index 0000000000..b169cdd6a6
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/setup_livy.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.generate_logfeeder_input_config
import generate_logfeeder_input_config
+from resource_management import Directory, File, PropertiesFile, Template,
InlineTemplate, format
+
+
+def setup_livy(env, type, upgrade_type = None, action = None):
+ import params
+
+ Directory([params.livy_pid_dir, params.livy_log_dir],
+ owner=params.livy_user,
+ group=params.user_group,
+ mode=0o775,
+ cd_access='a',
+ create_parents=True
+ )
+ if type == 'server' and action == 'config':
+ params.HdfsResource(params.livy_hdfs_user_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.livy_user,
+ mode=0o775
+ )
+ params.HdfsResource(None, action="execute")
+
+ if params.livy_recovery_store == 'filesystem':
+ params.HdfsResource(params.livy_recovery_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.livy_user,
+ mode=0o700
+ )
+ params.HdfsResource(None, action="execute")
+
+ generate_logfeeder_input_config('livy',
Template("input.config-livy.json.j2", extra_imports=[default]))
+
+ # create livy-env.sh in etc/conf dir
+ File(os.path.join(params.livy_conf, 'livy-env.sh'),
+ owner=params.livy_user,
+ group=params.livy_group,
+ content=InlineTemplate(params.livy_env_sh),
+ mode=0o644,
+ )
+
+ # create livy-client.conf in etc/conf dir
+ PropertiesFile(format("{livy_conf}/livy-client.conf"),
+ properties =
params.config['configurations']['livy-client-conf'],
+ key_value_delimiter = " ",
+ owner=params.livy_user,
+ group=params.livy_group,
+ )
+
+ # create livy.conf in etc/conf dir
+ PropertiesFile(format("{livy_conf}/livy.conf"),
+ properties = params.config['configurations']['livy-conf'],
+ key_value_delimiter = " ",
+ owner=params.livy_user,
+ group=params.livy_group,
+ )
+
+ # create log4j.properties in etc/conf dir
+ File(os.path.join(params.livy_conf, 'log4j.properties'),
+ owner=params.livy_user,
+ group=params.livy_group,
+ content=params.livy_log4j_properties,
+ mode=0o644,
+ )
+
+ # create spark-blacklist.properties in etc/conf dir
+ File(os.path.join(params.livy_conf, 'spark-blacklist.conf'),
+ owner=params.livy_user,
+ group=params.livy_group,
+ content=params.livy_spark_blacklist_properties,
+ mode=0o644,
+ )
+
+ Directory(params.livy_logs_dir,
+ owner=params.livy_user,
+ group=params.livy_group,
+ mode=0o755,
+ )
+
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/status_params.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/status_params.py
new file mode 100644
index 0000000000..da308ac05d
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/scripts/status_params.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.default import default
+
+config = Script.get_config()
+
+spark_user = config['configurations']['spark-env']['spark_user']
+spark_group = config['configurations']['spark-env']['spark_group']
+user_group = config['configurations']['cluster-env']['user_group']
+
+if 'hive-env' in config['configurations']:
+ hive_user = config['configurations']['hive-env']['hive_user']
+else:
+ hive_user = "hive"
+
+spark_pid_dir = config['configurations']['spark-env']['spark_pid_dir']
+spark_history_server_pid_file =
format("{spark_pid_dir}/spark-{spark_user}-org.apache.spark.deploy.history.HistoryServer-1.pid")
+spark_thrift_server_pid_file =
format("{spark_pid_dir}/spark-{spark_user}-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid")
+stack_name = default("/clusterLevelParams/stack_name", None)
+
+if "livy-env" in config['configurations']:
+ livy_user = config['configurations']['livy-env']['livy_user']
+ livy_group = config['configurations']['livy-env']['livy_group']
+ livy_pid_dir = config['configurations']['livy-env']['livy_pid_dir']
+ livy_server_pid_file = format("{livy_pid_dir}/livy-{livy_user}-server.pid")
\ No newline at end of file
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/templates/input.config-livy.json.j2
similarity index 53%
copy from
ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2
copy to
ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/templates/input.config-livy.json.j2
index 089897b978..96df20410e 100644
---
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/package/templates/input.config-livy.json.j2
@@ -18,14 +18,9 @@
{
"input":[
{
- "type":"spark_jobhistory_server",
+ "type":"livy_server",
"rowtype":"service",
- "path":"{{default('/configurations/spark-env/spark_log_dir',
'/var/log/spark')}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out"
- },
- {
- "type":"spark_thriftserver",
- "rowtype":"service",
- "path":"{{default('/configurations/spark-env/spark_log_dir',
'/var/log/spark')}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out"
+ "path":"{{default('/configurations/livy-env/livy_log_dir',
'/var/log/livy')}}/livy-{{default('/configurations/livy-env/livy_user',
'livy')}}-server.out"
}
],
"filter":[
@@ -34,25 +29,17 @@
"conditions":{
"fields":{
"type":[
- "spark_jobhistory_server",
- "spark_thriftserver",
- "livy2_server"
+ "livy_server"
]
}
},
- "log4j_format":"",
-
"multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})",
-
"message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}",
+ "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+
"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\((%{JAVAFILE:file})?:(%{JAVAMETHOD:method})?\\((%{INT:line_number})?\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
"post_map_values":{
"logtime":{
"map_date":{
- "target_date_pattern":"yy/MM/dd HH:mm:ss"
- }
- },
- "level":{
- "map_field_value":{
- "pre_value":"WARNING",
- "post_value":"WARN"
+ "target_date_pattern":"yyyy-MM-dd'T'HH:mm:ss,SSS"
}
}
}
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/role_command_order.json
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/role_command_order.json
new file mode 100644
index 0000000000..55a24334a3
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/role_command_order.json
@@ -0,0 +1,8 @@
+{
+ "general_deps" : {
+ "_comment" : "dependencies for LIVY",
+ "LIVY_SERVER-START" : ["SPARK_JOBHISTORYSERVER-START"],
+ "LIVY_SERVICE_CHECK-SERVICE_CHECK" : ["LIVY_SERVER-START"]
+ }
+}
+
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/themes/directories.json
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/themes/directories.json
new file mode 100644
index 0000000000..08eec0ec3f
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/LIVY/themes/directories.json
@@ -0,0 +1,89 @@
+{
+ "name": "directories",
+ "description": "Directories theme for LIVY service",
+ "configuration": {
+ "layouts": [
+ {
+ "name": "directories",
+ "tabs": [
+ {
+ "name": "directories",
+ "display-name": "Directories",
+ "layout": {
+ "tab-columns": "1",
+ "tab-rows": "4",
+ "sections": [
+ {
+ "name": "subsection-log-dirs",
+ "display-name": "LOG DIRS",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-log-dirs",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ },
+ {
+ "name": "subsection-pid-dirs",
+ "display-name": "PID DIRS",
+ "row-index": "1",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-pid-dirs",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "placement": {
+ "configuration-layout": "default",
+ "configs": [
+ {
+ "config": "livy-env/livy_log_dir",
+ "subsection-name": "subsection-log-dirs"
+ },
+ {
+ "config": "livy-env/livy_pid_dir",
+ "subsection-name": "subsection-pid-dirs"
+ }
+ ]
+ },
+ "widgets": [
+ {
+ "config": "livy-env/livy_log_dir",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "livy-env/livy_pid_dir",
+ "widget": {
+ "type": "text-field"
+ }
+ }
+ ]
+ }
+ }
+
\ No newline at end of file
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py
index 222810f5b3..655f8c5142 100644
---
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/params.py
@@ -206,16 +206,9 @@ elif spark_transport_mode.lower() == 'http':
spark_thrift_endpoint =
default("configurations/spark-hive-site-override/hive.server2.http.endpoint",
"cliservice")
# thrift server support
-spark_thrift_sparkconf = None
-spark_thrift_cmd_opts_properties = ''
-spark_thrift_fairscheduler_content = None
-
-if has_spark_thriftserver and 'spark-defaults' in config['configurations']:
- spark_thrift_sparkconf = config['configurations']['spark-defaults']
- spark_thrift_cmd_opts_properties =
config['configurations']['spark-env']['spark_thrift_cmd_opts']
-
- if 'spark-thrift-fairscheduler' in config['configurations'] and
'fairscheduler_content' in
config['configurations']['spark-thrift-fairscheduler']:
- spark_thrift_fairscheduler_content =
config['configurations']['spark-thrift-fairscheduler']['fairscheduler_content']
+spark_thrift_sparkconf = config['configurations']['spark-defaults']
+spark_thrift_cmd_opts_properties =
config['configurations']['spark-env']['spark_thrift_cmd_opts']
+spark_thrift_fairscheduler_content =
config['configurations']['spark-thrift-fairscheduler']['fairscheduler_content']
if is_hive_installed:
# update default metastore client properties (async wait for metastore
component) it is useful in case of
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/setup_spark.py
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/setup_spark.py
index 7d205be57f..3097f122b9 100644
---
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/setup_spark.py
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/scripts/setup_spark.py
@@ -141,14 +141,13 @@ def setup_spark(env, type, upgrade_type = None, action =
None):
group=params.spark_group,
mode=0o644)
- if params.spark_thrift_fairscheduler_content:
- # create spark-thrift-fairscheduler.xml
- File(os.path.join(params.spark_conf_dir,"spark-thrift-fairscheduler.xml"),
- owner=params.spark_user,
- group=params.spark_group,
- mode=0o755,
- content=InlineTemplate(params.spark_thrift_fairscheduler_content)
- )
+ # create spark-thrift-fairscheduler.xml
+ File(os.path.join(params.spark_conf_dir,"spark-thrift-fairscheduler.xml"),
+ owner=params.spark_user,
+ group=params.spark_group,
+ mode=0o755,
+ content=InlineTemplate(params.spark_thrift_fairscheduler_content)
+ )
if type == "client":
Logger.info('Spark client config.')
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2
index 089897b978..06a280b7a6 100644
---
a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/package/templates/input.config-spark.json.j2
@@ -35,8 +35,7 @@
"fields":{
"type":[
"spark_jobhistory_server",
- "spark_thriftserver",
- "livy2_server"
+ "spark_thriftserver"
]
}
},
diff --git
a/ambari-server/src/main/resources/stacks/BIGTOP/3.3.0/services/LIVY/metainfo.xml
b/ambari-server/src/main/resources/stacks/BIGTOP/3.3.0/services/LIVY/metainfo.xml
new file mode 100644
index 0000000000..61f32e92f9
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/BIGTOP/3.3.0/services/LIVY/metainfo.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>LIVY</name>
+ <version>0.8.0-1</version>
+ </service>
+ </services>
+</metainfo>
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
