Repository: ambari Updated Branches: refs/heads/trunk 66e2eb424 -> 618d843c1
AMBARI-13650. Bump the Ambari Spark definition to Spark 1.5.1 (Saisai Shao via smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/db224500 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/db224500 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/db224500 Branch: refs/heads/trunk Commit: db224500f253f2bd6c842f9097efe572943001cf Parents: 66e2eb4 Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Sun Nov 8 17:13:35 2015 -0800 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Sun Nov 8 17:13:35 2015 -0800 ---------------------------------------------------------------------- .../1.2.0.2.2/configuration/spark-defaults.xml | 17 ++----- .../SPARK/1.5.1.2.3/metainfo.xml | 50 ++++++++++++++++++++ .../configuration/spark-thrift-sparkconf.xml | 13 +---- .../stacks/HDP/2.3/services/SPARK/metainfo.xml | 45 ++---------------- 4 files changed, 60 insertions(+), 65 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/db224500/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml index 8b13779..05d1a3a 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml @@ -42,15 +42,6 @@ </property> <property> - <name>spark.yarn.applicationMaster.waitTries</name> - <value>10</value> - <description> - Set the number of times the ApplicationMaster waits for the the Spark master and then - also the number of tries it waits for the SparkContext to be initialized. - </description> - </property> - - <property> <name>spark.yarn.scheduler.heartbeat.interval-ms</name> <value>5000</value> <description> @@ -95,7 +86,7 @@ <name>spark.yarn.preserve.staging.files</name> <value>false</value> <description> - Set to true to preserve the staged files (Spark jar, app jar, distributed cache files) at the + Set to true to preserve the staged files (Spark jar, app jar, distributed cache files) at the end of the job rather then delete them. </description> </property> @@ -161,19 +152,19 @@ Location of the kerberos keytab file for the Spark History Server. </description> </property> - + <property> <name>spark.history.provider</name> <value>org.apache.spark.deploy.yarn.history.YarnHistoryProvider</value> <description>Name of history provider class</description> </property> - + <property> <name>spark.yarn.historyServer.address</name> <value>{{spark_history_server_host}}:{{spark_history_ui_port}}</value> <description>The address of the Spark history server (i.e. host.com:18080). The address should not contain a scheme (http://). Defaults to not being set since the history server is an optional service. This address is given to the YARN ResourceManager when the Spark application finishes to link the application from the ResourceManager UI to the Spark history server UI.</description> </property> - + <property> <name>spark.yarn.max.executor.failures</name> <value>3</value> http://git-wip-us.apache.org/repos/asf/ambari/blob/db224500/ambari-server/src/main/resources/common-services/SPARK/1.5.1.2.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.5.1.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK/1.5.1.2.3/metainfo.xml new file mode 100644 index 0000000..190d892 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SPARK/1.5.1.2.3/metainfo.xml @@ -0,0 +1,50 @@ +<?xml version="1.0"?> +<!--Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>SPARK</name> + <extends>common-services/SPARK/1.4.1.2.3</extends> + <displayName>Spark</displayName> + <comment>Apache Spark is a fast and general engine for large-scale data processing.</comment> + <version>1.5.1.2.3</version> + <!-- No new components added for 1.5.1 --> + <configuration-dependencies> + <config-type>spark-defaults</config-type> + <config-type>spark-env</config-type> + <config-type>spark-log4j-properties</config-type> + <config-type>spark-metrics-properties</config-type> + <config-type>spark-javaopts-properties</config-type> + <config-type>spark-thrift-sparkconf</config-type> + <config-type>spark-hive-site-override</config-type> + </configuration-dependencies> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>YARN</service> + </requiredServices> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/db224500/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml index c42841f..41d2e21 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml @@ -42,15 +42,6 @@ </property> <property> - <name>spark.yarn.applicationMaster.waitTries</name> - <value>10</value> - <description> - Set the number of times the ApplicationMaster waits for the the Spark master and then - also the number of tries it waits for the SparkContext to be initialized. - </description> - </property> - - <property> <name>spark.yarn.scheduler.heartbeat.interval-ms</name> <value>5000</value> <description> @@ -95,7 +86,7 @@ <name>spark.yarn.preserve.staging.files</name> <value>false</value> <description> - Set to true to preserve the staged files (Spark jar, app jar, distributed cache files) at the + Set to true to preserve the staged files (Spark jar, app jar, distributed cache files) at the end of the job rather then delete them. </description> </property> @@ -115,7 +106,7 @@ Specifies the parameters that are passed to the JVM of the Spark Application Master. </description> </property> - + <property> <name>spark.yarn.max.executor.failures</name> <value>3</value> http://git-wip-us.apache.org/repos/asf/ambari/blob/db224500/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml index 14161b4..0e20794 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml @@ -23,8 +23,8 @@ <services> <service> <name>SPARK</name> - <version>1.4.1.2.3</version> - <extends>common-services/SPARK/1.3.1.2.3</extends> + <version>1.5.1.2.3</version> + <extends>common-services/SPARK/1.4.1.2.3</extends> <osSpecifics> <osSpecific> <osFamily>redhat7,redhat6,suse11</osFamily> @@ -52,44 +52,7 @@ <requiredServices> <service>YARN</service> </requiredServices> - <components> - <component> - <name>SPARK_THRIFTSERVER</name> - <displayName>Spark Thrift Server</displayName> - <deleted>true</deleted> - <category>MASTER</category> - <cardinality>0+</cardinality> - <versionAdvertised>true</versionAdvertised> - <dependencies> - <dependency> - <name>HDFS/HDFS_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - <dependency> - <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - <dependency> - <name>YARN/YARN_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - </dependencies> - <commandScript> - <script>scripts/spark_thrift_server.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> - </components> + <!-- No new components compared to 1.4.1 --> <configuration-dependencies> <config-type>spark-defaults</config-type> <config-type>spark-env</config-type> @@ -98,7 +61,7 @@ <config-type>spark-javaopts-properties</config-type> <config-type>spark-thrift-sparkconf</config-type> <config-type>spark-hive-site-override</config-type> - </configuration-dependencies> + </configuration-dependencies> </service> </services> </metainfo>
