AMBARI-21226. Add two new Spark thrift server related configurations in Spark stack (Saisai Shao via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/24e2cacb Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/24e2cacb Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/24e2cacb Branch: refs/heads/branch-feature-AMBARI-20859 Commit: 24e2cacb8ede081824b9162ba124acdffa33e0ec Parents: df685b9 Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Tue Jun 13 19:57:06 2017 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Tue Jun 13 19:57:12 2017 -0700 ---------------------------------------------------------------------- .../configuration/spark-thrift-sparkconf.xml | 40 ++++++++++++++++++++ .../configuration/spark2-thrift-sparkconf.xml | 40 ++++++++++++++++++++ .../stacks/HDP/2.6/upgrades/config-upgrade.xml | 24 ++++++++++++ .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 26 +++++++++---- .../stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 32 ++++++++++------ 5 files changed, 142 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/24e2cacb/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/spark-thrift-sparkconf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/spark-thrift-sparkconf.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/spark-thrift-sparkconf.xml new file mode 100644 index 0000000..001ef1e --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/spark-thrift-sparkconf.xml @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="true"> + <property> + <name>spark.yarn.executor.failuresValidityInterval</name> + <value>2h</value> + <description> + Defines the validity interval for executor failure tracking. + Executor failures which are older than the validity interval will be ignored. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>spark.yarn.maxAppAttempts</name> + <value>1</value> + <description> + The maximum number of attempts that will be made to submit the application. + It should be no larger than the global number of max attempts in the YARN configuration. + </description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/24e2cacb/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-thrift-sparkconf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-thrift-sparkconf.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-thrift-sparkconf.xml new file mode 100644 index 0000000..001ef1e --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-thrift-sparkconf.xml @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="true"> + <property> + <name>spark.yarn.executor.failuresValidityInterval</name> + <value>2h</value> + <description> + Defines the validity interval for executor failure tracking. + Executor failures which are older than the validity interval will be ignored. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>spark.yarn.maxAppAttempts</name> + <value>1</value> + <description> + The maximum number of attempts that will be made to submit the application. + It should be no larger than the global number of max attempts in the YARN configuration. + </description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/24e2cacb/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml index 1610bb5..564dead 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml @@ -168,5 +168,29 @@ </component> </service> + <service name="SPARK"> + <component name="SPARK_THRIFTSERVER"> + <changes> + <definition xsi:type="configure" id="hdp_2_6_0_0_spark_thriftserver"> + <type>spark-thrift-sparkconf</type> + <set key="spark.yarn.executor.failuresValidityInterval" value="2h"/> + <set key="spark.yarn.maxAppAttempts" value="1"/> + </definition> + </changes> + </component> + </service> + + <service name="SPARK2"> + <component name="SPARK2_THRIFTSERVER"> + <changes> + <definition xsi:type="configure" id="hdp_2_6_0_0_spark2_thriftserver"> + <type>spark2-thrift-sparkconf</type> + <set key="spark.yarn.executor.failuresValidityInterval" value="2h"/> + <set key="spark.yarn.maxAppAttempts" value="1"/> + </definition> + </changes> + </component> + </service> + </services> </upgrade-config-changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/24e2cacb/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml index 1cdd184..6b827c9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml @@ -75,7 +75,7 @@ <service name="FLUME"> <component>FLUME_HANDLER</component> </service> - + <service name="ACCUMULO"> <component>ACCUMULO_TRACER</component> <component>ACCUMULO_GC</component> @@ -284,7 +284,7 @@ </execute-stage> </group> - + <group xsi:type="cluster" name="Upgrade service configs" title="Upgrade service configs"> <direction>UPGRADE</direction> <!-- prevent config changes on downgrade --> <skippable>true</skippable> <!-- May fix configuration problems manually --> @@ -322,7 +322,7 @@ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for Ranger Yarn plugin"> <task xsi:type="configure" id="hdp_2_6_maint_ranger_yarn_plugin_cluster_name"/> </execute-stage> - + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for YARN Preemption"> <task xsi:type="configure" id="hdp_2_6_yarn_preemption"/> </execute-stage> @@ -353,8 +353,18 @@ <task xsi:type="configure" id="hdp_2_6_maint_ranger_kms_plugin_cluster_name"/> </execute-stage> + <!--SPARK--> + <execute-stage service="SPARK" component="SPARK_THRIFTSERVER" title="Apply config changes for Spark ThriftServer"> + <task xsi:type="configure" id="hdp_2_6_0_0_spark_thriftserver"/> + </execute-stage> + + <!--SPARK2--> + <execute-stage service="SPARK2" component="SPARK2_THRIFTSERVER" title="Apply config changes for Spark2 ThriftServer"> + <task xsi:type="configure" id="hdp_2_6_0_0_spark2_thriftserver"/> + </execute-stage> + </group> - + <!-- After processing this group, the user-specified Kerberos descriptor will be updated to work with @@ -383,7 +393,7 @@ </task> </execute-stage> </group> - + <!-- Now, restart all of the services. --> <group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper"> <service-check>false</service-check> @@ -747,7 +757,7 @@ <component>FLUME_HANDLER</component> </service> </group> - + <group xsi:type="restart" name="ACCUMULO" title="Accumulo"> <service-check>false</service-check> <skippable>true</skippable> @@ -790,7 +800,7 @@ <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}} Pre-Check"> <direction>UPGRADE</direction> - + <execute-stage title="Check Component Versions"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction" /> </execute-stage> @@ -870,7 +880,7 @@ <function>setup_ranger_java_patches</function> </task> </pre-upgrade> - + <pre-downgrade copy-upgrade="true" /> <upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/24e2cacb/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml index 3e7e3d7..9227669 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml @@ -168,7 +168,7 @@ <component>HBASE_MASTER</component> </service> </group> - + <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check"> <direction>UPGRADE</direction> <skippable>true</skippable> @@ -184,7 +184,7 @@ <service>LOGSEARCH</service> </exclude> </group> - + <group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated"> <service-check>false</service-check> <skippable>true</skippable> @@ -192,7 +192,7 @@ <component>DATANODE</component> <component>NFS_GATEWAY</component> </service> - + <service name="HBASE"> <component>HBASE_REGIONSERVER</component> <component>PHOENIX_QUERY_SERVER</component> @@ -208,7 +208,7 @@ <message>The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services.</message> </batch> </group> - + <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check"> <direction>UPGRADE</direction> <skippable>true</skippable> @@ -460,7 +460,7 @@ <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}} Pre-Check"> <direction>UPGRADE</direction> - + <execute-stage title="Check Component Versions"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction" /> </execute-stage> @@ -469,7 +469,7 @@ <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}"> <skippable>true</skippable> <supports-auto-skip-failure>false</supports-auto-skip-failure> - + <execute-stage title="Confirm Finalize"> <direction>UPGRADE</direction> <task xsi:type="manual"> @@ -493,7 +493,7 @@ <function>finalize_rolling_upgrade</function> </task> </execute-stage> - + <execute-stage title="Save Cluster State"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction"> </task> @@ -544,7 +544,7 @@ <function>setup_ranger_java_patches</function> </task> </pre-upgrade> - + <pre-downgrade copy-upgrade="true" /> <upgrade> @@ -645,7 +645,7 @@ </upgrade> </component> </service> - + <service name="SLIDER"> <component name="SLIDER"> <upgrade> @@ -667,7 +667,7 @@ <summary>Verifying LZO codec path for mapreduce</summary> </task> </pre-upgrade> - + <pre-downgrade copy-upgrade="true" /> <upgrade> @@ -748,9 +748,9 @@ <summary>Verifying LZO codec path for Tez</summary> </task> </pre-upgrade> - + <pre-downgrade copy-upgrade="true" /> - + <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -830,6 +830,10 @@ </upgrade> </component> <component name="SPARK_THRIFTSERVER"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_6_0_0_spark_thriftserver"/> + </pre-upgrade> + <pre-downgrade /> <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -853,6 +857,10 @@ </upgrade> </component> <component name="SPARK2_THRIFTSERVER"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_6_0_0_spark2_thriftserver"/> + </pre-upgrade> + <pre-downgrade /> <upgrade> <task xsi:type="restart-task" /> </upgrade>