Repository: ambari Updated Branches: refs/heads/trunk 0b157069a -> 2f06c1cef
AMBARI-19702. Switch Pig default engine to tez in HDP 2.6 (Daniel Dai via alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2f06c1ce Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2f06c1ce Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2f06c1ce Branch: refs/heads/trunk Commit: 2f06c1cefac30e0a53d631cf2a2a6aef64a89e0c Parents: 0b15706 Author: Alejandro Fernandez <afernan...@hortonworks.com> Authored: Mon Jan 30 19:26:18 2017 -0800 Committer: Alejandro Fernandez <afernan...@hortonworks.com> Committed: Mon Jan 30 19:26:18 2017 -0800 ---------------------------------------------------------------------- .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 11 +++ .../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 5 ++ .../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 3 + .../stacks/HDP/2.4/upgrades/config-upgrade.xml | 11 +++ .../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 5 ++ .../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 3 + .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 11 +++ .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 5 ++ .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 3 + .../PIG/configuration/pig-properties.xml | 93 ++++++++++++++++++++ 10 files changed, 150 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index 1f3b5ad..3ea660d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -123,6 +123,17 @@ </component> </service> + <service name="PIG"> + <component name="PIG"> + <changes> + <definition xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez"> + <type>pig-properties</type> + <replace key="exectype" find="exectype=mapreduce" replace-with="exectype=tez" /> + </definition> + </changes> + </component> + </service> + <service name="HIVE"> <component name="HIVE_SERVER"> <changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml index a9b1480..6153a8c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml @@ -359,6 +359,11 @@ <task xsi:type="configure" id="hdp_2_5_0_0_add_sqoop_atlas_security_configs" /> </execute-stage> + <!-- PIG --> + <execute-stage service="PIG" component="PIG" title="Apply config changes for Pig"> + <task xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez" /> + </execute-stage> + <!-- HIVE --> <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for WebHCat Server"> <task xsi:type="configure" id="hdp_2_5_0_0_webhcat_server_update_configuration_paths"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml index a65b32c..daf249c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml @@ -831,6 +831,9 @@ <service name="PIG"> <component name="PIG"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez"/> + </pre-upgrade> <upgrade> <task xsi:type="restart-task" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml index 7179099..8c8997d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml @@ -57,6 +57,17 @@ </component> </service> + <service name="PIG"> + <component name="PIG"> + <changes> + <definition xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez"> + <type>pig-properties</type> + <replace key="exectype" find="exectype=mapreduce" replace-with="exectype=tez" /> + </definition> + </changes> + </component> + </service> + <service name="HIVE"> <component name="HIVE_SERVER"> <changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml index 7b9b062..f473071 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml @@ -408,6 +408,11 @@ <task xsi:type="configure" id="hdp_2_5_0_0_add_sqoop_atlas_security_configs" /> </execute-stage> + <!-- PIG --> + <execute-stage service="PIG" component="PIG" title="Apply config changes for Pig"> + <task xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez" /> + </execute-stage> + <!-- HIVE --> <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hive_audit_db"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml index 781149c..2be1ad7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml @@ -834,6 +834,9 @@ <service name="PIG"> <component name="PIG"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez"/> + </pre-upgrade> <upgrade> <task xsi:type="restart-task" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 157e0e9..2aca9b8 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -278,6 +278,17 @@ </component> </service> + <service name="PIG"> + <component name="PIG"> + <changes> + <definition xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez"> + <type>pig-properties</type> + <replace key="exectype" find="exectype=mapreduce" replace-with="exectype=tez" /> + </definition> + </changes> + </component> + </service> + <service name="HIVE"> <component name="HIVE_SERVER"> <changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 8092506..1adf75f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -413,6 +413,11 @@ </task> </execute-stage> + <!-- PIG --> + <execute-stage service="PIG" component="PIG" title="Apply config changes for Pig"> + <task xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez" /> + </execute-stage> + <!--HIVE--> <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties"> <task xsi:type="configure" id="hive_log4j_parameterize"> http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index a739551..6287789 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -734,6 +734,9 @@ <service name="PIG"> <component name="PIG"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez"/> + </pre-upgrade> <upgrade> <task xsi:type="restart-task" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.6/services/PIG/configuration/pig-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/PIG/configuration/pig-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/PIG/configuration/pig-properties.xml new file mode 100644 index 0000000..0f3ebdd --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/PIG/configuration/pig-properties.xml @@ -0,0 +1,93 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>content</name> + <display-name>pig-properties template</display-name> + <description>Describe all the Pig agent configurations</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments. +# see bin/pig -help + +# brief logging (no timestamps) +brief=false + +# debug level, INFO is default +debug=INFO + +# verbose print all log messages to screen (default to print only INFO and above to screen) +verbose=false + +# exectype local|mapreduce|tez, mapreduce is default +exectype=tez + +# Enable insertion of information about script into hadoop job conf +pig.script.info.enabled=true + +# Do not spill temp files smaller than this size (bytes) +pig.spill.size.threshold=5000000 + +# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) +# This should help reduce the number of files being spilled. +pig.spill.gc.activation.size=40000000 + +# the following two parameters are to help estimate the reducer number +pig.exec.reducers.bytes.per.reducer=1000000000 +pig.exec.reducers.max=999 + +# Temporary location to store the intermediate data. +pig.temp.dir=/tmp/ + +# Threshold for merging FRJoin fragment files +pig.files.concatenation.threshold=100 +pig.optimistic.files.concatenation=false; + +pig.disable.counter=false + +hcat.bin=/usr/bin/hcat + + </value> + <value-attributes> + <type>content</type> + <empty-value-valid>true</empty-value-valid> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration>
