AMBARI-20069. hive.enforce.bucketing=true.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/60aaaeab Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/60aaaeab Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/60aaaeab Branch: refs/heads/branch-feature-AMBARI-20053 Commit: 60aaaeab5e2e9c7581f982525950bf6290db3018 Parents: 1ecdee0 Author: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Authored: Tue Feb 21 02:15:11 2017 +0200 Committer: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Committed: Tue Feb 21 02:15:11 2017 +0200 ---------------------------------------------------------------------- .../main/resources/stacks/HDP/2.2/services/stack_advisor.py | 7 +++++-- .../main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml | 5 +++++ .../stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 6 ++++++ .../main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 2 ++ .../main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml | 5 +++++ .../stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 6 ++++++ .../main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 2 ++ .../main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml | 5 +++++ .../stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 6 ++++++ .../main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 1 + .../src/test/python/stacks/2.2/common/test_stack_advisor.py | 1 - .../src/test/python/stacks/2.3/common/test_stack_advisor.py | 2 -- 12 files changed, 43 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 0d7b8b9..ede41fc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -374,14 +374,12 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putHiveSiteProperty("hive.support.concurrency", "true") putHiveSiteProperty("hive.compactor.initiator.on", "true") putHiveSiteProperty("hive.compactor.worker.threads", "1") - putHiveSiteProperty("hive.enforce.bucketing", "true") putHiveSiteProperty("hive.exec.dynamic.partition.mode", "nonstrict") else: putHiveSiteProperty("hive.txn.manager", "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager") putHiveSiteProperty("hive.support.concurrency", "false") putHiveSiteProperty("hive.compactor.initiator.on", "false") putHiveSiteProperty("hive.compactor.worker.threads", "0") - putHiveSiteProperty("hive.enforce.bucketing", "false") putHiveSiteProperty("hive.exec.dynamic.partition.mode", "strict") hiveMetastoreHost = self.getHostWithComponent("HIVE", "HIVE_METASTORE", services, hosts) @@ -1497,6 +1495,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor): ldap_domain_property + " property, if you are using AD, if not, then " + ldap_baseDN_property + "!")}) + hive_enforce_bucketing = "hive.enforce.bucketing" + if hive_enforce_bucketing in properties and properties[hive_enforce_bucketing].lower() == "false": + validationItems.append({"config-name" : hive_enforce_bucketing, "item" : + self.getWarnItem("Set " + hive_enforce_bucketing + " to true otherwise there is a potential of data corruption!")}) + configurationValidationProblems = self.toConfigurationValidationProblems(validationItems, "hive-site") configurationValidationProblems.extend(parentValidationProblems) return configurationValidationProblems http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index ba155ab..cdd701a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -195,6 +195,11 @@ <regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/> <replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB"/> </definition> + + <definition xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property"> + <type>hive-site</type> + <set key="hive.enforce.bucketing" value="true"/> + </definition> </changes> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml index e6a3c94..ff42022 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml @@ -419,6 +419,12 @@ </task> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing property"> + <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property"> + <summary>Updating hive.enforce.bucketing property to true value</summary> + </task> + </execute-stage> + <!--OOZIE--> <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"> http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml index 7d8438f..b255bc0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml @@ -893,6 +893,8 @@ <summary>Update hive-env content</summary> </task> <task xsi:type="configure" id="hive_log4j_parameterize" /> + + <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property" /> </pre-upgrade> <pre-downgrade/> http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml index d19bb76..1bea263 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml @@ -96,6 +96,11 @@ <regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/> <replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB"/> </definition> + + <definition xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property"> + <type>hive-site</type> + <set key="hive.enforce.bucketing" value="true"/> + </definition> </changes> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml index 69a894c..86cd56a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml @@ -461,6 +461,12 @@ </task> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing property"> + <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property"> + <summary>Updating hive.enforce.bucketing property to true value</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties"> <task xsi:type="configure" id="webhcat_log4j_parameterize"> <summary>Updating the Webhcat Log4J properties to include parameterizations</summary> http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml index 0c9a8ea..6a8e9d7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml @@ -892,6 +892,8 @@ </task> <task xsi:type="configure" id="hive_log4j_parameterize" /> + <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property" /> + </pre-upgrade> <pre-downgrade/> http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 241ca8f..ca9cf47 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -358,6 +358,11 @@ <regex-replace key="content" find="property.llap.daemon.log.maxfilesize = ([0-9]+)MB" replace-with="property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB"/> <regex-replace key="content" find="property.llap.daemon.log.maxbackupindex = ([0-9]+)" replace-with="property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}"/> </definition> + + <definition xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property"> + <type>hive-site</type> + <set key="hive.enforce.bucketing" value="true"/> + </definition> </changes> </component> <component name="HIVE_SERVER_INTERACTIVE"> http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 0ef69cb..06cae78 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -478,6 +478,12 @@ <task xsi:type="configure" id="hdp_2_6_0_0_tez_append_heap_dump_options_for_tez_am"/> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing property"> + <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property"> + <summary>Updating hive.enforce.bucketing property to true value</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Appending heap dump options for HiveSever2 Interactive"> <task xsi:type="configure" id="hdp_2_6_0_0_hive_llap_append_heap_dump_options"/> </execute-stage> http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index 6309bcb..64edbb8 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -781,6 +781,7 @@ <task xsi:type="configure" id="hdp_2_6_0_0_hive_append_heap_dump_options"/> <task xsi:type="configure" id="hive_log4j_parameterize" /> <task xsi:type="configure" id="hive_llap_log4j_parameterize" /> + <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property" /> </pre-upgrade> <pre-downgrade /> http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 0c7af1a..b8762b3 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1300,7 +1300,6 @@ class TestHDP22StackAdvisor(TestCase): 'hive.compactor.initiator.on': 'false', 'hive.compactor.worker.threads': '0', 'hive.compute.query.using.stats': 'true', - 'hive.enforce.bucketing': 'false', 'hive.exec.dynamic.partition.mode': 'strict', 'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook', 'hive.exec.orc.compression.strategy': 'SPEED', http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 7e93815..443331a 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -605,7 +605,6 @@ class TestHDP23StackAdvisor(TestCase): 'hive.compactor.initiator.on': 'false', 'hive.compactor.worker.threads': '0', 'hive.compute.query.using.stats': 'true', - 'hive.enforce.bucketing': 'false', 'hive.exec.dynamic.partition.mode': 'strict', 'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook', 'hive.exec.orc.compression.strategy': 'SPEED', @@ -883,7 +882,6 @@ class TestHDP23StackAdvisor(TestCase): 'hive.compactor.initiator.on': 'false', 'hive.compactor.worker.threads': '0', 'hive.compute.query.using.stats': 'true', - 'hive.enforce.bucketing': 'false', 'hive.exec.dynamic.partition.mode': 'strict', 'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook', 'hive.exec.orc.compression.strategy': 'SPEED',