Revert "AMBARI-22162. Move out the druid configurations from hive-site to hive-interactive-site. (Slim Bouguerra via Swapan Shridhar)."
This reverts commit ce2a0a00921bde8d780c82561902773f76431fce. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3b8e8071 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3b8e8071 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3b8e8071 Branch: refs/heads/branch-feature-AMBARI-14714 Commit: 3b8e807100a988fafc4d55e7a0ad7644fac6cef2 Parents: 8cffd72 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Mon Oct 9 16:59:04 2017 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Mon Oct 9 16:59:04 2017 -0400 ---------------------------------------------------------------------- .../HIVE/0.12.0.2.0/configuration/hive-site.xml | 217 ++++++++++++++++++ .../configuration/hive-interactive-site.xml | 225 ------------------- .../stacks/HDP/2.6/services/stack_advisor.py | 12 +- .../stacks/HDP/2.6/upgrades/config-upgrade.xml | 21 +- 4 files changed, 234 insertions(+), 241 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/3b8e8071/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml index 69d1c69..d66cf4c 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml @@ -451,6 +451,223 @@ limitations under the License. <on-ambari-upgrade add="false"/> </property> + <!-- Druid related properties --> + <property> + <name>hive.druid.broker.address.default</name> + <value>localhost:8082</value> + <description>Host name of druid router if any or broker</description> + <on-ambari-upgrade add="false"/> + <depends-on> + <property> + <type>druid-router</type> + <name>druid.port</name> + </property> + </depends-on> + </property> + + <property> + <name>hive.druid.metadata.uri</name> + <value>jdbc:mysql://localhost:3355/druid</value> + <description>URI to connect to the database (for example jdbc:mysql://hostname:port/DBName)</description> + <on-ambari-upgrade add="false"/> + <depends-on> + <property> + <type>druid-common</type> + <name>druid.metadata.storage.connector.connectURI</name> + </property> + </depends-on> + </property> + + <property> + <name>hive.druid.coordinator.address.default</name> + <value>localhost:8082</value> + <description>Host name of druid router if any or broker</description> + <on-ambari-upgrade add="false"/> + <depends-on> + <property> + <type>druid-coordinator</type> + <name>druid.port</name> + </property> + </depends-on> + </property> + + <property> + <name>hive.druid.metadata.password</name> + <value>{{druid_metadata_password}}</value> + <property-type>PASSWORD</property-type> + <display-name>Druid Metadata Password</display-name> + <description>Druid meta data storage password</description> + <value-attributes> + <type>password</type> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.metadata.username</name> + <value>druid</value> + <description>Username used to connect to druid metadata storage</description> + <on-ambari-upgrade add="false"/> + <depends-on> + <property> + <type>druid-common</type> + <name>druid.metadata.storage.connector.user</name> + </property> + </depends-on> + </property> + + <property> + <name>hive.druid.indexer.segments.granularity</name> + <display-name>Default Granularity for the Druid segments</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>YEAR</value> + </entry> + <entry> + <value>MONTH</value> + </entry> + <entry> + <value>WEEK</value> + </entry> + <entry> + <value>DAY</value> + </entry> + <entry> + <value>HOUR</value> + </entry> + <entry> + <value>MINUTE</value> + </entry> + <entry> + <value>SECOND</value> + </entry> + </entries> + </value-attributes> + <value>MINUTE</value> + <description>Default Granularity for the segments created by the Druid storage handler, this can be overridden per table using table property druid.segment.granularity </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.druid.indexer.partition.size.max</name> + <value>5000000</value> + <description>Maximum number of records per segment partition</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.druid.indexer.memory.rownum.max</name> + <value>75000</value> + <description>Maximum number of records in memory while storing data in Druid</description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.select.distribute</name> + <value>true</value> + <description>If it is set to true, we distribute the execution of Druid Select queries</description> + <on-ambari-upgrade add="false"/> + <value-attributes> + <type>boolean</type> + </value-attributes> + </property> + + <property> + <name>hive.druid.basePersistDirectory</name> + <value></value> + <description> + Local temporary directory used to persist intermediate indexing state, + if empty (recommended) will default to JVM system property java.io.tmpdir. + </description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.storage.storageDirectory</name> + <value>{{druid_storage_dir}}</value> + <description> + Druid deep storage location for segments. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.druid.metadata.db.type</name> + <display-name>Druid metadata storage type </display-name> + <value-attributes> + <overridable>false</overridable> + <type>value-list</type> + <entries> + <entry> + <value>mysql</value> + <label>MYSQL</label> + </entry> + <entry> + <value>postgresql</value> + <label>POSTGRESQL</label> + </entry> + </entries> + </value-attributes> + <value>mysql</value> + <depends-on> + <property> + <type>druid-common</type> + <name>druid.metadata.storage.type</name> + </property> + </depends-on> + <description>Druid metadata storage type</description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.passiveWaitTimeMs</name> + <value>30000</value> + <description> + Wait time in ms default to 30 seconds. + </description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.working.directory</name> + <value>/tmp/druid-indexing</value> + <description> + Default hdfs working directory used to store some intermediate metadata. + </description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.maxTries</name> + <value>5</value> + <description> + Maximum number of http call retries before giving up. + </description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive.druid.bitmap.type</name> + <display-name>Druid metadata storage type </display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>roaring</value> + </entry> + <entry> + <value>concise</value> + </entry> + </entries> + </value-attributes> + <value>roaring</value> + <description>Druid Coding algorithm use to encode the bitmaps</description> + <on-ambari-upgrade add="false"/> + </property> + <!-- This property is removed in HDP 2.5 and higher. --> <property> <name>atlas.rest.address</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/3b8e8071/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml index 64cef3e..aae2efa 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml @@ -124,229 +124,4 @@ limitations under the License. <on-ambari-upgrade add="false"/> </property> - <!-- Druid related properties --> - <property> - <name>hive.druid.broker.address.default</name> - <value>localhost:8082</value> - <description>Host name of druid router if any or broker</description> - <on-ambari-upgrade add="false"/> - <depends-on> - <property> - <type>druid-router</type> - <name>druid.port</name> - </property> - </depends-on> - </property> - - <property> - <name>hive.druid.metadata.uri</name> - <value>jdbc:mysql://localhost:3355/druid</value> - <description>URI to connect to the database (for example jdbc:mysql://hostname:port/DBName)</description> - <on-ambari-upgrade add="false"/> - <depends-on> - <property> - <type>druid-common</type> - <name>druid.metadata.storage.connector.connectURI</name> - </property> - </depends-on> - </property> - - <property> - <name>hive.druid.coordinator.address.default</name> - <value>localhost:8082</value> - <description>Host name of druid router if any or broker</description> - <on-ambari-upgrade add="false"/> - <depends-on> - <property> - <type>druid-coordinator</type> - <name>druid.port</name> - </property> - </depends-on> - </property> - - <property> - <name>hive.druid.metadata.password</name> - <value>{{druid_metadata_password}}</value> - <property-type>PASSWORD</property-type> - <display-name>Druid Metadata Password</display-name> - <description>Druid meta data storage password</description> - <value-attributes> - <type>password</type> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.metadata.username</name> - <value>druid</value> - <description>Username used to connect to druid metadata storage</description> - <on-ambari-upgrade add="false"/> - <depends-on> - <property> - <type>druid-common</type> - <name>druid.metadata.storage.connector.user</name> - </property> - </depends-on> - </property> - - <property> - <name>hive.druid.indexer.segments.granularity</name> - <display-name>Default Granularity for the Druid segments</display-name> - <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>YEAR</value> - </entry> - <entry> - <value>MONTH</value> - </entry> - <entry> - <value>WEEK</value> - </entry> - <entry> - <value>DAY</value> - </entry> - <entry> - <value>HOUR</value> - </entry> - <entry> - <value>MINUTE</value> - </entry> - <entry> - <value>SECOND</value> - </entry> - </entries> - </value-attributes> - <value>DAY</value> - <description>Default Granularity for the segments created by the Druid storage handler, this can be overridden per table using table property druid.segment.granularity </description> - <on-ambari-upgrade add="false"/> - </property> - <property> - <name>hive.druid.indexer.partition.size.max</name> - <value>1000000</value> - <description>Maximum number of records per segment partition</description> - <on-ambari-upgrade add="false"/> - </property> - <property> - <name>hive.druid.indexer.memory.rownum.max</name> - <value>75000</value> - <description>Maximum number of records in memory while storing data in Druid</description> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.select.distribute</name> - <value>true</value> - <description>If it is set to true, we distribute the execution of Druid Select queries</description> - <on-ambari-upgrade add="false"/> - <value-attributes> - <type>boolean</type> - </value-attributes> - </property> - - <property> - <name>hive.druid.basePersistDirectory</name> - <value></value> - <description> - Local temporary directory used to persist intermediate indexing state, - if empty (recommended) will default to JVM system property java.io.tmpdir. - </description> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.storage.storageDirectory</name> - <value>{{druid_storage_dir}}</value> - <description> - Druid deep storage location for segments. - </description> - <on-ambari-upgrade add="false"/> - </property> - <property> - <name>hive.druid.metadata.db.type</name> - <display-name>Druid metadata storage type </display-name> - <value-attributes> - <overridable>false</overridable> - <type>value-list</type> - <entries> - <entry> - <value>mysql</value> - <label>MYSQL</label> - </entry> - <entry> - <value>postgresql</value> - <label>POSTGRESQL</label> - </entry> - </entries> - </value-attributes> - <value>mysql</value> - <depends-on> - <property> - <type>druid-common</type> - <name>druid.metadata.storage.type</name> - </property> - </depends-on> - <description>Druid metadata storage type</description> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.passiveWaitTimeMs</name> - <value>30000</value> - <description> - Wait time in ms default to 30 seconds. - </description> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.working.directory</name> - <value>/tmp/druid-indexing</value> - <description> - Default hdfs working directory used to store some intermediate metadata. - </description> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.maxTries</name> - <value>5</value> - <description> - Maximum number of http call retries before giving up. - </description> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.bitmap.type</name> - <display-name>Druid metadata storage type </display-name> - <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>roaring</value> - </entry> - <entry> - <value>concise</value> - </entry> - </entries> - </value-attributes> - <value>roaring</value> - <description>Druid Coding algorithm use to encode the bitmaps</description> - <on-ambari-upgrade add="false"/> - </property> - - <property> - <name>hive.druid.http.read.timeout</name> - <value>PT10M</value> - <description> - Maximum number of http call retries before giving up. - </description> - <on-ambari-upgrade add="false"/> - </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/3b8e8071/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py index b634e71..0d2925e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py @@ -605,7 +605,7 @@ class HDP26StackAdvisor(HDP25StackAdvisor): # druid is not in list of services to be installed servicesList = [service["StackServices"]["service_name"] for service in services["services"]] if 'DRUID' in servicesList: - putHiveInteractiveSiteProperty = self.putProperty(configurations, "hive-interactive-site", services) + putHiveSiteProperty = self.putProperty(configurations, "hive-site", services) if 'druid-coordinator' in services['configurations']: component_hosts = self.getHostsWithComponent("DRUID", 'DRUID_COORDINATOR', services, hosts) if component_hosts is not None and len(component_hosts) > 0: @@ -642,11 +642,11 @@ class HDP26StackAdvisor(HDP25StackAdvisor): else: druid_metadata_user = "" - putHiveInteractiveSiteProperty('hive.druid.broker.address.default', druid_broker_host_port) - putHiveInteractiveSiteProperty('hive.druid.coordinator.address.default', druid_coordinator_host_port) - putHiveInteractiveSiteProperty('hive.druid.metadata.uri', druid_metadata_uri) - putHiveInteractiveSiteProperty('hive.druid.metadata.username', druid_metadata_user) - putHiveInteractiveSiteProperty('hive.druid.metadata.db.type', druid_metadata_type) + putHiveSiteProperty('hive.druid.broker.address.default', druid_broker_host_port) + putHiveSiteProperty('hive.druid.coordinator.address.default', druid_coordinator_host_port) + putHiveSiteProperty('hive.druid.metadata.uri', druid_metadata_uri) + putHiveSiteProperty('hive.druid.metadata.username', druid_metadata_user) + putHiveSiteProperty('hive.druid.metadata.db.type', druid_metadata_type) def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts): http://git-wip-us.apache.org/repos/asf/ambari/blob/3b8e8071/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml index fd7e438..2b4c656 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml @@ -75,19 +75,11 @@ <set key ="atlas.jaas.ticketBased-KafkaClient.option.useTicketCache" value="true" if-type="cluster-env" if-key="security_enabled" if-value="true"/> </definition> - </changes> - </component> - <component name="HIVE_SERVER_INTERACTIVE"> - <changes> - <definition xsi:type="configure" id="llap_update_tez_shuffle_ssl_enable" summary="Update additional LLAP-Tez settings"> - <type>tez-interactive-site</type> - <set key="tez.runtime.shuffle.ssl.enable" value="false"/> - </definition> <definition xsi:type="configure" id="hdp_2_6_maint_druid_config_for_hive_hook" summary="Updating druid hive related properties"> <type>hive-site</type> <set key="hive.druid.metadata.password" value="{{druid_metadata_password}}" if-type="druid-common"/> - <set key="hive.druid.indexer.segments.granularity" value="DAY" if-type="druid-common"/> - <set key="hive.druid.indexer.partition.size.max" value="1000000" if-type="druid-common"/> + <set key="hive.druid.indexer.segments.granularity" value="MINUTE" if-type="druid-common"/> + <set key="hive.druid.indexer.partition.size.max" value="5000000" if-type="druid-common"/> <set key="hive.druid.indexer.memory.rownum.max" value="75000" if-type="druid-common"/> <set key="hive.druid.select.distribute" value="true" if-type="druid-common"/> <set key="hive.druid.basePersistDirectory" value="" if-type="druid-common"/> @@ -95,6 +87,15 @@ <set key="hive.druid.passiveWaitTimeMs" value="30000" if-type="druid-common"/> <set key="hive.druid.working.directory" value="/tmp/druid-indexing" if-type="druid-common"/> <set key="hive.druid.bitmap.type" value="roaring" if-type="druid-common"/> + + </definition> + </changes> + </component> + <component name="HIVE_SERVER_INTERACTIVE"> + <changes> + <definition xsi:type="configure" id="llap_update_tez_shuffle_ssl_enable" summary="Update additional LLAP-Tez settings"> + <type>tez-interactive-site</type> + <set key="tez.runtime.shuffle.ssl.enable" value="false"/> </definition> </changes> </component>