Ottomata has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/355471 )
Change subject: Changes needed for upgrading to Druid 0.10 ...................................................................... Changes needed for upgrading to Druid 0.10 Druid 0.10 depends on Java 8. The package should install it as a dependency when installing, but it will not be the default Java. Setting JAVA_HOME explicitly in daemon env.sh files. Reduce middlemanager indexer peon task druid.processing.buffer.sizeBytes to 356515840 for 'non-zero default value for druid.processing.numMergeBuffers'. Change default extensions to ones included with Druid 0.10. This reverts a revert of https://gerrit.wikimedia.org/r/#/c/351691. We need to upgrade Hadoop cluster to Java 8 before proceeding with this change. Bug: T164008 Change-Id: Ic9a091c3b601dd382b66770baf1e2494fb77eef9 --- M hieradata/eqiad/druid/middlemanager.yaml M modules/druid/manifests/broker.pp M modules/druid/manifests/coordinator.pp M modules/druid/manifests/historical.pp M modules/druid/manifests/init.pp M modules/druid/manifests/middlemanager.pp M modules/druid/manifests/overlord.pp 7 files changed, 24 insertions(+), 7 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/71/355471/1 diff --git a/hieradata/eqiad/druid/middlemanager.yaml b/hieradata/eqiad/druid/middlemanager.yaml index 1253682..9d47f0e 100644 --- a/hieradata/eqiad/druid/middlemanager.yaml +++ b/hieradata/eqiad/druid/middlemanager.yaml @@ -2,7 +2,7 @@ druid.worker.ip: "%{::fqdn}" druid.worker.capacity: 12 druid.processing.numThreads: 3 - druid.processing.buffer.sizeBytes: 536870912 + druid.processing.buffer.sizeBytes: 356515840 druid.server.http.numThreads: 20 druid.indexer.runner.javaOpts: "-server -Xmx2g -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Dhadoop.mapreduce.job.user.classpath.first=true" druid.indexer.task.defaultHadoopCoordinates: ["org.apache.hadoop:hadoop-client:cdh"] diff --git a/modules/druid/manifests/broker.pp b/modules/druid/manifests/broker.pp index 6873b51..815333c 100644 --- a/modules/druid/manifests/broker.pp +++ b/modules/druid/manifests/broker.pp @@ -11,6 +11,7 @@ # [*env*] # Hash of shell environment variables. # Default: { +# 'JAVA_HOME' => '/usr/lib/jvm/java-1.8.0-openjdk-amd64', # 'JMX_PORT' => 9661, # 'DRUID_HEAP_OPTS' => '-Xmx256m -Xms256m', # 'DRUID_EXTRA_JVM_OPTS' => '-XX:MaxDirectMemorySize=256m', @@ -86,6 +87,7 @@ } $default_env = { + 'JAVA_HOME' => $::druid::java_home, 'JMX_PORT' => 9661, 'DRUID_HEAP_OPTS' => '-Xmx256m -Xms256m', 'DRUID_EXTRA_JVM_OPTS' => '-XX:MaxDirectMemorySize=256m', diff --git a/modules/druid/manifests/coordinator.pp b/modules/druid/manifests/coordinator.pp index d54c02d..e571ac1 100644 --- a/modules/druid/manifests/coordinator.pp +++ b/modules/druid/manifests/coordinator.pp @@ -59,6 +59,7 @@ } $default_env = { + 'JAVA_HOME' => $::druid::java_home, 'JMX_PORT' => 9662, 'DRUID_HEAP_OPTS' => '-Xmx128m -Xms128m', 'DRUID_EXTRA_JVM_OPTS' => '-Dderby.stream.error.file=/var/log/druid/coordinator-derby.log', diff --git a/modules/druid/manifests/historical.pp b/modules/druid/manifests/historical.pp index a5cac6a..b02cced 100644 --- a/modules/druid/manifests/historical.pp +++ b/modules/druid/manifests/historical.pp @@ -79,6 +79,7 @@ } $default_env = { + 'JAVA_HOME' => $::druid::java_home, 'JMX_PORT' => 9663, 'DRUID_HEAP_OPTS' => '-Xmx256m -Xms256m', 'DRUID_EXTRA_JVM_OPTS' => '-XX:MaxDirectMemorySize=256m', diff --git a/modules/druid/manifests/init.pp b/modules/druid/manifests/init.pp index 8f901da..d065dad 100644 --- a/modules/druid/manifests/init.pp +++ b/modules/druid/manifests/init.pp @@ -54,6 +54,11 @@ # Hash of runtime.properties # See: Default $properties # +# [*java_home*] +# Path to JAVA_HOME. This will be set for all daemon environemnts in their env.sh files. +# This is done because default Java on Debian (Jessie) systems is Java 7, and Druid 0.10+ +# requires Java 8. +# # [*use_cdh*] # If this is true, the druid::cdh::hadoop::dependencies class # will be included, and deep storage will be configured to use @@ -77,9 +82,9 @@ # List extensions to load. Directories matching these names must exist # in druid.extensions.directory. # Default: [ -# 'druid-histogram', # 'druid-datasketches', -# 'druid-namespace-lookup' +# 'druid-histogram', +# 'druid-lookups-cached-global' # 'mysql-metadata-storage', # only if druid.metadata.storage.type == mysql # 'druid-hdfs-storage', # only if druid.storage.type == hdfs # ], @@ -141,7 +146,8 @@ # class druid( $properties = {}, - $use_cdh = false, + $java_home = '/usr/lib/jvm/java-1.8.0-openjdk-amd64', + $use_cdh = false, ) { if $use_cdh { @@ -220,11 +226,10 @@ $storage_extensions = [] } - $default_extensions = [ - 'druid-histogram', 'druid-datasketches', - 'druid-namespace-lookup', + 'druid-histogram', + 'druid-lookups-cached-global', ] # Get a unique list of extensions to load built up from # the defaults configured here. Note that if diff --git a/modules/druid/manifests/middlemanager.pp b/modules/druid/manifests/middlemanager.pp index b6d4198..6dd63f1 100644 --- a/modules/druid/manifests/middlemanager.pp +++ b/modules/druid/manifests/middlemanager.pp @@ -44,6 +44,9 @@ # See: https://github.com/druid-io/druid/pull/2815 # Default: '-server -Xmx128m -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dhadoop.mapreduce.job.user.classpath.first=true' # +# [*druid.indexer.runner.javaCommand*] +# Path to the Java executable that will be used to launch indexer Peon tasks. +# Default: $::druid::java_home/bin/java # # [*druid.indexer.task.baseTaskDir*] # Base temporary working directory for tasks. @@ -71,6 +74,9 @@ 'druid.worker.capacity' => 3, 'druid.indexer.runner.startPort' => 8100, + # Indexer tasks processes (peons) need to run + # with the same java that druid should use. + 'druid.indexer.runner.javaCommand' => "${::druid::java_home}/bin/java", 'druid.indexer.runner.javaOpts' => '-server -Xmx128m -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dhadoop.mapreduce.job.user.classpath.first=true', 'druid.indexer.task.baseTaskDir' => '/var/lib/druid/task', @@ -82,6 +88,7 @@ } $default_env = { + 'JAVA_HOME' => $::druid::java_home, 'JMX_PORT' => 9664, 'DRUID_HEAP_OPTS' => '-Xmx64m -Xms64m', } diff --git a/modules/druid/manifests/overlord.pp b/modules/druid/manifests/overlord.pp index 1c28e0b..721862d 100644 --- a/modules/druid/manifests/overlord.pp +++ b/modules/druid/manifests/overlord.pp @@ -60,6 +60,7 @@ } $default_env = { + 'JAVA_HOME' => $::druid::java_home, 'JMX_PORT' => 9665, 'DRUID_HEAP_OPTS' => '-Xmx128m -Xms128m', } -- To view, visit https://gerrit.wikimedia.org/r/355471 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ic9a091c3b601dd382b66770baf1e2494fb77eef9 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ottomata <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
