Gehel has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/369682 )
Change subject: [WIP] wdqs - moving to role / profiles
......................................................................
[WIP] wdqs - moving to role / profiles
Change-Id: Id2adbd2cd86e61831a16f7abbb73135f47352bc0
---
M hieradata/labs/wikidata-query/common.yaml
M hieradata/role/codfw/wdqs.yaml
M hieradata/role/common/wdqs.yaml
M manifests/site.pp
A modules/profile/manifests/wdqs.pp
M modules/role/manifests/wdqs.pp
A modules/role/manifests/wdqs/labs.pp
M modules/wdqs/manifests/init.pp
M modules/wdqs/manifests/service.pp
M modules/wdqs/manifests/updater.pp
10 files changed, 82 insertions(+), 57 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/82/369682/1
diff --git a/hieradata/labs/wikidata-query/common.yaml
b/hieradata/labs/wikidata-query/common.yaml
index bc94915..d6ce9c1 100644
--- a/hieradata/labs/wikidata-query/common.yaml
+++ b/hieradata/labs/wikidata-query/common.yaml
@@ -1,7 +1,9 @@
-"wdqs::gui::log_aggregator":
'deployment-logstash2.deployment-prep.eqiad.wmflabs:10514'
-"wdqs::updater::options": '-n wdq -s'
-"wdqs::use_git_deploy": false
-"wdqs::package_dir": '/srv/wdqs/blazegraph'
-"wdqs::data_dir": '/srv/wdqs/blazegraph'
-"wdqs::endpoint": ''
-"wdqs::blazegraph_heap_size": '8g'
+profile::wdqs::log_aggregator:
'deployment-logstash2.deployment-prep.eqiad.wmflabs:10514'
+profile::wdqs::use_git_deploy: false
+profile::wdqs::package_dir: '/srv/wdqs/blazegraph'
+profile::wdqs::data_dir: '/srv/wdqs/blazegraph'
+profile::wdqs::endpoint: ''
+profile::wdqs::updater_options: '-n wdq -s'
+profile::wdqs::config_file: 'RWStore.properties'
+profile::wdqs::blazegraph_options: ''
+profile::wdqs::blazegraph_heap_size: '8g'
diff --git a/hieradata/role/codfw/wdqs.yaml b/hieradata/role/codfw/wdqs.yaml
index 9083c4b..57d9e1e 100644
--- a/hieradata/role/codfw/wdqs.yaml
+++ b/hieradata/role/codfw/wdqs.yaml
@@ -1,3 +1,3 @@
lvs::realserver::realserver_ips:
- '10.2.1.32' # wdqs.svc.codfw.wmnet
-wdqs::blazegraph_options: '-Dhttp.proxyHost=webproxy.codfw.wmnet
-Dhttp.proxyPort=8080 -XX:+ExitOnOutOfMemoryError'
+profile::wdqs::blazegraph_options: '-Dhttp.proxyHost=webproxy.codfw.wmnet
-Dhttp.proxyPort=8080 -XX:+ExitOnOutOfMemoryError'
diff --git a/hieradata/role/common/wdqs.yaml b/hieradata/role/common/wdqs.yaml
index 1dbf8d2..1012c03 100644
--- a/hieradata/role/common/wdqs.yaml
+++ b/hieradata/role/common/wdqs.yaml
@@ -1,7 +1,11 @@
-wdqs::gui::log_aggregator: 'logstash1001.eqiad.wmnet:10514'
-wdqs::updater::options: '-n wdq -- -b 500 -T 1200'
-wdqs::service::config_file: /etc/wdqs/RWStore.properties
-wdqs::endpoint: 'https://query.wikidata.org'
+profile::wdqs::log_aggregator: 'logstash1001.eqiad.wmnet:10514'
+profile::wdqs::use_git_deploy: true
+profile::wdqs::package_dir: '/srv/deployment/wdqs/wdqs'
+profile::wdqs::data_dir: '/srv/wdqs'
+profile::wdqs::endpoint: 'https://query.wikidata.org'
+profile::wdqs::updater_options: '-n wdq -- -b 500 -T 1200'
+profile::wdqs::config_file: '/etc/wdqs/RWStore.properties'
+profile::wdqs::blazegraph_heap_size: '16g'
admin::groups:
- wdqs-admins
cluster: wdqs
diff --git a/manifests/site.pp b/manifests/site.pp
index 8f559b5..8d4dc8e 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -2323,12 +2323,10 @@
# Wikidata query service
node /^wdqs100[1-3]\.eqiad\.wmnet$/ {
role(wdqs)
- include ::lvs::realserver
}
node /^wdqs200[1-3]\.codfw\.wmnet$/ {
role(wdqs)
- include ::lvs::realserver
}
node 'wezen.codfw.wmnet' {
diff --git a/modules/profile/manifests/wdqs.pp
b/modules/profile/manifests/wdqs.pp
new file mode 100644
index 0000000..862b2eb
--- /dev/null
+++ b/modules/profile/manifests/wdqs.pp
@@ -0,0 +1,47 @@
+class profile::wdqs (
+ $log_aggregator = hiera('profile::wdqs::log_aggregator'),
+ $use_git_deploy = hiera('profile::wdqs::use_git_deploy'),
+ $package_dir = hiera('profile::wdqs::package_dir'),
+ $data_dir = hiera('profile::wdqs::data_dir'),
+ $endpoint = hiera('profile::wdqs::endpoint'),
+ $blazegraph_options = hiera('profile::wdqs::blazegraph_options'),
+ $blazegraph_heap_size = hiera('profile::wdqs::blazegraph_heap_size'),
+ $blazegraph_config_file = hiera('profile::wdqs::blazegraph_config_file'),
+) {
+ $nagios_contact_group = 'admins,wdqs-admins'
+
+ # Install services - both blazegraph and the updater
+ class { '::wdqs':
+ use_git_deploy => false,
+ package_dir => $package_dir,
+ data_dir => $data_dir,
+ endpoint => $endpoint,
+ blazegraph_options => $blazegraph_options,
+ blazegraph_heap_size => $blazegraph_heap_size,
+ blazegraph_config_file => $blazegraph_config_file,
+ }
+
+ # Service Web proxy
+ class { '::wdqs::gui':
+ log_aggregator => $log_aggregator,
+ }
+
+ # Firewall
+ ferm::service {
+ 'wdqs_http':
+ proto => 'tcp',
+ port => '80';
+ 'wdqs_https':
+ proto => 'tcp',
+ port => '443';
+ 'wdqs_internal_http':
+ proto => 'tcp',
+ port => '8888',
+ srange => '$DOMAIN_NETWORKS';
+ }
+
+ # Monitoring
+ class { '::wdqs::monitor::blazegraph': }
+ class { '::wdqs::monitor::updater': }
+ class { '::wdqs::monitor::services': }
+}
\ No newline at end of file
diff --git a/modules/role/manifests/wdqs.pp b/modules/role/manifests/wdqs.pp
index a58de6b..be77238 100644
--- a/modules/role/manifests/wdqs.pp
+++ b/modules/role/manifests/wdqs.pp
@@ -6,47 +6,11 @@
class role::wdqs {
include ::standard
include ::base::firewall
-
- $nagios_contact_group = 'admins,wdqs-admins'
-
- if $::realm == 'labs' {
- include role::labs::lvm::srv
- }
+ include ::role::lvs::realserver
+ include ::profile::wdqs
system::role { 'wdqs':
ensure => 'present',
description => 'Wikidata Query Service',
}
-
-
- # Install services - both blazegraph and the updater
- include ::wdqs
-
- # Service Web proxy
- include ::wdqs::gui
-
- ferm::service { 'wdqs_http':
- proto => 'tcp',
- port => '80',
- }
-
- ferm::service { 'wdqs_https':
- proto => 'tcp',
- port => '443',
- }
-
- ferm::service { 'wdqs_internal_http':
- proto => 'tcp',
- port => '8888',
- srange => '$DOMAIN_NETWORKS',
- }
-
- # Monitor Blazegraph
- include ::wdqs::monitor::blazegraph
-
- # Monitor Updater
- include ::wdqs::monitor::updater
-
- # Service monitoring
- include ::wdqs::monitor::services
}
diff --git a/modules/role/manifests/wdqs/labs.pp
b/modules/role/manifests/wdqs/labs.pp
new file mode 100644
index 0000000..8f83d98
--- /dev/null
+++ b/modules/role/manifests/wdqs/labs.pp
@@ -0,0 +1,4 @@
+class role::wdqs::labs () {
+ require role::labs::lvm::srv
+ require role::wdqs
+}
\ No newline at end of file
diff --git a/modules/wdqs/manifests/init.pp b/modules/wdqs/manifests/init.pp
index b035791..549b439 100644
--- a/modules/wdqs/manifests/init.pp
+++ b/modules/wdqs/manifests/init.pp
@@ -10,7 +10,8 @@
# - $log_dir: Directory where the logs go
# - $endpoint: External endpoint name
# - $blazegraph_heap_size: heapsize for blazegraph
-# - $blazegraph_options: options for Blazegraph statrup script
+# - $blazegraph_options: options for Blazegraph startup script
+# - $updater_options: options for updater startup script
class wdqs(
$use_git_deploy = true,
$username = 'blazegraph',
@@ -18,8 +19,10 @@
$data_dir = '/srv/wdqs',
$log_dir = '/var/log/wdqs',
$endpoint = '',
- $blazegraph_heap_size = '16g',
$blazegraph_options = '',
+ $blazegraph_heap_size = '16g',
+ $blazegraph_config_file = 'RWStore.properties',
+ $updater_options = '-n wdq -s',
) {
$deploy_user = 'deploy-service'
@@ -43,6 +46,7 @@
deploy_user => $deploy_user,
package_dir => $package_dir,
username => $username,
+ config_file => $blazegraph_config_file,
}
file { $log_dir:
@@ -110,7 +114,9 @@
}
# WDQS Updater service
- include wdqs::updater
+ class { 'wdqs::updater':
+ options => $updater_options,
+ }
# GC logs rotation is done by the JVM, but on JVM restart, the logs left by
# the previous instance are left alone. This cron takes care of cleaning up
diff --git a/modules/wdqs/manifests/service.pp
b/modules/wdqs/manifests/service.pp
index bcda337..f9721fd 100644
--- a/modules/wdqs/manifests/service.pp
+++ b/modules/wdqs/manifests/service.pp
@@ -6,7 +6,7 @@
$deploy_user,
$package_dir,
$username,
- $config_file='RWStore.properties',
+ $config_file,
) {
include ::wdqs::packages
diff --git a/modules/wdqs/manifests/updater.pp
b/modules/wdqs/manifests/updater.pp
index b34ac82..71c0989 100644
--- a/modules/wdqs/manifests/updater.pp
+++ b/modules/wdqs/manifests/updater.pp
@@ -3,7 +3,7 @@
# Wikidata Query Service updater service.
#
class wdqs::updater(
- $options = '-n wdq -s',
+ $options,
$package_dir = $::wdqs::package_dir,
$username = $::wdqs::username,
){
--
To view, visit https://gerrit.wikimedia.org/r/369682
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Id2adbd2cd86e61831a16f7abbb73135f47352bc0
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Gehel <guillaume.leder...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
