[MediaWiki-commits] [Gerrit] mediawiki...cp-jobqueue[master]: Initial add of the Scap3 configuration
Mobrovac has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/376527 )
Change subject: Initial add of the Scap3 configuration
..
Initial add of the Scap3 configuration
Bug: T175281
Change-Id: I4b925e234d3720c0c90956d32e27ac611d813195
---
A scap/config-files.yaml
A scap/cpjq
A scap/cpjq-canary
A scap/scap.cfg
A scap/templates/config.yaml.j2
A scap/vars.yaml
6 files changed, 147 insertions(+), 0 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/scap/config-files.yaml b/scap/config-files.yaml
new file mode 100644
index 000..0b0ded0
--- /dev/null
+++ b/scap/config-files.yaml
@@ -0,0 +1,5 @@
+---
+/etc/cp-jobqueue/config.yaml:
+ template: config.yaml.j2
+ erb_syntax: True
+ remote_vars: /etc/cp-jobqueue/config-vars.yaml
diff --git a/scap/cpjq b/scap/cpjq
new file mode 100644
index 000..d587d2f
--- /dev/null
+++ b/scap/cpjq
@@ -0,0 +1,9 @@
+scb1001.eqiad.wmnet
+scb1002.eqiad.wmnet
+scb1003.eqiad.wmnet
+scb1004.eqiad.wmnet
+scb2002.codfw.wmnet
+scb2003.codfw.wmnet
+scb2004.codfw.wmnet
+scb2005.codfw.wmnet
+scb2006.codfw.wmnet
diff --git a/scap/cpjq-canary b/scap/cpjq-canary
new file mode 100644
index 000..49e9dd7
--- /dev/null
+++ b/scap/cpjq-canary
@@ -0,0 +1 @@
+scb2001.codfw.wmnet
diff --git a/scap/scap.cfg b/scap/scap.cfg
new file mode 100644
index 000..b2f8ad9
--- /dev/null
+++ b/scap/scap.cfg
@@ -0,0 +1,13 @@
+[global]
+git_repo: cp-jobqueue
+git_deploy_dir: /srv/deployment
+git_repo_user: deploy-service
+ssh_user: deploy-service
+server_groups: canary, default
+canary_dsh_targets: cpjq-canary
+dsh_targets: cpjq
+git_submodules: True
+service_name: cp-jobqueue
+service_port: 7200
+lock_file: /tmp/scap.cpjq.lock
+config_deploy: True
diff --git a/scap/templates/config.yaml.j2 b/scap/templates/config.yaml.j2
new file mode 100644
index 000..8031b95
--- /dev/null
+++ b/scap/templates/config.yaml.j2
@@ -0,0 +1,98 @@
+spec:
+ x-sub-request-filters:
+- type: default
+ name: http
+ options:
+allow:
+ - pattern: /^https?:\/\//
+forward_headers:
+ user-agent: true
+ title: The Change Propagation for JobQueue root
+ paths:
+<%- if env == 'production' %>
+/{api:sys}/dedupe:
+ x-modules:
+- path: src/sys/deduplicator.js
+ options:
+redis:
+ path: <%= redis_path %>
+ password: <%= redis_pass if redis_pass else '' %>
+<%- endif %>
+/{api:sys}/queue:
+ x-modules:
+- path: src/sys/kafka.js
+ options:
+metadata_broker_list: <%= broker_list %>
+dc_name: <%= site %>
+consumer:
+ # JobQueue jobs might sent messages larget then 1 Meg,
+ # so we increase the max message size in kafka and have to
+ # ajust the consumers accordingly.
+ fetch.message.max.bytes: <%= kafka_max_bytes %>
+concurrency: <%= concurrency %>
+startup_delay: 6
+# no rules for first deployment
+templates: {}
+
+
+# Number of worker processes to spawn.
+# Set to 0 to run everything in a single process without clustering.
+# Use 'ncpu' to run as many workers as there are CPU units
+num_workers: ncpu
+
+# Log error messages and gracefully restart a worker if v8 reports that it
+# uses more heap (note: not RSS) than this many mb.
+worker_heap_limit_mb: 750
+
+# The maximum interval in ms that can pass between two beat messages
+# sent by each worker to the master before it is killed
+worker_heartbeat_timeout: 15000
+
+# Logger info
+logging:
+ level: warn
+ name: <%= log_name %>
+ streams:
+- host: <%= logstash_host %>
+ port: <%= logstash_port %>
+ type: gelf
+- level: info
+ path: <%= log_file %>
+ type: file
+ sampled_levels:
+trace/dedupe: 0.01
+
+# Statsd metrics reporter
+metrics:
+ name: <%= metrics_name %>
+ host: <%= metrics_host %>
+ port: <%= metrics_port %>
+ type: statsd
+
+services:
+ - name: <%= name %>
+# a relative path or the name of an npm package, if different from name
+module: hyperswitch
+# per-service config
+conf:
+ cors: "*"
+ port: <%= port %>
+ # interface: localhost # uncomment to only listen on localhost
+ # URL of the outbound proxy to use (complete with protocol)
+ proxy: <%= proxy if proxy else '' %>
+ # the list of domains for which not to use the proxy defined above
+ # no_proxy_list:
+ # - domain1.com
+ # - domain2.org
+ # the list of incoming request headers that can be logged; if left empty,
+ # the following headers are allowed: cache-control, content-length,
+ # content-type, if-match, user-agent, x-request-id
+ # log_header_whitelist:
+ # - cache-control
+ # - content-length
+ # - content-type
+ # - if-match
+ # -
[MediaWiki-commits] [Gerrit] mediawiki...cp-jobqueue[master]: Initial add of the Scap3 configuration
Mobrovac has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/376527 )
Change subject: Initial add of the Scap3 configuration
..
Initial add of the Scap3 configuration
Bug: T175281
Change-Id: I4b925e234d3720c0c90956d32e27ac611d813195
---
A scap/config-files.yaml
A scap/cpjq
A scap/cpjq-canary
A scap/scap.cfg
A scap/templates/config.yaml.j2
A scap/vars.yaml
6 files changed, 147 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/cp-jobqueue
refs/changes/27/376527/1
diff --git a/scap/config-files.yaml b/scap/config-files.yaml
new file mode 100644
index 000..0b0ded0
--- /dev/null
+++ b/scap/config-files.yaml
@@ -0,0 +1,5 @@
+---
+/etc/cp-jobqueue/config.yaml:
+ template: config.yaml.j2
+ erb_syntax: True
+ remote_vars: /etc/cp-jobqueue/config-vars.yaml
diff --git a/scap/cpjq b/scap/cpjq
new file mode 100644
index 000..d587d2f
--- /dev/null
+++ b/scap/cpjq
@@ -0,0 +1,9 @@
+scb1001.eqiad.wmnet
+scb1002.eqiad.wmnet
+scb1003.eqiad.wmnet
+scb1004.eqiad.wmnet
+scb2002.codfw.wmnet
+scb2003.codfw.wmnet
+scb2004.codfw.wmnet
+scb2005.codfw.wmnet
+scb2006.codfw.wmnet
diff --git a/scap/cpjq-canary b/scap/cpjq-canary
new file mode 100644
index 000..49e9dd7
--- /dev/null
+++ b/scap/cpjq-canary
@@ -0,0 +1 @@
+scb2001.codfw.wmnet
diff --git a/scap/scap.cfg b/scap/scap.cfg
new file mode 100644
index 000..b2f8ad9
--- /dev/null
+++ b/scap/scap.cfg
@@ -0,0 +1,13 @@
+[global]
+git_repo: cp-jobqueue
+git_deploy_dir: /srv/deployment
+git_repo_user: deploy-service
+ssh_user: deploy-service
+server_groups: canary, default
+canary_dsh_targets: cpjq-canary
+dsh_targets: cpjq
+git_submodules: True
+service_name: cp-jobqueue
+service_port: 7200
+lock_file: /tmp/scap.cpjq.lock
+config_deploy: True
diff --git a/scap/templates/config.yaml.j2 b/scap/templates/config.yaml.j2
new file mode 100644
index 000..8031b95
--- /dev/null
+++ b/scap/templates/config.yaml.j2
@@ -0,0 +1,98 @@
+spec:
+ x-sub-request-filters:
+- type: default
+ name: http
+ options:
+allow:
+ - pattern: /^https?:\/\//
+forward_headers:
+ user-agent: true
+ title: The Change Propagation for JobQueue root
+ paths:
+<%- if env == 'production' %>
+/{api:sys}/dedupe:
+ x-modules:
+- path: src/sys/deduplicator.js
+ options:
+redis:
+ path: <%= redis_path %>
+ password: <%= redis_pass if redis_pass else '' %>
+<%- endif %>
+/{api:sys}/queue:
+ x-modules:
+- path: src/sys/kafka.js
+ options:
+metadata_broker_list: <%= broker_list %>
+dc_name: <%= site %>
+consumer:
+ # JobQueue jobs might sent messages larget then 1 Meg,
+ # so we increase the max message size in kafka and have to
+ # ajust the consumers accordingly.
+ fetch.message.max.bytes: <%= kafka_max_bytes %>
+concurrency: <%= concurrency %>
+startup_delay: 6
+# no rules for first deployment
+templates: {}
+
+
+# Number of worker processes to spawn.
+# Set to 0 to run everything in a single process without clustering.
+# Use 'ncpu' to run as many workers as there are CPU units
+num_workers: ncpu
+
+# Log error messages and gracefully restart a worker if v8 reports that it
+# uses more heap (note: not RSS) than this many mb.
+worker_heap_limit_mb: 750
+
+# The maximum interval in ms that can pass between two beat messages
+# sent by each worker to the master before it is killed
+worker_heartbeat_timeout: 15000
+
+# Logger info
+logging:
+ level: warn
+ name: <%= log_name %>
+ streams:
+- host: <%= logstash_host %>
+ port: <%= logstash_port %>
+ type: gelf
+- level: info
+ path: <%= log_file %>
+ type: file
+ sampled_levels:
+trace/dedupe: 0.01
+
+# Statsd metrics reporter
+metrics:
+ name: <%= metrics_name %>
+ host: <%= metrics_host %>
+ port: <%= metrics_port %>
+ type: statsd
+
+services:
+ - name: <%= name %>
+# a relative path or the name of an npm package, if different from name
+module: hyperswitch
+# per-service config
+conf:
+ cors: "*"
+ port: <%= port %>
+ # interface: localhost # uncomment to only listen on localhost
+ # URL of the outbound proxy to use (complete with protocol)
+ proxy: <%= proxy if proxy else '' %>
+ # the list of domains for which not to use the proxy defined above
+ # no_proxy_list:
+ # - domain1.com
+ # - domain2.org
+ # the list of incoming request headers that can be logged; if left empty,
+ # the following headers are allowed: cache-control, content-length,
+ # content-type, if-match, user-agent, x-request-id
+ # log_header_whitelist:
+ # - cache-control
+ # - content-length
+ # - content-type
+ # -
