Mobrovac has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/376527 )
Change subject: Initial add of the Scap3 configuration
......................................................................
Initial add of the Scap3 configuration
Bug: T175281
Change-Id: I4b925e234d3720c0c90956d32e27ac611d813195
---
A scap/config-files.yaml
A scap/cpjq
A scap/cpjq-canary
A scap/scap.cfg
A scap/templates/config.yaml.j2
A scap/vars.yaml
6 files changed, 147 insertions(+), 0 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/scap/config-files.yaml b/scap/config-files.yaml
new file mode 100644
index 0000000..0b0ded0
--- /dev/null
+++ b/scap/config-files.yaml
@@ -0,0 +1,5 @@
+---
+/etc/cp-jobqueue/config.yaml:
+ template: config.yaml.j2
+ erb_syntax: True
+ remote_vars: /etc/cp-jobqueue/config-vars.yaml
diff --git a/scap/cpjq b/scap/cpjq
new file mode 100644
index 0000000..d587d2f
--- /dev/null
+++ b/scap/cpjq
@@ -0,0 +1,9 @@
+scb1001.eqiad.wmnet
+scb1002.eqiad.wmnet
+scb1003.eqiad.wmnet
+scb1004.eqiad.wmnet
+scb2002.codfw.wmnet
+scb2003.codfw.wmnet
+scb2004.codfw.wmnet
+scb2005.codfw.wmnet
+scb2006.codfw.wmnet
diff --git a/scap/cpjq-canary b/scap/cpjq-canary
new file mode 100644
index 0000000..49e9dd7
--- /dev/null
+++ b/scap/cpjq-canary
@@ -0,0 +1 @@
+scb2001.codfw.wmnet
diff --git a/scap/scap.cfg b/scap/scap.cfg
new file mode 100644
index 0000000..b2f8ad9
--- /dev/null
+++ b/scap/scap.cfg
@@ -0,0 +1,13 @@
+[global]
+git_repo: cp-jobqueue
+git_deploy_dir: /srv/deployment
+git_repo_user: deploy-service
+ssh_user: deploy-service
+server_groups: canary, default
+canary_dsh_targets: cpjq-canary
+dsh_targets: cpjq
+git_submodules: True
+service_name: cp-jobqueue
+service_port: 7200
+lock_file: /tmp/scap.cpjq.lock
+config_deploy: True
diff --git a/scap/templates/config.yaml.j2 b/scap/templates/config.yaml.j2
new file mode 100644
index 0000000..8031b95
--- /dev/null
+++ b/scap/templates/config.yaml.j2
@@ -0,0 +1,98 @@
+spec: &spec
+ x-sub-request-filters:
+ - type: default
+ name: http
+ options:
+ allow:
+ - pattern: /^https?:\/\//
+ forward_headers:
+ user-agent: true
+ title: The Change Propagation for JobQueue root
+ paths:
+<%- if env == 'production' %>
+ /{api:sys}/dedupe:
+ x-modules:
+ - path: src/sys/deduplicator.js
+ options:
+ redis:
+ path: <%= redis_path %>
+ password: <%= redis_pass if redis_pass else '' %>
+<%- endif %>
+ /{api:sys}/queue:
+ x-modules:
+ - path: src/sys/kafka.js
+ options:
+ metadata_broker_list: <%= broker_list %>
+ dc_name: <%= site %>
+ consumer:
+ # JobQueue jobs might sent messages larget then 1 Meg,
+ # so we increase the max message size in kafka and have to
+ # ajust the consumers accordingly.
+ fetch.message.max.bytes: <%= kafka_max_bytes %>
+ concurrency: <%= concurrency %>
+ startup_delay: 60000
+ # no rules for first deployment
+ templates: {}
+
+
+# Number of worker processes to spawn.
+# Set to 0 to run everything in a single process without clustering.
+# Use 'ncpu' to run as many workers as there are CPU units
+num_workers: ncpu
+
+# Log error messages and gracefully restart a worker if v8 reports that it
+# uses more heap (note: not RSS) than this many mb.
+worker_heap_limit_mb: 750
+
+# The maximum interval in ms that can pass between two beat messages
+# sent by each worker to the master before it is killed
+worker_heartbeat_timeout: 15000
+
+# Logger info
+logging:
+ level: warn
+ name: <%= log_name %>
+ streams:
+ - host: <%= logstash_host %>
+ port: <%= logstash_port %>
+ type: gelf
+ - level: info
+ path: <%= log_file %>
+ type: file
+ sampled_levels:
+ trace/dedupe: 0.01
+
+# Statsd metrics reporter
+metrics:
+ name: <%= metrics_name %>
+ host: <%= metrics_host %>
+ port: <%= metrics_port %>
+ type: statsd
+
+services:
+ - name: <%= name %>
+ # a relative path or the name of an npm package, if different from name
+ module: hyperswitch
+ # per-service config
+ conf:
+ cors: "*"
+ port: <%= port %>
+ # interface: localhost # uncomment to only listen on localhost
+ # URL of the outbound proxy to use (complete with protocol)
+ proxy: <%= proxy if proxy else '' %>
+ # the list of domains for which not to use the proxy defined above
+ # no_proxy_list:
+ # - domain1.com
+ # - domain2.org
+ # the list of incoming request headers that can be logged; if left empty,
+ # the following headers are allowed: cache-control, content-length,
+ # content-type, if-match, user-agent, x-request-id
+ # log_header_whitelist:
+ # - cache-control
+ # - content-length
+ # - content-type
+ # - if-match
+ # - user-agent
+ # - x-request-id
+ user_agent: ChangePropagation-JobQueue/WMF
+ spec: *spec
diff --git a/scap/vars.yaml b/scap/vars.yaml
new file mode 100644
index 0000000..02c639a
--- /dev/null
+++ b/scap/vars.yaml
@@ -0,0 +1,21 @@
+# log-related variables
+log_name: cp-jobqueue
+logstash_host: localhost
+logstash_port: 12201
+log_file: /tmp/cpjq.log
+# metrics
+metrics_name: cp-jobqueue
+metrics_host: localhost
+metrics_port: 8252
+# service
+name: cp-jobqueue
+port: 7200
+proxy:
+site: datacenter1
+broker_list: localhost:9092
+jobrunner_uri: http://localhost/rpc/RunSingleJob.php
+concurrency: 50
+redis_path: /var/run/redis.sock
+redis_pass:
+env: production
+kafka_max_bytes: 4194304
--
To view, visit https://gerrit.wikimedia.org/r/376527
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I4b925e234d3720c0c90956d32e27ac611d813195
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/cp-jobqueue
Gerrit-Branch: master
Gerrit-Owner: Mobrovac <mobro...@wikimedia.org>
Gerrit-Reviewer: Mobrovac <mobro...@wikimedia.org>
Gerrit-Reviewer: Ppchelko <ppche...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
