Mobrovac has submitted this change and it was merged. ( 
https://gerrit.wikimedia.org/r/376527 )

Change subject: Initial add of the Scap3 configuration
......................................................................


Initial add of the Scap3 configuration

Bug: T175281
Change-Id: I4b925e234d3720c0c90956d32e27ac611d813195
---
A scap/config-files.yaml
A scap/cpjq
A scap/cpjq-canary
A scap/scap.cfg
A scap/templates/config.yaml.j2
A scap/vars.yaml
6 files changed, 147 insertions(+), 0 deletions(-)

Approvals:
  Mobrovac: Verified; Looks good to me, approved



diff --git a/scap/config-files.yaml b/scap/config-files.yaml
new file mode 100644
index 0000000..0b0ded0
--- /dev/null
+++ b/scap/config-files.yaml
@@ -0,0 +1,5 @@
+---
+/etc/cp-jobqueue/config.yaml:
+  template: config.yaml.j2
+  erb_syntax: True
+  remote_vars: /etc/cp-jobqueue/config-vars.yaml
diff --git a/scap/cpjq b/scap/cpjq
new file mode 100644
index 0000000..d587d2f
--- /dev/null
+++ b/scap/cpjq
@@ -0,0 +1,9 @@
+scb1001.eqiad.wmnet
+scb1002.eqiad.wmnet
+scb1003.eqiad.wmnet
+scb1004.eqiad.wmnet
+scb2002.codfw.wmnet
+scb2003.codfw.wmnet
+scb2004.codfw.wmnet
+scb2005.codfw.wmnet
+scb2006.codfw.wmnet
diff --git a/scap/cpjq-canary b/scap/cpjq-canary
new file mode 100644
index 0000000..49e9dd7
--- /dev/null
+++ b/scap/cpjq-canary
@@ -0,0 +1 @@
+scb2001.codfw.wmnet
diff --git a/scap/scap.cfg b/scap/scap.cfg
new file mode 100644
index 0000000..b2f8ad9
--- /dev/null
+++ b/scap/scap.cfg
@@ -0,0 +1,13 @@
+[global]
+git_repo: cp-jobqueue
+git_deploy_dir: /srv/deployment
+git_repo_user: deploy-service
+ssh_user: deploy-service
+server_groups: canary, default
+canary_dsh_targets: cpjq-canary
+dsh_targets: cpjq
+git_submodules: True
+service_name: cp-jobqueue
+service_port: 7200
+lock_file: /tmp/scap.cpjq.lock
+config_deploy: True
diff --git a/scap/templates/config.yaml.j2 b/scap/templates/config.yaml.j2
new file mode 100644
index 0000000..8031b95
--- /dev/null
+++ b/scap/templates/config.yaml.j2
@@ -0,0 +1,98 @@
+spec: &spec
+  x-sub-request-filters:
+    - type: default
+      name: http
+      options:
+        allow:
+          - pattern: /^https?:\/\//
+            forward_headers:
+              user-agent: true
+  title: The Change Propagation for JobQueue root
+  paths:
+<%- if env == 'production' %>
+    /{api:sys}/dedupe:
+      x-modules:
+        - path: src/sys/deduplicator.js
+          options:
+            redis:
+              path: <%= redis_path %>
+              password: <%= redis_pass if redis_pass else '' %>
+<%- endif %>
+    /{api:sys}/queue:
+      x-modules:
+        - path: src/sys/kafka.js
+          options:
+            metadata_broker_list: <%= broker_list %>
+            dc_name: <%= site %>
+            consumer:
+              # JobQueue jobs might sent messages larget then 1 Meg,
+              # so we increase the max message size in kafka and have to
+              # ajust the consumers accordingly.
+              fetch.message.max.bytes: <%= kafka_max_bytes %>
+            concurrency: <%= concurrency %>
+            startup_delay: 60000
+            # no rules for first deployment
+            templates: {}
+
+
+# Number of worker processes to spawn.
+# Set to 0 to run everything in a single process without clustering.
+# Use 'ncpu' to run as many workers as there are CPU units
+num_workers: ncpu
+
+# Log error messages and gracefully restart a worker if v8 reports that it
+# uses more heap (note: not RSS) than this many mb.
+worker_heap_limit_mb: 750
+
+# The maximum interval in ms that can pass between two beat messages
+# sent by each worker to the master before it is killed
+worker_heartbeat_timeout: 15000
+
+# Logger info
+logging:
+  level: warn
+  name: <%= log_name %>
+  streams:
+    - host: <%= logstash_host %>
+      port: <%= logstash_port %>
+      type: gelf
+    - level: info
+      path: <%= log_file %>
+      type: file
+  sampled_levels:
+    trace/dedupe: 0.01
+
+# Statsd metrics reporter
+metrics:
+  name: <%= metrics_name %>
+  host: <%= metrics_host %>
+  port: <%= metrics_port %>
+  type: statsd
+
+services:
+  - name: <%= name %>
+    # a relative path or the name of an npm package, if different from name
+    module: hyperswitch
+    # per-service config
+    conf:
+      cors: "*"
+      port: <%= port %>
+      # interface: localhost # uncomment to only listen on localhost
+      # URL of the outbound proxy to use (complete with protocol)
+      proxy: <%= proxy if proxy else '' %>
+      # the list of domains for which not to use the proxy defined above
+      # no_proxy_list:
+      #   - domain1.com
+      #   - domain2.org
+      # the list of incoming request headers that can be logged; if left empty,
+      # the following headers are allowed: cache-control, content-length,
+      # content-type, if-match, user-agent, x-request-id
+      # log_header_whitelist:
+      #   - cache-control
+      #   - content-length
+      #   - content-type
+      #   - if-match
+      #   - user-agent
+      #   - x-request-id
+      user_agent: ChangePropagation-JobQueue/WMF
+      spec: *spec
diff --git a/scap/vars.yaml b/scap/vars.yaml
new file mode 100644
index 0000000..02c639a
--- /dev/null
+++ b/scap/vars.yaml
@@ -0,0 +1,21 @@
+# log-related variables
+log_name: cp-jobqueue
+logstash_host: localhost
+logstash_port: 12201
+log_file: /tmp/cpjq.log
+# metrics
+metrics_name: cp-jobqueue
+metrics_host: localhost
+metrics_port: 8252
+# service
+name: cp-jobqueue
+port: 7200
+proxy:
+site: datacenter1
+broker_list: localhost:9092
+jobrunner_uri: http://localhost/rpc/RunSingleJob.php
+concurrency: 50
+redis_path: /var/run/redis.sock
+redis_pass:
+env: production
+kafka_max_bytes: 4194304

-- 
To view, visit https://gerrit.wikimedia.org/r/376527
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I4b925e234d3720c0c90956d32e27ac611d813195
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/cp-jobqueue
Gerrit-Branch: master
Gerrit-Owner: Mobrovac <mobro...@wikimedia.org>
Gerrit-Reviewer: Mobrovac <mobro...@wikimedia.org>
Gerrit-Reviewer: Ppchelko <ppche...@wikimedia.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to