Mobrovac has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/376527 )
Change subject: Initial add of the Scap3 configuration ...................................................................... Initial add of the Scap3 configuration Bug: T175281 Change-Id: I4b925e234d3720c0c90956d32e27ac611d813195 --- A scap/config-files.yaml A scap/cpjq A scap/cpjq-canary A scap/scap.cfg A scap/templates/config.yaml.j2 A scap/vars.yaml 6 files changed, 147 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/cp-jobqueue refs/changes/27/376527/1 diff --git a/scap/config-files.yaml b/scap/config-files.yaml new file mode 100644 index 0000000..0b0ded0 --- /dev/null +++ b/scap/config-files.yaml @@ -0,0 +1,5 @@ +--- +/etc/cp-jobqueue/config.yaml: + template: config.yaml.j2 + erb_syntax: True + remote_vars: /etc/cp-jobqueue/config-vars.yaml diff --git a/scap/cpjq b/scap/cpjq new file mode 100644 index 0000000..d587d2f --- /dev/null +++ b/scap/cpjq @@ -0,0 +1,9 @@ +scb1001.eqiad.wmnet +scb1002.eqiad.wmnet +scb1003.eqiad.wmnet +scb1004.eqiad.wmnet +scb2002.codfw.wmnet +scb2003.codfw.wmnet +scb2004.codfw.wmnet +scb2005.codfw.wmnet +scb2006.codfw.wmnet diff --git a/scap/cpjq-canary b/scap/cpjq-canary new file mode 100644 index 0000000..49e9dd7 --- /dev/null +++ b/scap/cpjq-canary @@ -0,0 +1 @@ +scb2001.codfw.wmnet diff --git a/scap/scap.cfg b/scap/scap.cfg new file mode 100644 index 0000000..b2f8ad9 --- /dev/null +++ b/scap/scap.cfg @@ -0,0 +1,13 @@ +[global] +git_repo: cp-jobqueue +git_deploy_dir: /srv/deployment +git_repo_user: deploy-service +ssh_user: deploy-service +server_groups: canary, default +canary_dsh_targets: cpjq-canary +dsh_targets: cpjq +git_submodules: True +service_name: cp-jobqueue +service_port: 7200 +lock_file: /tmp/scap.cpjq.lock +config_deploy: True diff --git a/scap/templates/config.yaml.j2 b/scap/templates/config.yaml.j2 new file mode 100644 index 0000000..8031b95 --- /dev/null +++ b/scap/templates/config.yaml.j2 @@ -0,0 +1,98 @@ +spec: &spec + x-sub-request-filters: + - type: default + name: http + options: + allow: + - pattern: /^https?:\/\// + forward_headers: + user-agent: true + title: The Change Propagation for JobQueue root + paths: +<%- if env == 'production' %> + /{api:sys}/dedupe: + x-modules: + - path: src/sys/deduplicator.js + options: + redis: + path: <%= redis_path %> + password: <%= redis_pass if redis_pass else '' %> +<%- endif %> + /{api:sys}/queue: + x-modules: + - path: src/sys/kafka.js + options: + metadata_broker_list: <%= broker_list %> + dc_name: <%= site %> + consumer: + # JobQueue jobs might sent messages larget then 1 Meg, + # so we increase the max message size in kafka and have to + # ajust the consumers accordingly. + fetch.message.max.bytes: <%= kafka_max_bytes %> + concurrency: <%= concurrency %> + startup_delay: 60000 + # no rules for first deployment + templates: {} + + +# Number of worker processes to spawn. +# Set to 0 to run everything in a single process without clustering. +# Use 'ncpu' to run as many workers as there are CPU units +num_workers: ncpu + +# Log error messages and gracefully restart a worker if v8 reports that it +# uses more heap (note: not RSS) than this many mb. +worker_heap_limit_mb: 750 + +# The maximum interval in ms that can pass between two beat messages +# sent by each worker to the master before it is killed +worker_heartbeat_timeout: 15000 + +# Logger info +logging: + level: warn + name: <%= log_name %> + streams: + - host: <%= logstash_host %> + port: <%= logstash_port %> + type: gelf + - level: info + path: <%= log_file %> + type: file + sampled_levels: + trace/dedupe: 0.01 + +# Statsd metrics reporter +metrics: + name: <%= metrics_name %> + host: <%= metrics_host %> + port: <%= metrics_port %> + type: statsd + +services: + - name: <%= name %> + # a relative path or the name of an npm package, if different from name + module: hyperswitch + # per-service config + conf: + cors: "*" + port: <%= port %> + # interface: localhost # uncomment to only listen on localhost + # URL of the outbound proxy to use (complete with protocol) + proxy: <%= proxy if proxy else '' %> + # the list of domains for which not to use the proxy defined above + # no_proxy_list: + # - domain1.com + # - domain2.org + # the list of incoming request headers that can be logged; if left empty, + # the following headers are allowed: cache-control, content-length, + # content-type, if-match, user-agent, x-request-id + # log_header_whitelist: + # - cache-control + # - content-length + # - content-type + # - if-match + # - user-agent + # - x-request-id + user_agent: ChangePropagation-JobQueue/WMF + spec: *spec diff --git a/scap/vars.yaml b/scap/vars.yaml new file mode 100644 index 0000000..02c639a --- /dev/null +++ b/scap/vars.yaml @@ -0,0 +1,21 @@ +# log-related variables +log_name: cp-jobqueue +logstash_host: localhost +logstash_port: 12201 +log_file: /tmp/cpjq.log +# metrics +metrics_name: cp-jobqueue +metrics_host: localhost +metrics_port: 8252 +# service +name: cp-jobqueue +port: 7200 +proxy: +site: datacenter1 +broker_list: localhost:9092 +jobrunner_uri: http://localhost/rpc/RunSingleJob.php +concurrency: 50 +redis_path: /var/run/redis.sock +redis_pass: +env: production +kafka_max_bytes: 4194304 -- To view, visit https://gerrit.wikimedia.org/r/376527 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I4b925e234d3720c0c90956d32e27ac611d813195 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/cp-jobqueue Gerrit-Branch: master Gerrit-Owner: Mobrovac <mobro...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits