BBlack has uploaded a new change for review.
https://gerrit.wikimedia.org/r/312511
Change subject: cache_upload: storage experiment is the new normal
......................................................................
cache_upload: storage experiment is the new normal
Bug: T145661
Change-Id: I9aef4cb931ca5091cb5f7aee697671f212eb158b
---
M modules/role/manifests/cache/upload.pp
M modules/varnish/templates/upload-backend.inc.vcl.erb
2 files changed, 17 insertions(+), 71 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/11/312511/1
diff --git a/modules/role/manifests/cache/upload.pp
b/modules/role/manifests/cache/upload.pp
index c704875..554d11e 100644
--- a/modules/role/manifests/cache/upload.pp
+++ b/modules/role/manifests/cache/upload.pp
@@ -72,10 +72,7 @@
# Because of this, pass_random does more harm than good in the
# upload-frontend case. All tiers of backend share the same policies.
- $upload_storage_experiment = hiera('upload_storage_experiment', false)
-
$be_vcl_config = merge($common_vcl_config, {
- 'upload_storage_experiment' => $upload_storage_experiment,
'pass_random' => true,
})
@@ -84,47 +81,22 @@
'pass_random' => false,
})
- if $upload_storage_experiment {
- $sda = $::role::cache::2layer::storage_parts[0]
- $sdb = $::role::cache::2layer::storage_parts[1]
- $ssm = $::role::cache::2layer::storage_size * 2 * 1024
- $bin0_size = floor($ssm * 0.03)
- $bin1_size = floor($ssm * 0.20)
- $bin2_size = floor($ssm * 0.43)
- $bin3_size = floor($ssm * 0.30)
- $bin4_size = floor($ssm * 0.04)
- $upload_storage_args = join([
- "-s bin0=file,/srv/${sda}/varnish.bin0,${bin0_size}M",
- "-s bin1=file,/srv/${sdb}/varnish.bin1,${bin1_size}M",
- "-s bin2=file,/srv/${sda}/varnish.bin2,${bin2_size}M",
- "-s bin3=file,/srv/${sdb}/varnish.bin3,${bin3_size}M",
- "-s bin4=file,/srv/${sda}/varnish.bin4,${bin4_size}M",
- ], ' ')
- }
- else {
- if ($::role::cache::2layer::varnish_version4) {
- # The persistent storage backend is deprecated and buggy in
Varnish 4.
- # Use "file" instead. See T142810, T142848 and
- # https://www.varnish-cache.org/docs/trunk/phk/persistent.html
- $storage_backend = 'file'
- $storage_mma_1 = ''
- $storage_mma_2 = ''
- }
- else {
- $storage_backend = 'persistent'
- $storage_mma_1 = ",${::role::cache::2layer::mma[0]}"
- $storage_mma_2 = ",${::role::cache::2layer::mma[1]}"
- }
-
- $storage_size_bigobj = floor($::role::cache::2layer::storage_size / 6)
- $storage_size_up = $::role::cache::2layer::storage_size -
$storage_size_bigobj
- $upload_storage_args = join([
- "-s
main1=${storage_backend},/srv/${::role::cache::2layer::storage_parts[0]}/varnish.main1,${storage_size_up}G${storage_mma_1}",
- "-s
main2=${storage_backend},/srv/${::role::cache::2layer::storage_parts[1]}/varnish.main2,${storage_size_up}G${storage_mma_2}",
- "-s
bigobj1=file,/srv/${::role::cache::2layer::storage_parts[0]}/varnish.bigobj1,${storage_size_bigobj}G",
- "-s
bigobj2=file,/srv/${::role::cache::2layer::storage_parts[1]}/varnish.bigobj2,${storage_size_bigobj}G",
- ], ' ')
- }
+ # See T145661 for storage binning rationale
+ $sda = $::role::cache::2layer::storage_parts[0]
+ $sdb = $::role::cache::2layer::storage_parts[1]
+ $ssm = $::role::cache::2layer::storage_size * 2 * 1024
+ $bin0_size = floor($ssm * 0.03)
+ $bin1_size = floor($ssm * 0.20)
+ $bin2_size = floor($ssm * 0.43)
+ $bin3_size = floor($ssm * 0.30)
+ $bin4_size = floor($ssm * 0.04)
+ $upload_storage_args = join([
+ "-s bin0=file,/srv/${sda}/varnish.bin0,${bin0_size}M",
+ "-s bin1=file,/srv/${sdb}/varnish.bin1,${bin1_size}M",
+ "-s bin2=file,/srv/${sda}/varnish.bin2,${bin2_size}M",
+ "-s bin3=file,/srv/${sdb}/varnish.bin3,${bin3_size}M",
+ "-s bin4=file,/srv/${sda}/varnish.bin4,${bin4_size}M",
+ ], ' ')
# default_ttl=7d
$common_runtime_params = ['default_ttl=604800']
diff --git a/modules/varnish/templates/upload-backend.inc.vcl.erb
b/modules/varnish/templates/upload-backend.inc.vcl.erb
index 27a2c11..c320022 100644
--- a/modules/varnish/templates/upload-backend.inc.vcl.erb
+++ b/modules/varnish/templates/upload-backend.inc.vcl.erb
@@ -1,9 +1,9 @@
// Varnish VCL include file for upload backends
include "upload-common.inc.vcl";
-<%- if @vcl_config.fetch('upload_storage_experiment') -%>
sub pick_stevedore {
// Select a storage size class/bin
+ // See T145661 for storage binning rationale
if (beresp.http.Content-Length !~ "^[0-9]+$") { // even possible on
upload?
set beresp.storage_hint = "bin1";
}
@@ -26,23 +26,6 @@
}
}
}
-<%- else -%>
-sub pick_main_stevedore {
- if (std.random(0, 2) < 1) {
- set beresp.<%= @storage_hint_storage %> = "main1";
- } else {
- set beresp.<%= @storage_hint_storage %> = "main2";
- }
-}
-
-sub pick_large_object_stevedore {
- if (std.random(0, 2) < 1) {
- set beresp.<%= @storage_hint_storage %> = "bigobj1";
- } else {
- set beresp.<%= @storage_hint_storage %> = "bigobj2";
- }
-}
-<%- end -%>
sub cluster_be_recv_pre_purge { }
@@ -96,16 +79,7 @@
}
<%- end -%>
- <%- if @vcl_config.fetch('upload_storage_experiment') -%>
call pick_stevedore;
- <%- else -%>
- // Select a random big object storage backend for objects >= 100 MB
- if (beresp.http.Content-Length ~ "^[0-9]{9}") {
- call pick_large_object_stevedore;
- } else {
- call pick_main_stevedore;
- }
- <%- end -%>
if (beresp.http.Content-Range) {
// Varnish itself doesn't ask for ranges, so this must have been
--
To view, visit https://gerrit.wikimedia.org/r/312511
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I9aef4cb931ca5091cb5f7aee697671f212eb158b
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BBlack <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
