ArielGlenn has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/328158 )
Change subject: make (most) snapshot shell scripts files instead of templates ...................................................................... make (most) snapshot shell scripts files instead of templates and add a little file of directory paths to be sourced by said scripts Change-Id: I926df4424d70ba458c1ec85fb3a62a9c33304f29 --- R modules/snapshot/files/cron/create-media-per-project-lists.sh R modules/snapshot/files/cron/dumpcentralauth.sh R modules/snapshot/files/cron/dumpcirrussearch.sh R modules/snapshot/files/cron/dumpcontentxlation.sh A modules/snapshot/files/cron/list-last-good-dumps.sh R modules/snapshot/files/cron/wikidatadumps-shared.sh M modules/snapshot/manifests/dumps/dirs.pp D modules/snapshot/templates/cron/list-last-good-dumps.sh.erb A modules/snapshot/templates/set_dump_dirs.sh.erb 9 files changed, 50 insertions(+), 26 deletions(-) Approvals: ArielGlenn: Looks good to me, approved jenkins-bot: Verified diff --git a/modules/snapshot/templates/cron/create-media-per-project-lists.sh.erb b/modules/snapshot/files/cron/create-media-per-project-lists.sh similarity index 78% rename from modules/snapshot/templates/cron/create-media-per-project-lists.sh.erb rename to modules/snapshot/files/cron/create-media-per-project-lists.sh index 0976cda..d464d29 100755 --- a/modules/snapshot/templates/cron/create-media-per-project-lists.sh.erb +++ b/modules/snapshot/files/cron/create-media-per-project-lists.sh @@ -2,13 +2,13 @@ ############################################################# # This file is maintained by puppet! -# modules/snapshot/templates/cron/create-media-per-project-lists.sh.erb +# modules/snapshot/cron/create-media-per-project-lists.sh ############################################################# +source /usr/local/etc/set_dump_dirs.sh + DATE=`/bin/date '+%Y%m%d'` -outputdir="<%= scope.lookupvar('snapshot::dumps::dirs::datadir') -%>/public/other/imageinfo/$DATE" -repodir="<%= scope.lookupvar('snapshot::dumps::dirs::repodir') -%>" -confsdir="<%= scope.lookupvar('snapshot::dumps::dirs::confsdir') %>" +outputdir="${datadir}/public/other/imageinfo/$DATE" configfile="${confsdir}/wikidump.conf.media" errors=0 diff --git a/modules/snapshot/templates/cron/dumpcentralauth.sh.erb b/modules/snapshot/files/cron/dumpcentralauth.sh similarity index 95% rename from modules/snapshot/templates/cron/dumpcentralauth.sh.erb rename to modules/snapshot/files/cron/dumpcentralauth.sh index 523d9f7..c68f633 100755 --- a/modules/snapshot/templates/cron/dumpcentralauth.sh.erb +++ b/modules/snapshot/files/cron/dumpcentralauth.sh @@ -1,8 +1,10 @@ #!/bin/bash ############################################################# # This file is maintained by puppet! -# modules/snapshot/templates/cron/dumpcentralauth.sh.erb +# modules/snapshot/cron/dumpcentralauth.sh ############################################################# + +source /usr/local/etc/set_dump_dirs.sh usage() { echo "Usage: $0 --site <datacenter> [--config <pathtofile>] [--dryrun]" @@ -14,7 +16,6 @@ exit 1 } -confsdir="<%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>" configfile="${confsdir}/wikidump.conf" dryrun="false" site="" diff --git a/modules/snapshot/templates/cron/dumpcirrussearch.sh.erb b/modules/snapshot/files/cron/dumpcirrussearch.sh similarity index 93% rename from modules/snapshot/templates/cron/dumpcirrussearch.sh.erb rename to modules/snapshot/files/cron/dumpcirrussearch.sh index 2e04961..29f30cc 100644 --- a/modules/snapshot/templates/cron/dumpcirrussearch.sh.erb +++ b/modules/snapshot/files/cron/dumpcirrussearch.sh @@ -1,20 +1,24 @@ #!/bin/bash ############################################################# # This file is maintained by puppet! -# modules/snapshot/templates/cron/dumpcirrussearch.sh.erb +# modules/snapshot/cron/dumpcirrussearch.sh ############################################################# # # Generate a json dump of cirrussearch indices for all enabled # wikis and remove old ones. + +source /usr/local/etc/set_dump_dirs.sh + usage() { echo "Usage: $0 [--config <pathtofile>] [--dryrun]" echo echo " --config path to configuration file for dump generation" - echo " (default value: <%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>/wikidump.conf" + echo " (default value: ${confsdir}/wikidump.conf" echo " --dryrun don't run dump, show what would have been done" exit 1 } -configFile="<%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>/wikidump.conf" + +configFile="${confsdir}/wikidump.conf" dryrun="false" while [ $# -gt 0 ]; do diff --git a/modules/snapshot/templates/cron/dumpcontentxlation.sh.erb b/modules/snapshot/files/cron/dumpcontentxlation.sh similarity index 88% rename from modules/snapshot/templates/cron/dumpcontentxlation.sh.erb rename to modules/snapshot/files/cron/dumpcontentxlation.sh index baa3a40..8b9f28f 100644 --- a/modules/snapshot/templates/cron/dumpcontentxlation.sh.erb +++ b/modules/snapshot/files/cron/dumpcontentxlation.sh @@ -1,8 +1,10 @@ #!/bin/bash ############################################################# # This file is maintained by puppet! -# modules/snapshot/templates/cron/dumpcontentxlation.sh.erb +# modules/snapshot/cron/dumpcontentxlation.sh ############################################################# + +source /usr/local/etc/set_dump_dirs.sh checkval() { setting=$1 @@ -49,10 +51,8 @@ # MAIN ##################### -confsdir="<%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>" -repodir="<%= scope.lookupvar('snapshot::dumps::dirs::repodir') -%>" configfile="${confsdir}/wikidump.conf" -otherdir="<%= scope.lookupvar('snapshot::dumps::dirs::datadir') -%>/public/other" +otherdir="${datadir}/public/other" dryrun="false" ##################### diff --git a/modules/snapshot/files/cron/list-last-good-dumps.sh b/modules/snapshot/files/cron/list-last-good-dumps.sh new file mode 100644 index 0000000..84d8ee0 --- /dev/null +++ b/modules/snapshot/files/cron/list-last-good-dumps.sh @@ -0,0 +1,13 @@ +#!/bin/bash +############################################################# +# This file is maintained by puppet! +# /modules/snapshot/cron/list-last-good-dumps.sh +############################################################# + +source /usr/local/etc/set_dump_dirs.sh + +# generate lists of most recent completed successful dumps for rsync (dirs, files) +python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5 --dirlisting 'rsync-dirlist-last-%s-good.txt' --configfile "${confsdir}/wikidump.conf.monitor" --rsynclists --relpath --outputdir "${datadir}/public/" +python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5 --filelisting 'rsync-filelist-last-%s-good.txt' --configfile "${confsdir}/wikidump.conf.monitor" --rsynclists --relpath --outputdir "${datadir}/public/" --toplevel +# these lists can be used for rsync excl/incl on our side, providing shares that "just work" for the mirrors +python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5 --rsynclisting 'rsync-inc-last-%s.txt' --configfile "${confsdir}/wikidump.conf.monitor" --relpath --outputdir "${datadir}/public/" --toplevel diff --git a/modules/snapshot/templates/cron/wikidatadumps-shared.sh.erb b/modules/snapshot/files/cron/wikidatadumps-shared.sh similarity index 92% rename from modules/snapshot/templates/cron/wikidatadumps-shared.sh.erb rename to modules/snapshot/files/cron/wikidatadumps-shared.sh index 4c42206..bcb65b7 100644 --- a/modules/snapshot/templates/cron/wikidatadumps-shared.sh.erb +++ b/modules/snapshot/files/cron/wikidatadumps-shared.sh @@ -1,14 +1,14 @@ #!/bin/bash ############################################################# # This file is maintained by puppet! -# modules/snapshot/templates/cron/wikidatadumps-shared.sh.erb +# modules/snapshot/cron/wikidatadumps-shared.sh ############################################################# # # Shared variable and function declarations for creating Wikidata dumps # # Marius Hoch < h...@online.de > -confdir="<%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>" +source /usr/local/etc/set_dump_dirs.sh configfile="${confdir}/wikidump.conf" today=`date +'%Y%m%d'` diff --git a/modules/snapshot/manifests/dumps/dirs.pp b/modules/snapshot/manifests/dumps/dirs.pp index ceb53a8..e1c3f3e 100644 --- a/modules/snapshot/manifests/dumps/dirs.pp +++ b/modules/snapshot/manifests/dumps/dirs.pp @@ -57,4 +57,14 @@ } $repodir = '/srv/deployment/dumps/dumps/xmldumps-backup' + + file { '/usr/local/etc/set_dump_dirs.sh': + ensure => 'present', + path => '/usr/local/etc/set_dump_dirs.sh', + mode => '0755', + owner => 'root', + group => 'root', + content => template('snapshot/set_dump_dirs.sh.erb'), + } + } diff --git a/modules/snapshot/templates/cron/list-last-good-dumps.sh.erb b/modules/snapshot/templates/cron/list-last-good-dumps.sh.erb deleted file mode 100644 index 05aa945..0000000 --- a/modules/snapshot/templates/cron/list-last-good-dumps.sh.erb +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -############################################################# -# This file is maintained by puppet! -# /modules/snapshot/templates/cron/list-last-good-dumps.sh.erb -############################################################# - -# generate lists of most recent completed successful dumps for rsync (dirs, files) -python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5 --dirlisting 'rsync-dirlist-last-%s-good.txt' --configfile <%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>/wikidump.conf.monitor --rsynclists --relpath --outputdir <%= scope.lookupvar('snapshot::dumps::dirs::datadir') -%>/public/ -python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5 --filelisting 'rsync-filelist-last-%s-good.txt' --configfile <%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>/wikidump.conf.monitor --rsynclists --relpath --outputdir <%= scope.lookupvar('snapshot::dumps::dirs::datadir') -%>/public/ --toplevel -# these lists can be used for rsync excl/incl on our side, providing shares that "just work" for the mirrors -python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5 --rsynclisting 'rsync-inc-last-%s.txt' --configfile <%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>/wikidump.conf.monitor --relpath --outputdir <%= scope.lookupvar('snapshot::dumps::dirs::datadir') -%>/public/ --toplevel diff --git a/modules/snapshot/templates/set_dump_dirs.sh.erb b/modules/snapshot/templates/set_dump_dirs.sh.erb new file mode 100644 index 0000000..26cecee --- /dev/null +++ b/modules/snapshot/templates/set_dump_dirs.sh.erb @@ -0,0 +1,7 @@ +#!/bin/bash + +# set some directory paths many cron jobs need +confsdir="<%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>" +repodir="<%= scope.lookupvar('snapshot::dumps::dirs::repodir') -%>" +datadir="<%= scope.lookupvar('snapshot::dumps::dirs::datadir') -%>" +dumpsdir="<%= scope.lookupvar('snapshot::dumps::dirs::dumpsdir') -%>" -- To view, visit https://gerrit.wikimedia.org/r/328158 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I926df4424d70ba458c1ec85fb3a62a9c33304f29 Gerrit-PatchSet: 3 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: ArielGlenn <ar...@wikimedia.org> Gerrit-Reviewer: ArielGlenn <ar...@wikimedia.org> Gerrit-Reviewer: Volans <rcocci...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits