[MediaWiki-commits] [Gerrit] add script for rsyncs of public data between dataset hosts - change (operations/puppet)
ArielGlenn has submitted this change and it was merged.
Change subject: add script for rsyncs of public data between dataset hosts
..
add script for rsyncs of public data between dataset hosts
Change-Id: I827bd6807c0b51d1d896ae337a4b9bf2f3a2fbc7
---
A files/misc/scripts/rsync-dumps.sh
1 file changed, 72 insertions(+), 0 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/files/misc/scripts/rsync-dumps.sh
b/files/misc/scripts/rsync-dumps.sh
new file mode 100755
index 000..ae20c8f
--- /dev/null
+++ b/files/misc/scripts/rsync-dumps.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# NOTE: this file is maintained in puppet
+
+# rsync XML dumps and other public datasets between the various dataset
servers,
+# allowing for the possibility that dumps may be produced on any or all of them
+# for different sets of wikis
+
+# this script takes one possible argument: "dryrun" in which case it prints
what it would
+# do instead of actually doing the rsyncs
+
+if [ "$1" == "dryrun" ]; then
+DRYRUN=1
+else
+DRYRUN=0
+fi
+
+RSYNCBW="4"
+
+
+HOST=`/bin/hostname`
+
+RSYNCCMD='/usr/bin/rsync'
+RSYNCARGS=( '-q' "--bwlimit=$RSYNCBW" '-a' '--delete' )
+EXCLUDES=( '--exclude=wikidump_*' '--exclude=md5temp.*' )
+RSYNCSRC='/data/xmldatadumps/public/'
+
+MAILCMD='/usr/bin/mail'
+MAILARGS=( '-E' '-s' "DUMPS RSYNC $HOST" 'ops-du...@wikimedia.org' )
+
+SERVERS=( 'dataset2' 'dataset1001' 'localhost.localdomain' )
+# remove self from the list
+for ((i=0; i<"${#SERVERS[*]}"; i++)); do
+if [ "${SERVERS[$i]}" == "$HOST" ]; then
+ unset "SERVERS[$i]"
+fi
+done
+
+do_rsyncs () {
+for s in "${SERVERS[@]}"; do
+ running=`pgrep -u root -f "$s::data/xmldatadumps/public/"`
+ if [ ! -z "$running" ]; then
+ exit 0
+ fi
+ RSYNCDEST="$s::data/xmldatadumps/public/"
+ if [ $DRYRUN -eq 0 ]; then
+ $RSYNCCMD "${RSYNCARGS[@]}" "${EXCLUDES[@]}" "${EXTRAARGS[@]}"
"${RSYNCSRC}" "${RSYNCDEST}" 2>&1 | $MAILCMD "${MAILARGS[@]}"
+ else
+ echo -n $RSYNCCMD "${RSYNCARGS[@]}" "${EXCLUDES[@]}"
"${EXTRAARGS[@]}" "${RSYNCSRC}" "${RSYNCDEST}"
+ echo '|' "$MAILCMD" "${MAILARGS[@]}"
+ fi
+done
+}
+
+case "$HOST" in
+'dataset1001' )
+ # all dumps are produced on this host except these
+ DIRS=( '/enwiki/' ) # must have leading/trailing slash
+ DIRARGS="${DIRS[@]/#/--include=}"
+ EXTRAARGS=( "${DIRARGS[@]}" "${DIRARGS[@]/%/**}" '--exclude=*' )
+ do_rsyncs
+ ;;
+'localhost.localdomain'|'dataset2' )
+ # directories for which this host produces dumps
+ DIRS=( '/enwiki/' ) # must have leading/trailing slash
+ EXTRAARGS=( "${DIRS[@]/#/--exclude=}" )
+ do_rsyncs
+ ;;
+* )
+echo "No rsync stanza available for $HOST"
+ ;;
+esac
--
To view, visit https://gerrit.wikimedia.org/r/71964
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I827bd6807c0b51d1d896ae337a4b9bf2f3a2fbc7
Gerrit-PatchSet: 2
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn
Gerrit-Reviewer: ArielGlenn
Gerrit-Reviewer: jenkins-bot
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] add script for rsyncs of public data between dataset hosts - change (operations/puppet)
ArielGlenn has uploaded a new change for review.
https://gerrit.wikimedia.org/r/71964
Change subject: add script for rsyncs of public data between dataset hosts
..
add script for rsyncs of public data between dataset hosts
Change-Id: I827bd6807c0b51d1d896ae337a4b9bf2f3a2fbc7
---
A files/misc/scripts/rsync-dumps.sh
1 file changed, 72 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/64/71964/1
diff --git a/files/misc/scripts/rsync-dumps.sh
b/files/misc/scripts/rsync-dumps.sh
new file mode 100755
index 000..ae20c8f
--- /dev/null
+++ b/files/misc/scripts/rsync-dumps.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# NOTE: this file is maintained in puppet
+
+# rsync XML dumps and other public datasets between the various dataset
servers,
+# allowing for the possibility that dumps may be produced on any or all of them
+# for different sets of wikis
+
+# this script takes one possible argument: "dryrun" in which case it prints
what it would
+# do instead of actually doing the rsyncs
+
+if [ "$1" == "dryrun" ]; then
+DRYRUN=1
+else
+DRYRUN=0
+fi
+
+RSYNCBW="4"
+
+
+HOST=`/bin/hostname`
+
+RSYNCCMD='/usr/bin/rsync'
+RSYNCARGS=( '-q' "--bwlimit=$RSYNCBW" '-a' '--delete' )
+EXCLUDES=( '--exclude=wikidump_*' '--exclude=md5temp.*' )
+RSYNCSRC='/data/xmldatadumps/public/'
+
+MAILCMD='/usr/bin/mail'
+MAILARGS=( '-E' '-s' "DUMPS RSYNC $HOST" 'ops-du...@wikimedia.org' )
+
+SERVERS=( 'dataset2' 'dataset1001' 'localhost.localdomain' )
+# remove self from the list
+for ((i=0; i<"${#SERVERS[*]}"; i++)); do
+if [ "${SERVERS[$i]}" == "$HOST" ]; then
+ unset "SERVERS[$i]"
+fi
+done
+
+do_rsyncs () {
+for s in "${SERVERS[@]}"; do
+ running=`pgrep -u root -f "$s::data/xmldatadumps/public/"`
+ if [ ! -z "$running" ]; then
+ exit 0
+ fi
+ RSYNCDEST="$s::data/xmldatadumps/public/"
+ if [ $DRYRUN -eq 0 ]; then
+ $RSYNCCMD "${RSYNCARGS[@]}" "${EXCLUDES[@]}" "${EXTRAARGS[@]}"
"${RSYNCSRC}" "${RSYNCDEST}" 2>&1 | $MAILCMD "${MAILARGS[@]}"
+ else
+ echo -n $RSYNCCMD "${RSYNCARGS[@]}" "${EXCLUDES[@]}"
"${EXTRAARGS[@]}" "${RSYNCSRC}" "${RSYNCDEST}"
+ echo '|' "$MAILCMD" "${MAILARGS[@]}"
+ fi
+done
+}
+
+case "$HOST" in
+'dataset1001' )
+ # all dumps are produced on this host except these
+ DIRS=( '/enwiki/' ) # must have leading/trailing slash
+ DIRARGS="${DIRS[@]/#/--include=}"
+ EXTRAARGS=( "${DIRARGS[@]}" "${DIRARGS[@]/%/**}" '--exclude=*' )
+ do_rsyncs
+ ;;
+'localhost.localdomain'|'dataset2' )
+ # directories for which this host produces dumps
+ DIRS=( '/enwiki/' ) # must have leading/trailing slash
+ EXTRAARGS=( "${DIRS[@]/#/--exclude=}" )
+ do_rsyncs
+ ;;
+* )
+echo "No rsync stanza available for $HOST"
+ ;;
+esac
--
To view, visit https://gerrit.wikimedia.org/r/71964
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I827bd6807c0b51d1d896ae337a4b9bf2f3a2fbc7
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
