http://www.mediawiki.org/wiki/Special:Code/MediaWiki/97360
Revision: 97360
Author: ezachte
Date: 2011-09-17 02:29:06 +0000 (Sat, 17 Sep 2011)
Log Message:
-----------
extra validation checks + introduction rsync
Modified Paths:
--------------
trunk/wikistats/dumps/publish.sh
Modified: trunk/wikistats/dumps/publish.sh
===================================================================
--- trunk/wikistats/dumps/publish.sh 2011-09-17 02:27:46 UTC (rev 97359)
+++ trunk/wikistats/dumps/publish.sh 2011-09-17 02:29:06 UTC (rev 97360)
@@ -1,13 +1,34 @@
#!/bin/bash
+clear
+now=`date +%s`
+
+function echo2 {
+ echo $1
+ echo $1 >> log_publish_sh.txt
+}
+
+clear
+echo2 "----------------"
+echo2 "Start publish.sh $1 $2"
+date >> log_publish_sh.txt
+
+# Validate project code
+if [ "$1" == "" ]
+then
+ echo2 "Project code missing! Specify as 1st argument one of
wb,wk,wn,wp,wq,ws,wv,wx"
+ exit
+fi
+
+# Abort when 3rd argument specifies a threshold in days, which is not met
+# This prevents costly processing step when new month has just started and
most ocunting still needs to be done
abort_before=$3
day_of_month=$(date +"%d")
if [ $day_of_month -lt ${abort_before:=0} ]
then
- echo publish.sh: day of month $day_of_month lt $abort_before - exit
+ echo2 "publish.sh: day of month $day_of_month lt $abort_before - exit"
exit
fi
-clear
case "$1" in
wb) project='Wikibooks' ; dir='wikibooks' ;;
wk) project='Wiktionaries' ; dir='wiktionary' ;;
@@ -19,51 +40,63 @@
wx) project='Wikispecial' ; dir='wikispecial' ;;
*) project='unknown' ; dir='...' ;;
esac
-echo "Publish $project" >> WikiCountsLogConcise.txt
-echo "Publish $project"
+echo2 "Publish $project"
#case "$2" in
# en) dir="$dir/EN" ;
#esac
-echo "Copy csv files"
-csv=/a/wikistats/csv_$1
-htdocs=/mnt/htdocs/$dir/csv
-echo csv $csv
-echo htdocs $htdocs
+htdocs="/mnt/htdocs/$dir/csv"
+csv="/a/wikistats/csv_$1"
+archive="/mnt/dumps/xmldatadumps/public/other/pagecounts-ez/wikistats" # odd
name, temp location
-chmod 774 $csv/*
-#cp $csv/csv_$1.zip $csv/csv.zip
-#rm $csv/csv.zip # temp, obsolete file
-#rm -f $htdocs/*
-cp $csv/* $htdocs -rvf
+rm -f $csv/csv.zip
+chmod 774 $csv/*.csv
-cp /a/wikistats/zip_all/csv_$1.zip /mnt/htdocs
+echo2 ""
+echo2 "Copy new/updated csv files from $csv -> $htdocs"
-publish=\#publish.txt
+echo2 ""
+rsync -av $csv/*.csv $htdocs
+
+echo2 ""
+echo2 "Copy csv and html zip files -> public archive $archive"
+echo2 ""
+rsync -av /a/wikistats/zip_all/csv*.zip $archive >> log_publish_sh.txt
+rsync -av /a/wikistats/zip_all/out*.zip $archive >> log_publish_sh.txt
-for lang in EN AST BG BR CA CS DA DE EO ES FR HE HU ID IT JA NL NN PL PT RO RU
SK SL SR SV WA ZH
-#for lang in EN
+publish="#publish.txt"
+
+for lang in AST BG BR CA CS DA DE EO ES FR HE HU ID IT JA NL NN PL PT RO RU SK
SL SR SV WA ZH
+#for lang in EN
do
- out=/a/out/out_$1/$lang
- htdocs=/mnt/htdocs/$dir/$lang
- echo out $out
- echo htdocs $htdocs
- echo dir $dir
- echo lang $lang
+ out=/a/out/out_$1/$lang/
+ htdocs=/mnt/htdocs/$dir/$lang/
+
+ echo2 ""
+ echo2 "Language $lang"
+ echo2 ""
+ echo2 "Copy new and updated files from $out -> $htdocs"
+ rsync -a $out/* $htdocs/ >> log_publish_sh.txt
+ ls -l $htdocs
+
# if test -e $out/WikiReportsPublish.txt
# then mv $out/WikiReportsPublish.txt "$out/$publish" ; fi ;
-
#if test -e $out/$publish
# then
- echo "New reports generated:\n"
- cat $out/$publish
- echo "\n\nCopy $lang\n\n"
- cp $out/* $htdocs -rvf
- rm $out/$publish
+# echo2 "New reports generated"
+# cat $out/$publish
+# echo2 "Copy $lang"
+# cp $out/* $htdocs -rvf
+# rm $out/$publish
# else echo "No new files, do not publish"
# fi
done
+echo2 ""
+echo2 "Ready"
+date >> log_publish_sh.txt
+echo2 "-----"
+
_______________________________________________
MediaWiki-CVS mailing list
MediaWiki-CVS@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-cvs
