jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/379745 )
Change subject: Refactor bin scripts to use default variables
......................................................................
Refactor bin scripts to use default variables
All variables are now defined in defaults.sh,
which is sourced in each Shell script.
Using variables may allow us to make the scripts
work in different environments (Docker vs Toolforge)
Change-Id: I8427992b79fc4f4de6a7741438885466ceb2d433
---
M bin/categorize_images.sh
M bin/create_all_monuments_tables.sh
A bin/defaults.sh
M bin/update_monuments.sh
M bin/update_monuments_min.sh
5 files changed, 41 insertions(+), 32 deletions(-)
Approvals:
Lokal Profil: Looks good to me, approved
jenkins-bot: Verified
diff --git a/bin/categorize_images.sh b/bin/categorize_images.sh
index d314bde..86f917e 100755
--- a/bin/categorize_images.sh
+++ b/bin/categorize_images.sh
@@ -2,15 +2,11 @@
#
# Script to categorize images
-echo_time() {
- echo "$(date +%F_%T) $*"
-}
-
-PYWIKIBOT_BIN=/data/project/heritage/pywikibot/pwb.py
-ERFGOED_PATH=/data/project/heritage/erfgoedbot
+CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+. $CURRENT_DIR/defaults.sh
# Make sure we are in our homedir
-cd /data/project/heritage/ || exit
+cd $HOME_DIR || exit
# Categorize some images
echo_time "Categorize images..."
diff --git a/bin/create_all_monuments_tables.sh
b/bin/create_all_monuments_tables.sh
index 6585900..eb331f6 100755
--- a/bin/create_all_monuments_tables.sh
+++ b/bin/create_all_monuments_tables.sh
@@ -1,25 +1,26 @@
#!/bin/bash
# Script to create all the tables from scratch
-ERFGOED_PATH=/data/project/heritage/erfgoedbot
-DATABASE=s51138__heritage_p
+CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+. $CURRENT_DIR/defaults.sh
# The python script expects to be in it's own directory:
cd $ERFGOED_PATH || exit
+
# First create the sql statements
python $ERFGOED_PATH/monument_tables.py
python $ERFGOED_PATH/fill_table_monuments_all.py
for i in $ERFGOED_PATH/sql/create_table_monuments*; do
echo "$i"
- mysql -h tools-db $DATABASE < "$i"
+ $MYSQL_BIN -h $DB_SERVER $DATABASE < "$i"
done
for i in $ERFGOED_PATH/sql/create_table_wlpa_*; do
echo "$i"
- mysql -h tools-db $DATABASE < "$i"
+ $MYSQL_BIN -h $DB_SERVER $DATABASE < "$i"
done
# Admin tree for browsing ISO codes
-mysql -h tools-db $DATABASE < $ERFGOED_PATH/sql/create_table_admin_tree.sql
+$MYSQL_BIN -h $DB_SERVER $DATABASE <
$ERFGOED_PATH/sql/create_table_admin_tree.sql
# Something with commonscat
-mysql -h tools-db $DATABASE < $ERFGOED_PATH/sql/create_table_commonscat.sql
+$MYSQL_BIN -h $DB_SERVER $DATABASE <
$ERFGOED_PATH/sql/create_table_commonscat.sql
# Tracking the images table
-mysql -h tools-db $DATABASE < $ERFGOED_PATH/sql/create_table_image.sql
+$MYSQL_BIN -h $DB_SERVER $DATABASE < $ERFGOED_PATH/sql/create_table_image.sql
diff --git a/bin/defaults.sh b/bin/defaults.sh
new file mode 100644
index 0000000..f6e676d
--- /dev/null
+++ b/bin/defaults.sh
@@ -0,0 +1,17 @@
+echo_time() {
+ echo "$(date +%F_%T) $*"
+}
+
+# Paths
+: ${HOME_DIR:=/data/project/heritage}
+: ${SOURCE_PATH:=$HOME_DIR/heritage}
+: ${ERFGOED_PATH:=$SOURCE_PATH/erfgoedbot}
+
+# Database config
+: ${DATABASE:=s51138__heritage_p}
+: ${DB_SERVER:=tools-db}
+
+# Executables
+: ${PYWIKIBOT_BIN:=$HOME_DIR/pywikibot/pwb.py}
+: ${PHP_BIN:=php}
+: ${MYSQL_BIN:=mysql}
diff --git a/bin/update_monuments.sh b/bin/update_monuments.sh
index 64ec729..7b4dfcb 100755
--- a/bin/update_monuments.sh
+++ b/bin/update_monuments.sh
@@ -2,18 +2,13 @@
#
# Script to make a full update of the monuments databse and run a bunch of
tasks
-echo_time() {
- echo "$(date +%F_%T) $*"
-}
-
-PYWIKIBOT_BIN=/data/project/heritage/pywikibot/pwb.py
-ERFGOED_PATH=/data/project/heritage/erfgoedbot
-DATABASE=s51138__heritage_p
+CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+. $CURRENT_DIR/defaults.sh
echo_time "Starting full monument update."
# Make sure we are in our homedir
-cd /data/project/heritage/ || exit
+cd $HOME_DIR || exit
# Load any config changes into the source tables
echo_time "Load changes to monuments_config..."
@@ -23,7 +18,7 @@
# Recreate the source tables
echo_time "Recreating the source tables..."
for i in $ERFGOED_PATH/sql/create_table_monuments*; do
- mysql -h tools-db $DATABASE < "$i"
+ $MYSQL_BIN -h $DB_SERVER $DATABASE < "$i"
done
# Update all of the source tables
@@ -32,7 +27,7 @@
# Update the all monuments table
echo_time "Update monuments_all table..."
-mysql -h tools-db $DATABASE < $ERFGOED_PATH/sql/fill_table_monuments_all.sql
+$MYSQL_BIN -h $DB_SERVER $DATABASE <
$ERFGOED_PATH/sql/fill_table_monuments_all.sql
## Update the image table. Is now another job
# echo_time "Update image table..."
@@ -40,7 +35,7 @@
# Update admin structure tree
echo_time "Update admin structure tree..."
-php ./erfgoedbot/populate_adm_tree.php
+$PHP_BIN $ERFGOED_PATH/populate_adm_tree.php
# Make statistics
echo_time "Make statistics..."
@@ -48,7 +43,7 @@
# Make more detailed statistics
echo_time "Make more detailed statistics..."
-php ./public_html/maintenance/_buildStats.php
+$PHP_BIN ./public_html/maintenance/_buildStats.php
# Update the list of unused monuments
echo_time "Update unused images list..."
@@ -66,7 +61,7 @@
# Dump the database
echo_time "Dump database..."
-mysqldump --host=tools-db --single-transaction $DATABASE > monuments_db-new.sql
+mysqldump --host=$DB_SERVER --single-transaction $DATABASE >
monuments_db-new.sql
nice gzip monuments_db-new.sql
# Atomically replace the provided file
@@ -76,6 +71,6 @@
# Refill prox_search table. Which will be used by layar server.
echo_time "Refill prox_search table..."
-php ./prox_search/fill_table_prox_search.php
+$PHP_BIN ./prox_search/fill_table_prox_search.php
echo_time "Done with the update!"
diff --git a/bin/update_monuments_min.sh b/bin/update_monuments_min.sh
index 4ed01bc..655bb28 100755
--- a/bin/update_monuments_min.sh
+++ b/bin/update_monuments_min.sh
@@ -1,12 +1,12 @@
#!/bin/bash
#
# Minimal script to update the monuments database
-PYWIKIBOT_BIN=/data/project/heritage/pywikibot/pwb.py
-ERFGOED_PATH=/data/project/heritage/erfgoedbot
-DATABASE=s51138__heritage_p
+
+CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+. $CURRENT_DIR/defaults.sh
# First have the erfgoed bot update everything in it's config
$PYWIKIBOT_BIN $ERFGOED_PATH/update_database.py -fullupdate
# Update the all monuments table
-mysql -h tools-db $DATABASE < $ERFGOED_PATH/sql/fill_table_monuments_all.sql
+$MYSQL_BIN -h $DB_SERVER $DATABASE <
$ERFGOED_PATH/sql/fill_table_monuments_all.sql
--
To view, visit https://gerrit.wikimedia.org/r/379745
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I8427992b79fc4f4de6a7741438885466ceb2d433
Gerrit-PatchSet: 2
Gerrit-Project: labs/tools/heritage
Gerrit-Branch: master
Gerrit-Owner: Jean-Frédéric <[email protected]>
Gerrit-Reviewer: Lokal Profil <[email protected]>
Gerrit-Reviewer: Multichill <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
