Gehel has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/341566 )
Change subject: osm - waterline import script fix and adding logging
......................................................................
osm - waterline import script fix and adding logging
The waterline import script was failing as it tried to download updates to
a directory to which it has no write permissions. This was made harder to
identify as this script logs its output to /dev/null. Both issues are fixed.
Bug: T159631
Change-Id: Icf4172d7f72dcf6c5bcf6a60372d00aa239910dc
---
M modules/osm/manifests/import_waterlines.pp
M modules/osm/templates/import_waterlines.erb
A modules/osm/templates/waterlines.logrotate.erb
3 files changed, 43 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/66/341566/1
diff --git a/modules/osm/manifests/import_waterlines.pp
b/modules/osm/manifests/import_waterlines.pp
index b89555e..2fde648 100644
--- a/modules/osm/manifests/import_waterlines.pp
+++ b/modules/osm/manifests/import_waterlines.pp
@@ -9,6 +9,8 @@
$proxy = 'webproxy.eqiad.wmnet:8080'
) {
+ $log_dir = '/var/log/waterlines'
+
$proxy_opt = $proxy ? {
undef => '',
'' => '',
@@ -23,12 +25,24 @@
content => template('osm/import_waterlines.erb'),
}
+ file { $log_dir:
+ ensure => directory,
+ owner => 'postgres',
+ group => 'postgres',
+ mode => '0755',
+ }
+
+ logrotate::conf { 'eventlogging':
+ ensure => present,
+ content => template('osm/logrotate.erb'),
+ }
+
cron { 'import_waterlines':
ensure => present,
hour => 9,
minute => 13,
monthday => 1,
user => 'postgres',
- command => '/usr/local/bin/import_waterlines >/dev/null 2>&1',
+ command => "/usr/local/bin/import_waterlines > ${log_dir}/import.log
2>&1",
}
}
diff --git a/modules/osm/templates/import_waterlines.erb
b/modules/osm/templates/import_waterlines.erb
index 94149a8..86f65b1 100644
--- a/modules/osm/templates/import_waterlines.erb
+++ b/modules/osm/templates/import_waterlines.erb
@@ -2,6 +2,16 @@
set -e
+echo "`date` - Starting to import waterlines"
+
+temp_dir=$(mktemp -d)
+
+function cleanup {
+ echo "removing temporary directory ${temp_dir}"
+ if [[ ! -z ${temp_dir} && -d ${temp_dir} ]]; then rm -rf ${temp_dir}; fi
+}
+trap cleanup EXIT
+
if [ "${1}" == "create" ]; then
SHP2PGSQL_OPERATION="-c"
TRANSFORMATION=""
@@ -10,9 +20,11 @@
TRANSFORMATION='/^BEGIN;$/aTRUNCATE water_polygons;'
fi
-cd /srv/downloads
+cd ${temp_dir}
curl -O <%= @proxy_opt %>
http://data.openstreetmapdata.com/water-polygons-split-3857.zip
unzip water-polygons-split-3857.zip
rm water-polygons-split-3857.zip
shp2pgsql ${SHP2PGSQL_OPERATION} -s 3857:900913 -g way
water-polygons-split-3857/water_polygons.shp water_polygons | sed
"${TRANSFORMATION}" | psql <%= @database %>
rm -rf water-polygons-split-3857
+
+echo "`date` - waterlines import completed"
diff --git a/modules/osm/templates/waterlines.logrotate.erb
b/modules/osm/templates/waterlines.logrotate.erb
new file mode 100644
index 0000000..892ea73
--- /dev/null
+++ b/modules/osm/templates/waterlines.logrotate.erb
@@ -0,0 +1,15 @@
+#####################################################################
+### THIS FILE IS MANAGED BY PUPPET
+#####################################################################
+
+<%= @log_dir %>/import.log {
+ daily
+ notifempty
+ maxage 30
+ rotate 5
+ dateext
+ compress
+ delaycompress
+ missingok
+ nocreate
+}
--
To view, visit https://gerrit.wikimedia.org/r/341566
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Icf4172d7f72dcf6c5bcf6a60372d00aa239910dc
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Gehel <guillaume.leder...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
