Gehel has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/341566 )
Change subject: osm - waterline import script fix and adding logging ...................................................................... osm - waterline import script fix and adding logging The waterline import script was failing as it tried to download updates to a directory to which it has no write permissions. This was made harder to identify as this script logs its output to /dev/null. Both issues are fixed. Bug: T159631 Change-Id: Icf4172d7f72dcf6c5bcf6a60372d00aa239910dc --- M modules/osm/manifests/import_waterlines.pp M modules/osm/templates/import_waterlines.erb A modules/osm/templates/waterlines.logrotate.erb 3 files changed, 43 insertions(+), 2 deletions(-) Approvals: Pnorman: Looks good to me, but someone else must approve MaxSem: Looks good to me, but someone else must approve jenkins-bot: Verified Gehel: Looks good to me, approved diff --git a/modules/osm/manifests/import_waterlines.pp b/modules/osm/manifests/import_waterlines.pp index b89555e..1a9f793 100644 --- a/modules/osm/manifests/import_waterlines.pp +++ b/modules/osm/manifests/import_waterlines.pp @@ -9,6 +9,8 @@ $proxy = 'webproxy.eqiad.wmnet:8080' ) { + $log_dir = '/var/log/waterlines' + $proxy_opt = $proxy ? { undef => '', '' => '', @@ -23,12 +25,24 @@ content => template('osm/import_waterlines.erb'), } + file { $log_dir: + ensure => directory, + owner => 'postgres', + group => 'postgres', + mode => '0755', + } + + logrotate::conf { 'eventlogging': + ensure => present, + content => template('osm/logrotate.erb'), + } + cron { 'import_waterlines': ensure => present, hour => 9, minute => 13, monthday => 1, user => 'postgres', - command => '/usr/local/bin/import_waterlines >/dev/null 2>&1', + command => "/usr/local/bin/import_waterlines > ${log_dir}/import.log 2>&1", } } diff --git a/modules/osm/templates/import_waterlines.erb b/modules/osm/templates/import_waterlines.erb index 94149a8..86f65b1 100644 --- a/modules/osm/templates/import_waterlines.erb +++ b/modules/osm/templates/import_waterlines.erb @@ -2,6 +2,16 @@ set -e +echo "`date` - Starting to import waterlines" + +temp_dir=$(mktemp -d) + +function cleanup { + echo "removing temporary directory ${temp_dir}" + if [[ ! -z ${temp_dir} && -d ${temp_dir} ]]; then rm -rf ${temp_dir}; fi +} +trap cleanup EXIT + if [ "${1}" == "create" ]; then SHP2PGSQL_OPERATION="-c" TRANSFORMATION="" @@ -10,9 +20,11 @@ TRANSFORMATION='/^BEGIN;$/aTRUNCATE water_polygons;' fi -cd /srv/downloads +cd ${temp_dir} curl -O <%= @proxy_opt %> http://data.openstreetmapdata.com/water-polygons-split-3857.zip unzip water-polygons-split-3857.zip rm water-polygons-split-3857.zip shp2pgsql ${SHP2PGSQL_OPERATION} -s 3857:900913 -g way water-polygons-split-3857/water_polygons.shp water_polygons | sed "${TRANSFORMATION}" | psql <%= @database %> rm -rf water-polygons-split-3857 + +echo "`date` - waterlines import completed" diff --git a/modules/osm/templates/waterlines.logrotate.erb b/modules/osm/templates/waterlines.logrotate.erb new file mode 100644 index 0000000..892ea73 --- /dev/null +++ b/modules/osm/templates/waterlines.logrotate.erb @@ -0,0 +1,15 @@ +##################################################################### +### THIS FILE IS MANAGED BY PUPPET +##################################################################### + +<%= @log_dir %>/import.log { + daily + notifempty + maxage 30 + rotate 5 + dateext + compress + delaycompress + missingok + nocreate +} -- To view, visit https://gerrit.wikimedia.org/r/341566 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Icf4172d7f72dcf6c5bcf6a60372d00aa239910dc Gerrit-PatchSet: 3 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Gehel <guillaume.leder...@wikimedia.org> Gerrit-Reviewer: Alexandros Kosiaris <akosia...@wikimedia.org> Gerrit-Reviewer: Gehel <guillaume.leder...@wikimedia.org> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: MaxSem <maxsem.w...@gmail.com> Gerrit-Reviewer: Pnorman <penor...@mac.com> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits