Alexandros Kosiaris has submitted this change and it was merged.
Change subject: osm module
......................................................................
osm module
Module to handle installation of osm2pgsql, osmosis, and population and
syncing with from planet.osm
While at it enabled the hstore extension in postgres module
Also upload the already populated grants for osmosis and fix a couple of
bugs in the postgresql module while at it
Change-Id: Id3db056ace7a073831e90ebdaf47f6d9b2596ec5
---
A modules/osm/.rspec
A modules/osm/Rakefile
A modules/osm/manifests/packages.pp
A modules/osm/manifests/populatedb.pp
A modules/osm/manifests/usergrants.pp
A modules/osm/spec/classes/osm_packages_spec.rb
A modules/osm/spec/defines/osm_populatedb_spec.rb
A modules/osm/spec/defines/osm_usergrants_spec.rb
A modules/osm/spec/fixtures/manifests/site.pp
A modules/osm/spec/spec_helper.rb
M modules/postgresql/manifests/postgis.pp
M modules/postgresql/manifests/spatialdb.pp
M modules/postgresql/spec/defines/postgresql_spatial_db_spec.rb
13 files changed, 215 insertions(+), 7 deletions(-)
Approvals:
Alexandros Kosiaris: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/osm/.rspec b/modules/osm/.rspec
new file mode 100644
index 0000000..f449dae
--- /dev/null
+++ b/modules/osm/.rspec
@@ -0,0 +1,2 @@
+--format doc
+--color
diff --git a/modules/osm/Rakefile b/modules/osm/Rakefile
new file mode 100644
index 0000000..d9226a8
--- /dev/null
+++ b/modules/osm/Rakefile
@@ -0,0 +1,37 @@
+require 'rake'
+require 'fileutils'
+
+require 'rspec/core/rake_task'
+
+modulename = File.basename(File.expand_path(File.dirname(__FILE__)))
+
+symlinks = { 'spec/fixtures/modules/%s/files' % modulename =>
'../../../../files',
+ 'spec/fixtures/modules/%s/manifests' % modulename => '../../../../manifests',
+ 'spec/fixtures/modules/%s/templates' % modulename => '../../../../templates',
+}
+
+
+task :setup do
+ FileUtils.mkdir_p('spec/fixtures/modules/%s' % modulename)
+ symlinks.each do |x|
+ if !File.exist?(x[0])
+ FileUtils.ln_s(x[1], x[0])
+ end
+ end
+end
+
+task :teardown do
+ symlinks.each { |x| FileUtils.rm(x[0], :force => true) }
+ FileUtils.rmdir('spec/fixtures/modules/%s' % modulename)
+ FileUtils.rmdir('spec/fixtures/modules')
+end
+
+RSpec::Core::RakeTask.new(:realspec) do |t|
+ t.fail_on_error = false
+ t.pattern = 'spec/*/*_spec.rb'
+end
+
+task :spec => [ :setup, :realspec, :teardown]
+
+task :default => :spec do
+end
diff --git a/modules/osm/manifests/packages.pp
b/modules/osm/manifests/packages.pp
new file mode 100644
index 0000000..83a8527
--- /dev/null
+++ b/modules/osm/manifests/packages.pp
@@ -0,0 +1,10 @@
+#
+
+class osm::packages($ensure='present') {
+ package { [
+ 'osm2pgsql',
+ 'osmosis',
+ ]:
+ ensure => $ensure,
+ }
+}
diff --git a/modules/osm/manifests/populatedb.pp
b/modules/osm/manifests/populatedb.pp
new file mode 100644
index 0000000..66e63e6
--- /dev/null
+++ b/modules/osm/manifests/populatedb.pp
@@ -0,0 +1,39 @@
+#
+# Definition: osm::populatedb
+#
+# This definition provides a way to load planet_osm in a gis enabled db
+# You must have downloaded the pbf first and placed it in a configurable place
+#
+# Parameters:
+#
+# Actions:
+# load a planet.osm
+#
+# Requires:
+# Class['postgresql::postgis']
+# define['postgresql::spatialdb']
+#
+# Sample Usage:
+# osm::populatedb { 'mydb': input_pbf_file => '/myfile.pbf' }
+#
+define osm::populatedb(
+ $input_pbf_file,
+ ) {
+
+ # Check if our db tables exist
+ $tables_exist = "/usr/bin/psql --tuples-only -c \'SELECT
table_catalog,table_name FROM information_schema.tables;\' | /bin/grep \'^
${name}\' | /bin/grep \'planet_osm\'"
+
+ exec { "load_900913-${name}":
+ command => "/usr/bin/psql -d ${name} -f
/usr/share/osm2pgsql/osm2pgsql/900913.sql",
+ user => 'postgres',
+ unless => $tables_exist,
+ }
+
+ $load_planet_cmd = inline_template("<%- data=@memoryfree.split(' ');
multi={'MB' => 1, 'GB' => 1000}[data[1]]-%>/usr/bin/osm2pgsql -s -C <%=
data[0].to_i*multi %> -d <%= @name %> --number-processes <%= @processorcount %>
<%= @input_pbf_file %>")
+ exec { "load_planet_osm-${name}":
+ command => $load_planet_cmd,
+ user => 'postgres',
+ refreshonly => true,
+ subscribe => Exec["load_900913-${name}"],
+ }
+}
diff --git a/modules/osm/manifests/usergrants.pp
b/modules/osm/manifests/usergrants.pp
new file mode 100644
index 0000000..62429a6
--- /dev/null
+++ b/modules/osm/manifests/usergrants.pp
@@ -0,0 +1,40 @@
+#
+# Definition: osm::usergrants
+#
+# This definition provides a way to add the needed rights to spatial dbs
+#
+# Parameters:
+#
+# Actions:
+# Grant/revoke rights
+#
+# Requires:
+# Class['postgresql::postgis']
+# define['postgresql::spatialdb']
+#
+# Sample Usage:
+# osm::usergrants { 'mydb': postgresql_user => 'myser' }
+#
+define osm::usergrants(
+ $postgresql_user,
+ $ensure = 'present',
+ ) {
+
+ # Check if our db exists and store it
+ $db_exists = "/usr/bin/psql --tuples-only -c \'SELECT datname FROM
pg_catalog.pg_database;\' | /bin/grep \'^ ${name}\'"
+
+
+ if $ensure == 'present' {
+ exec { "grant_osm_rights-${name}":
+ command => "/usr/bin/psql -d ${name} -c \"GRANT SELECT ON
geometry_columns, spatial_ref_sys, planet_osm_line, planet_osm_nodes,
planet_osm_point, planet_osm_rels, planet_osm_roads, planet_osm_ways,
planet_osm_polygon TO ${postgresql_user};\"",
+ user => 'postgres',
+ onlyif => $db_exists,
+ }
+ } elsif $ensure == 'absent' {
+ exec { "revoke_osm_rights-${name}":
+ command => "/usr/bin/psql -d ${name} -c \"GRANT SELECT ON
geometry_columns, spatial_ref_sys, planet_osm_line, planet_osm_nodes,
planet_osm_point, planet_osm_rels, planet_osm_roads, planet_osm_ways,
planet_osm_polygon FROM ${postgresql_user};\"",
+ user => 'postgres',
+ onlyif => $db_exists,
+ }
+ }
+}
diff --git a/modules/osm/spec/classes/osm_packages_spec.rb
b/modules/osm/spec/classes/osm_packages_spec.rb
new file mode 100644
index 0000000..e5f7280
--- /dev/null
+++ b/modules/osm/spec/classes/osm_packages_spec.rb
@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe 'osm::packages', :type => :class do
+ let(:params) { {
+ :ensure => 'present',
+ }
+ }
+
+ context 'ensure present' do
+ it { should contain_package('osm2pgsql').with_ensure('present') }
+ it { should contain_package('osmosis').with_ensure('present') }
+ end
+end
+
+describe 'osm::packages', :type => :class do
+ let(:params) { {
+ :ensure => 'absent',
+ }
+ }
+
+ context 'ensure absent' do
+ it { should contain_package('osm2pgsql').with_ensure('absent') }
+ it { should contain_package('osmosis').with_ensure('absent') }
+ end
+end
diff --git a/modules/osm/spec/defines/osm_populatedb_spec.rb
b/modules/osm/spec/defines/osm_populatedb_spec.rb
new file mode 100644
index 0000000..b1873f3
--- /dev/null
+++ b/modules/osm/spec/defines/osm_populatedb_spec.rb
@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe 'osm::populatedb', :type => :define do
+ let(:title) { 'somedb' }
+ let(:params) { {
+ :input_pbf_file => '/nonexistent',
+ }
+ }
+ context 'with ensure present' do
+ it { should contain_exec('load_900913-somedb') }
+ it { should contain_exec('load_planet_osm-somedb') }
+ end
+end
diff --git a/modules/osm/spec/defines/osm_usergrants_spec.rb
b/modules/osm/spec/defines/osm_usergrants_spec.rb
new file mode 100644
index 0000000..3430159
--- /dev/null
+++ b/modules/osm/spec/defines/osm_usergrants_spec.rb
@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe 'osm::usergrants', :type => :define do
+ let(:title) { 'somedb' }
+ let(:params) { {
+ :postgresql_user => 'someuser',
+ :ensure => 'present',
+ }
+ }
+ context 'with ensure present' do
+ it { should contain_exec('grant_osm_rights-somedb') }
+ end
+end
+
+describe 'osm::usergrants', :type => :define do
+ let(:title) { 'somedb' }
+ let(:params) { {
+ :postgresql_user => 'someuser',
+ :ensure => 'absent',
+ }
+ }
+ context 'with ensure absent' do
+ it { should contain_exec('revoke_osm_rights-somedb') }
+ end
+end
diff --git a/modules/osm/spec/fixtures/manifests/site.pp
b/modules/osm/spec/fixtures/manifests/site.pp
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/modules/osm/spec/fixtures/manifests/site.pp
diff --git a/modules/osm/spec/spec_helper.rb b/modules/osm/spec/spec_helper.rb
new file mode 100644
index 0000000..d3923f8
--- /dev/null
+++ b/modules/osm/spec/spec_helper.rb
@@ -0,0 +1,8 @@
+require 'rspec-puppet'
+
+fixture_path = File.expand_path(File.join(__FILE__, '..', 'fixtures'))
+
+RSpec.configure do |c|
+ c.module_path = File.join(fixture_path, 'modules')
+ c.manifest_dir = File.join(fixture_path, 'manifests')
+end
diff --git a/modules/postgresql/manifests/postgis.pp
b/modules/postgresql/manifests/postgis.pp
index fc3dde7..1ca536f 100644
--- a/modules/postgresql/manifests/postgis.pp
+++ b/modules/postgresql/manifests/postgis.pp
@@ -17,7 +17,10 @@
$ensure='present'
) {
- package { "postgresql-${pgversion}-postgis":
+ package { [
+ "postgresql-${pgversion}-postgis",
+ "postgresql-contrib-${pgversion}",
+ ]:
ensure => $ensure,
}
}
diff --git a/modules/postgresql/manifests/spatialdb.pp
b/modules/postgresql/manifests/spatialdb.pp
index 482ba42..9000b04 100644
--- a/modules/postgresql/manifests/spatialdb.pp
+++ b/modules/postgresql/manifests/spatialdb.pp
@@ -47,28 +47,33 @@
command => "/usr/bin/psql -d ${name} -f
/usr/share/postgresql/${pg_version}/contrib/postgis-1.5/spatial_ref_sys.sql",
user => 'postgres',
refreshonly => true,
- require => Exec["create_postgis-${name}"],
+ subscribe => Exec["create_postgis-${name}"],
}
exec { "grant_ref_sys_cmd-${name}":
command => "/usr/bin/psql -d ${name} -c \"GRANT SELECT ON
spatial_ref_sys TO PUBLIC;\"",
user => 'postgres',
refreshonly => true,
- require => Exec["create_spatial_ref_sys-${name}"],
+ subscribe => Exec["create_spatial_ref_sys-${name}"],
}
# Create comments
exec { "create_comments-${name}":
- command => "/usr/bin/psql -d ${name} -f
/usr/share/postgresql/${pg_version}/contrib/comments.sql",
+ command => "/usr/bin/psql -d ${name} -f
/usr/share/postgresql/${pg_version}/contrib/postgis_comments.sql",
user => 'postgres',
refreshonly => true,
- require => Exec["create_spatial_ref_sys-${name}"],
+ subscribe => Exec["create_spatial_ref_sys-${name}"],
}
exec { "grant_comments_cmd-${name}":
command => "/usr/bin/psql -d ${name} -c \"GRANT ALL ON
geometry_columns TO PUBLIC;\"",
user => 'postgres',
refreshonly => true,
- require => Exec["create_comments-${name}"],
+ subscribe => Exec["create_comments-${name}"],
}
-
+ exec { "create_extension_hstore-${name}":
+ command => "/usr/bin/psql -d ${name} -c \"CREATE EXTENSION
hstore;\"",
+ user => 'postgres',
+ refreshonly => true,
+ subscribe => Exec["create_comments-${name}"],
+ }
Exec["create_db-${name}"] -> Exec["create_plpgsql_lang-${name}"]
Exec["create_plpgsql_lang-${name}"] -> Exec["create_postgis-${name}"]
} elsif $ensure == 'absent' {
diff --git a/modules/postgresql/spec/defines/postgresql_spatial_db_spec.rb
b/modules/postgresql/spec/defines/postgresql_spatial_db_spec.rb
index fa25d46..8e49e10 100644
--- a/modules/postgresql/spec/defines/postgresql_spatial_db_spec.rb
+++ b/modules/postgresql/spec/defines/postgresql_spatial_db_spec.rb
@@ -12,6 +12,7 @@
it { should contain_exec('create_postgis-somedb') }
it { should contain_exec('create_spatial_ref_sys-somedb') }
it { should contain_exec('create_comments-somedb') }
+ it { should contain_exec('create_extension_hstore-somedb') }
end
end
--
To view, visit https://gerrit.wikimedia.org/r/119408
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Id3db056ace7a073831e90ebdaf47f6d9b2596ec5
Gerrit-PatchSet: 4
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Alexandros Kosiaris <akosia...@wikimedia.org>
Gerrit-Reviewer: Alexandros Kosiaris <akosia...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
