This is an automated email from the git hooks/post-receive script. sebastic pushed a commit to branch master in repository imposm.
commit 03271ca7a8799c0c9e21408cd32d97ba926496ba Author: David Paleino <da...@debian.org> Date: Mon Dec 10 19:23:45 2012 +0100 Imported Upstream version 2.5.0 --- CHANGES | 17 +- MANIFEST.in | 1 + PKG-INFO | 23 +- README => README.rst | 2 - imposm.egg-info/PKG-INFO | 23 +- imposm.egg-info/SOURCES.txt | 14 +- imposm/app.py | 102 +++++-- imposm/cache/tc_tmp.py | 577 ----------------------------------- imposm/db/postgis.py | 257 +++++++++++----- imposm/dbimporter.py | 175 ++++++++--- imposm/geom.py | 272 ++++++++++++++++- imposm/mapping.py | 134 ++++---- imposm/psqldb.py | 66 ++-- imposm/reader.py | 32 +- imposm/test/test_dbimporter.py | 100 ++++++ imposm/test/test_geom.py | 33 -- imposm/test/test_imported.py | 27 +- imposm/test/test_limit_to.py | 85 ++++++ imposm/{util.py => util/__init__.py} | 57 ++-- imposm/util/geom.py | 144 +++++++++ imposm/util/lib.py | 107 +++++++ imposm/util/ogr.py | 130 ++++++++ imposm/version.py | 2 +- imposm/writer.py | 25 +- setup.py | 2 +- 25 files changed, 1457 insertions(+), 950 deletions(-) diff --git a/CHANGES b/CHANGES index 67844a6..377e331 100644 --- a/CHANGES +++ b/CHANGES @@ -1,7 +1,20 @@ Changelog --------- -2.4.0 2012-xx-xx +2.5.0 2012-12-06 +~~~~~~~~~~~~~~~~ + +- PostGIS 2 support +- new --limit-to option to limit imports to polygons +- added --quiet option that only logs progress once per minute +- add StringIndex and Index mappings for PostgreSQL +- always drop tables _and_ views with same name before creating new + table/view. allows to change mappings from views to tables and + vice versa. +- internal refactoring to make support for non SQL databases easier + + +2.4.0 2012-03-30 ~~~~~~~~~~~~~~~~ - new Class and Type field types @@ -39,7 +52,7 @@ Changelog - update SRS in GeneralizedTables and UnionTables - new waterareas_gen0|1 in default style - new area field in landusages table -- new meter_to_mapunit function to use same mapping +- new meter_to_mapunit function to use same mapping for EPSG:4326 and projected SRS 2.2.0 2011-06-01 diff --git a/MANIFEST.in b/MANIFEST.in index 0411940..2f5ff17 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include setup.py +include README.rst include LICENSE include CHANGES include imposm/900913.sql diff --git a/PKG-INFO b/PKG-INFO index ddfdd06..5ce96c2 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,14 +1,12 @@ Metadata-Version: 1.0 Name: imposm -Version: 2.4.0 +Version: 2.5.0 Summary: OpenStreetMap importer for PostGIS. Home-page: http://imposm.org/ Author: Oliver Tonnhofer Author-email: o...@omniscale.de License: Apache Software License 2.0 -Description: .. # -*- restructuredtext -*- - - Imposm is an importer for OpenStreetMap data. It reads XML and PBF files and +Description: Imposm is an importer for OpenStreetMap data. It reads XML and PBF files and can import the data into PostgreSQL/PostGIS databases. It is designed to create databases that are optimized for rendering/WMS @@ -23,7 +21,20 @@ Description: .. # -*- restructuredtext -*- Changelog --------- - 2.4.0 2012-xx-xx + 2.5.0 2012-12-06 + ~~~~~~~~~~~~~~~~ + + - PostGIS 2 support + - new --limit-to option to limit imports to polygons + - added --quiet option that only logs progress once per minute + - add StringIndex and Index mappings for PostgreSQL + - always drop tables _and_ views with same name before creating new + table/view. allows to change mappings from views to tables and + vice versa. + - internal refactoring to make support for non SQL databases easier + + + 2.4.0 2012-03-30 ~~~~~~~~~~~~~~~~ - new Class and Type field types @@ -61,7 +72,7 @@ Description: .. # -*- restructuredtext -*- - update SRS in GeneralizedTables and UnionTables - new waterareas_gen0|1 in default style - new area field in landusages table - - new meter_to_mapunit function to use same mapping + - new meter_to_mapunit function to use same mapping for EPSG:4326 and projected SRS 2.2.0 2011-06-01 diff --git a/README b/README.rst similarity index 93% rename from README rename to README.rst index 21f5e77..7ef4fe3 100644 --- a/README +++ b/README.rst @@ -1,5 +1,3 @@ -.. # -*- restructuredtext -*- - Imposm is an importer for OpenStreetMap data. It reads XML and PBF files and can import the data into PostgreSQL/PostGIS databases. diff --git a/imposm.egg-info/PKG-INFO b/imposm.egg-info/PKG-INFO index ddfdd06..5ce96c2 100644 --- a/imposm.egg-info/PKG-INFO +++ b/imposm.egg-info/PKG-INFO @@ -1,14 +1,12 @@ Metadata-Version: 1.0 Name: imposm -Version: 2.4.0 +Version: 2.5.0 Summary: OpenStreetMap importer for PostGIS. Home-page: http://imposm.org/ Author: Oliver Tonnhofer Author-email: o...@omniscale.de License: Apache Software License 2.0 -Description: .. # -*- restructuredtext -*- - - Imposm is an importer for OpenStreetMap data. It reads XML and PBF files and +Description: Imposm is an importer for OpenStreetMap data. It reads XML and PBF files and can import the data into PostgreSQL/PostGIS databases. It is designed to create databases that are optimized for rendering/WMS @@ -23,7 +21,20 @@ Description: .. # -*- restructuredtext -*- Changelog --------- - 2.4.0 2012-xx-xx + 2.5.0 2012-12-06 + ~~~~~~~~~~~~~~~~ + + - PostGIS 2 support + - new --limit-to option to limit imports to polygons + - added --quiet option that only logs progress once per minute + - add StringIndex and Index mappings for PostgreSQL + - always drop tables _and_ views with same name before creating new + table/view. allows to change mappings from views to tables and + vice versa. + - internal refactoring to make support for non SQL databases easier + + + 2.4.0 2012-03-30 ~~~~~~~~~~~~~~~~ - new Class and Type field types @@ -61,7 +72,7 @@ Description: .. # -*- restructuredtext -*- - update SRS in GeneralizedTables and UnionTables - new waterareas_gen0|1 in default style - new area field in landusages table - - new meter_to_mapunit function to use same mapping + - new meter_to_mapunit function to use same mapping for EPSG:4326 and projected SRS 2.2.0 2011-06-01 diff --git a/imposm.egg-info/SOURCES.txt b/imposm.egg-info/SOURCES.txt index 241add1..c5a8d44 100644 --- a/imposm.egg-info/SOURCES.txt +++ b/imposm.egg-info/SOURCES.txt @@ -1,7 +1,7 @@ CHANGES LICENSE MANIFEST.in -README +README.rst internal.proto setup.cfg setup.py @@ -18,7 +18,6 @@ imposm/merge.py imposm/multipolygon.py imposm/psqldb.py imposm/reader.py -imposm/util.py imposm/version.py imposm/writer.py imposm.egg-info/PKG-INFO @@ -30,18 +29,21 @@ imposm.egg-info/requires.txt imposm.egg-info/top_level.txt imposm/cache/__init__.py imposm/cache/internal.cc -imposm/cache/kc.c imposm/cache/osm.py imposm/cache/tc.c imposm/cache/tc.pyx -imposm/cache/tc_tmp.py imposm/db/__init__.py imposm/db/config.py imposm/db/postgis.py imposm/test/__init__.py imposm/test/test_cache.py +imposm/test/test_dbimporter.py imposm/test/test_field_types.py -imposm/test/test_geom.py imposm/test/test_imported.py +imposm/test/test_limit_to.py imposm/test/test_multipolygon.py -imposm/test/test_tag_mapper.py \ No newline at end of file +imposm/test/test_tag_mapper.py +imposm/util/__init__.py +imposm/util/geom.py +imposm/util/lib.py +imposm/util/ogr.py \ No newline at end of file diff --git a/imposm/app.py b/imposm/app.py index f6124ba..ea3b250 100644 --- a/imposm/app.py +++ b/imposm/app.py @@ -1,11 +1,11 @@ # Copyright 2011 Omniscale (http://omniscale.com) -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,6 +44,7 @@ from imposm.db.config import DB from imposm.cache import OSMCache from imposm.reader import ImposmReader from imposm.mapping import TagMapper +from imposm.geom import load_geom try: n_cpu = multiprocessing.cpu_count() @@ -73,7 +74,9 @@ def main(argv=None): default=False, help='show this help message and exit') parser.add_option('--debug', action='store_true', default=False, help='show debug information') - + parser.add_option('--quiet', action='store_true', + default=False, help='only print progress every 60 seconds') + parser.add_option('-m', '--mapping-file', dest='mapping_file', metavar='<file>') parser.add_option('-h', '--host', dest='host', metavar='<host>') @@ -84,7 +87,7 @@ def main(argv=None): parser.add_option('--connection', dest='connection', help="connection string like postgis://user:pass@host:port/database," " this overwrites the -h/-p/-d/-U options") - + parser.add_option('-c', '--concurrency', dest='concurrency', metavar='N', type='int', default=n_cpu) @@ -94,8 +97,8 @@ def main(argv=None): action='store_true') parser.add_option('--cache-dir', dest='cache_dir', default='.', help="path where node/ways/relations should be cached [current working dir]") - - + + parser.add_option('--table-prefix', dest='table_prefix', default=None, metavar='osm_new_', help='prefix for imported tables') @@ -122,22 +125,35 @@ def main(argv=None): 'move backup tables to production tables') parser.add_option('--remove-backup-tables', dest='remove_backup_tables', default=False, action='store_true') - parser.add_option('-n', '--dry-run', dest='dry_run', default=False, action='store_true') + parser.add_option('--limit-to', dest='limit_to', metavar='file', + help='limit imported geometries to (multi)polygons in EPSG:4326') + (options, args) = parser.parse_args(argv) setup_logging(debug=options.debug) - if (argv and len(argv) == 0) or len(sys.argv) == 1: + if (argv and len(argv) == 0) or (not argv and len(sys.argv) == 1): + options.help = True + + if not any([options.read, options.write, options.optimize, options.deploy_tables, + options.recover_tables, options.remove_backup_tables]): options.help = True if options.help: parser.print_help() sys.exit(1) - + + if options.quiet: + logger = imposm.util.QuietProgressLog + logger_parser = imposm.util.QuietParserProgress + else: + logger = imposm.util.ProgressLog + logger_parser = imposm.util.ParserProgress + if options.proj: if ':' not in options.proj: print 'ERROR: --proj should be in EPSG:00000 format' @@ -145,31 +161,48 @@ def main(argv=None): # check proj if meter_to_mapunit needs to do anything if options.proj.lower() == 'epsg:4326': imposm.mapping.import_srs_is_geographic = True - + mapping_file = os.path.join(os.path.dirname(__file__), 'defaultmapping.py') if options.mapping_file: print 'loading %s as mapping' % options.mapping_file mapping_file = options.mapping_file + polygon = None + if options.limit_to: + logger.message('## reading --limit-to %s' % options.limit_to) + polygon_timer = imposm.util.Timer('reading', logger) + polygon = load_geom(options.limit_to) + polygon_timer.stop() + if polygon is None: + print 'ERROR: No valid polygon/multipolygon found' + sys.exit(1) + mappings = {} execfile(mapping_file, mappings) - tag_mapping = TagMapper([m for n, m in mappings.iteritems() - if isinstance(m, imposm.mapping.Mapping)]) + tag_mapping = TagMapper([m for n, m in mappings.iteritems() + if isinstance(m, imposm.mapping.Mapping)], limit_to=polygon) if 'IMPOSM_MULTIPOLYGON_REPORT' in os.environ: imposm.config.imposm_multipolygon_report = float(os.environ['IMPOSM_MULTIPOLYGON_REPORT']) if 'IMPOSM_MULTIPOLYGON_MAX_RING' in os.environ: imposm.config.imposm_multipolygon_max_ring = int(os.environ['IMPOSM_MULTIPOLYGON_MAX_RING']) + if options.table_prefix: + options.table_prefix = options.table_prefix.rstrip('_') + '_' + if options.table_prefix_production: + options.table_prefix_production = options.table_prefix_production.rstrip('_') + '_' + if options.table_prefix_backup: + options.table_prefix_backup = options.table_prefix_backup.rstrip('_') + '_' + if (options.write or options.optimize or options.deploy_tables or options.remove_backup_tables or options.recover_tables): db_conf = mappings['db_conf'] if options.table_prefix: db_conf.prefix = options.table_prefix else: - options.table_prefix = db_conf.prefix - + options.table_prefix = db_conf.prefix.rstrip('_') + '_' + if options.connection: from imposm.db.config import db_conf_from_string db_conf = db_conf_from_string(options.connection, db_conf) @@ -183,14 +216,12 @@ def main(argv=None): if options.user: from getpass import getpass db_conf.password = getpass('password for %(user)s at %(host)s:' % db_conf) - + if options.proj: db_conf.proj = options.proj - - logger = imposm.util.ProgressLog - + imposm_timer = imposm.util.Timer('imposm', logger) - + if options.read: if not options.merge_cache: cache_files = glob.glob(os.path.join(options.cache_dir, 'imposm_*.cache')) @@ -204,18 +235,18 @@ def main(argv=None): sys.exit(2) for cache_file in cache_files: os.unlink(cache_file) - + cache = OSMCache(options.cache_dir) - + if options.read: read_timer = imposm.util.Timer('reading', logger) - + if not args: print "no file(s) supplied" sys.exit(2) reader = ImposmReader(tag_mapping, cache=cache, merge=options.merge_cache, - pool_size=options.concurrency, logger=logger) + pool_size=options.concurrency, logger=logger_parser) reader.estimated_coords = imposm.util.estimate_records(args) for arg in args: logger.message('## reading %s' % arg) @@ -225,7 +256,7 @@ def main(argv=None): if options.write: db = DB(db_conf) write_timer = imposm.util.Timer('writing', logger) - + logger.message('## dropping/creating tables') if not options.dry_run: db.create_tables(tag_mapping.mappings) @@ -237,7 +268,7 @@ def main(argv=None): db.create_views(mappings, ignore_errors=True) db.commit() - writer = ImposmWriter(tag_mapping, db, cache=cache, + writer = ImposmWriter(tag_mapping, db, cache=cache, pool_size=options.concurrency, logger=logger, dry_run=options.dry_run) writer.relations() @@ -251,14 +282,19 @@ def main(argv=None): generalized_timer = imposm.util.Timer('generalizing tables', logger) db.create_generalized_tables(mappings) generalized_timer.stop() - + logger.message('## creating union views') view_timer = imposm.util.Timer('creating views', logger) db.create_views(mappings) view_timer.stop() + logger.message('## creating geometry indexes') + index_timer = imposm.util.Timer('creating indexes', logger) + db.post_insert(mappings); + index_timer.stop() + db.commit() - + write_timer.stop() if options.optimize: @@ -267,27 +303,27 @@ def main(argv=None): logger.message('## optimizing tables') db.optimize(mappings) optimize_timer.stop() - + if options.recover_tables: assert not options.deploy_tables, ('cannot swap and recover production ' 'tables at the same time') options.table_prefix, options.table_prefix_backup = \ options.table_prefix_backup, options.table_prefix - + if options.deploy_tables or options.recover_tables: db = DB(db_conf) - db.swap_tables(options.table_prefix, + db.swap_tables(options.table_prefix, options.table_prefix_production, options.table_prefix_backup) db.remove_views(options.table_prefix) db.db_conf.prefix = options.table_prefix_production db.create_views(mappings) db.commit() - + if options.remove_backup_tables: db = DB(db_conf) db.remove_tables(options.table_prefix_backup) db.commit() - + imposm_timer.stop() if __name__ == '__main__': diff --git a/imposm/cache/tc_tmp.py b/imposm/cache/tc_tmp.py deleted file mode 100644 index bfea081..0000000 --- a/imposm/cache/tc_tmp.py +++ /dev/null @@ -1,577 +0,0 @@ -from imposm.base import Node, Way, Relation -import sys -import os - -import marshal - -from ctypes import CDLL -from ctypes.util import find_library as _find_library -from ctypes import ( - c_void_p, - c_char_p, - c_int, - c_double, - c_long, - c_int64, - c_uint32, - byref, - cast, - Structure, - POINTER, - pointer, - create_string_buffer, - addressof, - sizeof, - memmove, - string_at, - CFUNCTYPE, -) - -default_locations = dict( - darwin=dict( - paths = ['/opt/local/lib'], - exts = ['.dylib'], - ), - win32=dict( - paths = [os.path.dirname(os.__file__) + '/../../../DLLs'], - exts = ['.dll'] - ) -) - -#additional_lib_path = os.environ.get('MAPPROXY_LIB_PATH') -#if additional_lib_path: -# additional_lib_path = additional_lib_path.split(os.pathsep) -# additional_lib_path.reverse() -# for locs in default_locations.values(): -# for path in additional_lib_path: -# locs['paths'].insert(0, path) -# -def load_library(lib_names, locations_conf=default_locations): - """ - Load the `lib_name` library with ctypes. - If ctypes.util.find_library does not find the library, - different path and filename extensions will be tried. - - Retruns the loaded library or None. - """ - if isinstance(lib_names, basestring): - lib_names = [lib_names] - - for lib_name in lib_names: - lib = load_library_(lib_name, locations_conf) - if lib is not None: return lib - -def load_library_(lib_name, locations_conf=default_locations): - lib_path = find_library(lib_name) - - if lib_path: - return CDLL(lib_path) - - if sys.platform in locations_conf: - paths = locations_conf[sys.platform]['paths'] - exts = locations_conf[sys.platform]['exts'] - lib_path = find_library(lib_name, paths, exts) - - if lib_path: - return CDLL(lib_path) - - -def find_library(lib_name, paths=None, exts=None): - """ - Search for library in all permutations of `paths` and `exts`. - If nothing is found None is returned. - """ - if not paths or not exts: - lib = _find_library(lib_name) - if lib is None and lib_name.startswith('lib'): - lib = _find_library(lib_name[3:]) - return lib - - for lib_name in [lib_name] + ([lib_name[3:]] if lib_name.startswith('lib') else []): - for path in paths: - for ext in exts: - lib_path = os.path.join(path, lib_name + ext) - if os.path.exists(lib_path): - return lib_path - - return None - -#cdef extern from "Python.h": -# object PyString_FromStringAndSize(char *s, Py_ssize_t len) -# -#cdef extern from "marshal.h": -# object PyMarshal_ReadObjectFromString(char *string, Py_ssize_t len) -# object PyMarshal_WriteObjectToString(object value, int version) -# - -TCBDB = c_void_p -BDBCUR = c_void_p -TCCMP = CFUNCTYPE(c_int, c_char_p, c_int, c_char_p, c_int, c_void_p) - -def init_libtokyo_util(): - libtokyocabinet = load_library('libtokyocabinet') - - if libtokyocabinet is None: return - - #??? - #libtokyocabinet.TCCMP.argtypes = [] - #libtokyocabinet.TCCMP.restype = c_int - # - libtokyocabinet.tccmpint32.argtypes = [] - libtokyocabinet.tccmpint32.restype = c_int - - libtokyocabinet.tccmpint64.argtypes = [] - libtokyocabinet.tccmpint64.restype = c_long - - libtokyocabinet.tcbdbnew.argtypes = [] - libtokyocabinet.tcbdbnew.restype = TCBDB - - libtokyocabinet.tcbdbdel.argtypes = [TCBDB] - libtokyocabinet.tcbdbdel.restype = None - - libtokyocabinet.tcbdbecode.argtypes = [TCBDB] - libtokyocabinet.tcbdbecode.restype = c_int - - libtokyocabinet.tcbdbtune.argtypes = [TCBDB, c_int, c_int, c_int, c_int, c_int, c_int] - libtokyocabinet.tcbdbtune.restype = c_int - - libtokyocabinet.tcbdbsetcache.argtypes = [TCBDB, c_int, c_int] - libtokyocabinet.tcbdbsetcache.restype = c_int - - CMPFUNC = CFUNCTYPE(c_long) - libtokyocabinet.tcbdbsetcmpfunc.argtypes = [TCBDB, c_void_p, c_void_p] - libtokyocabinet.tcbdbsetcmpfunc.restype = c_int - - libtokyocabinet.tcbdbsetmutex.argtypes = [TCBDB] - libtokyocabinet.tcbdbsetmutex.restype = c_int - - libtokyocabinet.tcbdbopen.argtypes = [TCBDB, c_char_p, c_int] - libtokyocabinet.tcbdbopen.restype = c_int - - libtokyocabinet.tcbdbclose.argtypes = [TCBDB] - libtokyocabinet.tcbdbclose.restype = c_int - - libtokyocabinet.tcbdbput.argtypes = [TCBDB, c_void_p, c_int, c_void_p, c_int] - libtokyocabinet.tcbdbput.restype = c_int - - libtokyocabinet.tcbdbget.argtypes = [TCBDB, c_void_p, c_int, POINTER(c_int)] - libtokyocabinet.tcbdbget.restype = c_void_p - - libtokyocabinet.tcbdbget3.argtypes = [TCBDB, c_void_p, c_int, POINTER(c_int)] - libtokyocabinet.tcbdbget3.restype = c_void_p - - libtokyocabinet.tcbdbrnum.argtypes = [TCBDB] - libtokyocabinet.tcbdbrnum.restype = c_long - - libtokyocabinet.tcbdbcurnew.argtypes = [TCBDB] - libtokyocabinet.tcbdbcurnew.restypes = BDBCUR - - libtokyocabinet.tcbdbcurdel.argtypes = [BDBCUR] - libtokyocabinet.tcbdbcurdel.restypes = None - - libtokyocabinet.tcbdbcurfirst.argtypes = [BDBCUR] - libtokyocabinet.tcbdbcurfirst.restypes = c_int - - libtokyocabinet.tcbdbcurnext.argtypes = [BDBCUR] - libtokyocabinet.tcbdbcurnext.restypes = c_int - - libtokyocabinet.tcbdbcurkey3.argtypes = [BDBCUR, POINTER(c_int)] - libtokyocabinet.tcbdbcurkey3.restypes = c_void_p - - libtokyocabinet.tcbdbcurval3.argtypes = [BDBCUR, POINTER(c_int)] - libtokyocabinet.tcbdbcurval3.restypes = c_void_p - - return libtokyocabinet - -libtokyocabinet = init_libtokyo_util() - -BDBFOPEN = 1 << 0 -BDBFFATAL = 1 << 1 - -BDBOREADER = 1 << 0 # /* open as a reader */ -BDBOWRITER = 1 << 1 # /* open as a writer */ -BDBOCREAT = 1 << 2 # /* writer creating */ -BDBOTRUNC = 1 << 3 # /* writer truncating */ -BDBONOLCK = 1 << 4 # /* open without locking */ -BDBOLCKNB = 1 << 5 # /* lock without blocking */ -BDBOTSYNC = 1 << 6 # /* synchronize every transaction */ - -BDBTLARGE = 1 << 0 # /* use 64-bit bucket array */ -BDBTDEFLATE = 1 << 1 # /* compress each page with Deflate */ -BDBTBZIP = 1 << 2 # /* compress each record with BZIP2 */ -BDBTTCBS = 1 << 3 # /* compress each page with TCBS */ -BDBTEXCODEC = 1 << 4 # /* compress each record with outer functions */ - -COORD_FACTOR = 11930464.7083 # ((2<<31)-1)/360.0 - -def _coord_to_uint32(x): - return int((x + 180.0) * COORD_FACTOR) - -def _uint32_to_coord(x): - return ((x / COORD_FACTOR) - 180.0) - -def coord_struct(x, y): - return Coord(x=_coord_to_uint32(x), y=_coord_to_uint32(y)) - -class Coord(Structure): - _fields_ = [("x", c_uint32), - ("y", c_uint32)] - -_modes = { - 'w': BDBOWRITER | BDBOCREAT, - 'r': BDBOREADER | BDBONOLCK, -} - -class BDB(object): - def __init__(self, filename, mode='w', estimated_records=0): - self.db = libtokyocabinet.tcbdbnew() - self._cur = None - self._opened = 0 - self.filename = filename - self._tune_db(estimated_records) - - tccmpint64 = TCCMP(('tccmpint64', libtokyocabinet)) - libtokyocabinet.tcbdbsetcmpfunc(self.db, tccmpint64, None) - - # libtokyocabinet.tcbdbsetcmpfunc(self.db, libtokyocabinet.tccmpint64._get_address(), None) - if not libtokyocabinet.tcbdbopen(self.db, filename, _modes[mode]): - raise IOError(libtokyocabinet.tcbdbecode(self.db)) - self._opened = 1 - - def _tune_db(self, estimated_records): - if estimated_records: - lmemb = 128 # default - nmemb = -1 - fpow = 13 # 2^13 = 8196 - bnum = int((estimated_records*3)/lmemb) - libtokyocabinet.tcbdbtune(self.db, lmemb, nmemb, bnum, 5, fpow, BDBTLARGE | BDBTDEFLATE) - else: - libtokyocabinet.tcbdbtune(self.db, -1, -1, -1, 5, 13, BDBTLARGE | BDBTDEFLATE) - - def get(self, osmid): - """ - Return object with given id. - Returns None if id is not stored. - """ - ret_size = c_int() - ret = libtokyocabinet.tcbdbget3(self.db, byref(c_int64(osmid)), 8, byref(ret_size)) - if not ret: return None - data = string_at(ret, ret_size.value) - return self._obj(osmid, marshal.loads(data)) - - def get_raw(self, osmid): - """ - Return object with given id. - Returns None if id is not stored. - """ - ret_size = c_int() - ret = libtokyocabinet.tcbdbget3(self.db, byref(c_int64(osmid)), 8, byref(ret_size)) - if not ret: return None - return string_at(ret, ret_size.value) - - def put(self, osmid, data): - return self.put_marshaled(osmid, marshal.dumps(data, 2)) - - def put_marshaled(self, osmid, data): - return libtokyocabinet.tcbdbput(self.db, byref(c_int64(osmid)), 8, byref(create_string_buffer(data)), len(data)) - - def _obj(self, osmid, data): - """ - Create an object from the id and unmarshaled data. - Should be overridden by subclasses. - """ - return data - - def __iter__(self): - """ - Return an iterator over the database. - Resets any existing iterator. - """ - if self._cur: - libtokyocabinet.tcbdbcurdel(self._cur) - self._cur = None - self._cur = libtokyocabinet.tcbdbcurnew(self.db) - print self.db, self._cur - #libtokyocabinet.tcbdbcurfirst(self._cur) - #if not libtokyocabinet.tcbdbcurfirst(self._cur): - # return iter([]) - return self - - def __contains__(self, osmid): - ret_size = c_int() - ret = tcbdbget3(self.db, byref(c_int64(osmid)), 8, byref(ret_size)); - if ret: - return 1 - else: - return 0 - - def __len__(self): - return libtokyocabinet.tcbdbrnum(self.db) - - def next(self): - """ - Return next item as object. - """ - #cdef int64_t osmid - if not self._cur: raise StopIteration - - osmid, data = self._get_cur() - - # advance cursor, set to NULL if at the end - if libtokyocabinet.tcbdbcurnext(self._cur) == 0: - libtokyocabinet.tcbdbcurdel(self._cur) - self._cur = None - - # return objectified item - return self._obj(osmid, data) - - def _get_cur(self): - """ - Return the current object at the current cursor position - as a tuple of the id and the unmarshaled data. - """ - size = c_int() - ret = libtokyocabinet.tcbdbcurkey3(self._cur, byref(size)) - osmid = byref(ret)[0] - ret = libtokyocabinet.tcbdbcurval3(self._cur, byref(size)) - data = string_at(ret, size.value) - value = marshal.loads(data) - return osmid, value - - def close(self): - if self._opened: - libtokyocabinet.tcbdbclose(self.db) - self._opened = 0 - - def __dealloc__(self): - if self._opened: - libtokyocabinet.tcbdbclose(self.db) - libtokyocabinet.tcbdbdel(self.db) - -class CoordDB(BDB): - def put(self, osmid, x, y): - return self._put(osmid, x, y) - - def put_marshaled(self, osmid, x, y): - return self._put(osmid, x, y) - - def _put(self, osmid, x, y): - p = coord_struct(x, y) - return libtokyocabinet.tcbdbput(self.db, byref(c_int64(osmid)), 8, byref(p), 8) - - def get(self, osmid): - ret_size = c_int - value = libtokyocabinet.tcbdbget3(self.db, byref(c_int64(osmid)), 8, byref(ret_size)) - if not value: return - coord = Coord() - memmove(addressof(coord), value, sizeof(Coord)) - return _uint32_to_coord(coord.x), _uint32_to_coord(coord.y) - - def get_coords(self, refs): - ret_size = c_int() - coords = list() - for osmid in refs: - value = libtokyocabinet.tcbdbget3(self.db, byref(c_int64(osmid)), 8, byref(ret_size)) - if not value: return - coord = Coord() - memmove(addressof(coord), value, sizeof(Coord)) - coords.append((_uint32_to_coord(coord.x), _uint32_to_coord(coord.y))) - - return coords - - def _get_cur(self): - size = c_int() - ret = tcbdbcurkey3(self._cur, byref(size)) - osmid = byref(ret)[0] - value = libtokyocabinet.tcbdbcurval3(self._cur, byef(size)) - return osmid, (_uint32_to_coord(value.x), _uint32_to_coord(value.y)) - - def _obj(self, osmid, data): - return osmid, data - -class NodeDB(BDB): - def put(self, osmid, tags, pos): - return self.put_marshaled(osmid, marshal.dumps((tags, pos), 2)) - - def put_marshaled(self, osmid, data): - return libtokyocabinet.tcbdbput(self.db, byref(c_int64(osmid)), 8, byref(create_string_buffer(data)), len(data)) - - def _obj(self, osmid, data): - return Node(osmid, data[0], data[1]) - -class InsertedWayDB(BDB): - def put(self, osmid): - return libtokyocabinet.tcbdbput(self.db, byref(c_int64(osmid)), 8, 'x', 1); - - def __next__(self): - """ - Return next item as object. - """ - size = None - - if not self._cur: raise StopIteration - - ret = libtokyocabinet.tcbdbcurkey3(self._cur, byref(size)) - osmid = byref(ret)[0] - - # advance cursor, set to NULL if at the end - if libtokyocabinet.tcbdbcurnext(self._cur) == 0: - libtokyocabinet.tcbdbcurdel(self._cur) - self._cur = None - - return osmid - -class RefTagDB(BDB): - """ - Database for items with references and tags (i.e. ways/relations). - """ - def put(self, osmid, tags, refs): - return self.put_marshaled(osmid, marshal.dumps((tags, refs), 2)) - - def put_marshaled(self, osmid, data): - return libtokyocabinet.tcbdbput(self.db, byref(c_int64(osmid)), sizeof(int64_t), byref(create_string_buffer(data)), len(data)) - -class WayDB(RefTagDB): - def _obj(self, osmid, data): - return Way(osmid, data[0], data[1]) - -class RelationDB(RefTagDB): - def _obj(self, osmid, data): - return Relation(osmid, data[0], data[1]) - -from imposm.cache.internal import DeltaCoords as _DeltaCoords -from collections import deque -import bisect - -def unzip_nodes(nodes): - ids, lons, lats = [], [], [] - last_id = last_lon = last_lat = 0 - for id, lon_f, lat_f in nodes: - lon = _coord_to_uint32(lon_f) - lat = _coord_to_uint32(lat_f) - - ids.append(id - last_id) - lons.append(lon - last_lon) - lats.append(lat - last_lat) - last_id = id - last_lon = lon - last_lat = lat - - return ids, lons, lats - -def zip_nodes(ids, lons, lats): - nodes = [] - last_id = last_lon = last_lat = 0 - - for i in range(len(ids)): - last_id += ids[i] - last_lon += lons[i] - last_lat += lats[i] - - nodes.append(( - last_id, - _uint32_to_coord(last_lon), - _uint32_to_coord(last_lat) - )) - return nodes - -class DeltaNodes(object): - def __init__(self, data=None): - self.nodes = [] - self.changed = False - if data: - self.deserialize(data) - - def changed(self): - return self.changed - - def get(self, osmid): - i = bisect.bisect(self.nodes, (osmid, )) - if i != len(self.nodes) and self.nodes[i][0] == osmid: - return self.nodes[i][1:] - return None - - def add(self, osmid, lon, lat): - # todo: overwrite - self.changed = True - if self.nodes and self.nodes[-1][0] < osmid: - self.nodes.append((osmid, lon, lat)) - else: - bisect.insort(self.nodes, (osmid, lon, lat)) - - def serialize(self): - ids, lons, lats = unzip_nodes(self.nodes) - nodes = _DeltaCoords() - nodes.ids.extend(ids) - nodes.lons.extend(lons) - nodes.lats.extend(lats) - return nodes.SerializeToString() - - def deserialize(self, data): - nodes = _DeltaCoords() - nodes.ParseFromString(data) - self.nodes = zip_nodes( - nodes.ids, nodes.lons, nodes.lats) - -class DeltaCoordsDB(object): - def __init__(self, filename, mode='w', estimated_records=0, delta_nodes_cache_size=100, delta_nodes_size=6): - self.db = BDB(filename, mode, estimated_records) - self.mode = mode - self.delta_nodes = {} - self.delta_node_ids = deque() - self.delta_nodes_cache_size = delta_nodes_cache_size - self.delta_nodes_size = delta_nodes_size - - def put(self, osmid, lon, lat): - if self.mode == 'r': - return None - delta_id = osmid >> self.delta_nodes_size - if delta_id not in self.delta_nodes: - self.fetch_delta_node(delta_id) - delta_node = self.delta_nodes[delta_id] - delta_node.add(osmid, lon, lat) - return True - - put_marshaled = put - - def get(self, osmid): - delta_id = osmid >> self.delta_nodes_size - if delta_id not in self.delta_nodes: - self.fetch_delta_node(delta_id) - return self.delta_nodes[delta_id].get(osmid) - - def get_coords(self, osmids): - coords = [] - for osmid in osmids: - coord = self.get(osmid) - if coord is None: - return - coords.append(coord) - return coords - - def close(self): - for node_id, node in self.delta_nodes.iteritems(): - self._put(node_id, node) - self.delta_nodes = {} - self.delta_node_ids = deque() - self.db.close() - - def _put(self, delta_id, delta_node): - data = delta_node.serialize() - self.db.put_marshaled(delta_id, data) - - def _get(self, delta_id): - return DeltaNodes(data=self.db.get_raw(delta_id)) - - def fetch_delta_node(self, delta_id): - if len(self.delta_node_ids) >= self.delta_nodes_cache_size: - rm_id = self.delta_node_ids.popleft() - rm_node = self.delta_nodes.pop(rm_id) - if rm_node.changed: - self._put(rm_id, rm_node) - new_node = self._get(delta_id) - if new_node is None: - new_node = DeltaNodes() - self.delta_nodes[delta_id] = new_node - self.delta_node_ids.append(delta_id) - diff --git a/imposm/db/postgis.py b/imposm/db/postgis.py index b6b732e..6514027 100644 --- a/imposm/db/postgis.py +++ b/imposm/db/postgis.py @@ -1,11 +1,11 @@ -# Copyright 2011 Omniscale (http://omniscale.com) -# +# Copyright 2011-2012 Omniscale (http://omniscale.com) +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -13,6 +13,8 @@ # limitations under the License. import time +import uuid +from contextlib import contextmanager import psycopg2 import psycopg2.extensions @@ -23,14 +25,26 @@ log = logging.getLogger(__name__) from imposm import config from imposm.mapping import UnionView, GeneralizedTable, Mapping +unknown = object() + class PostGISDB(object): - def __init__(self, db_conf): + insert_data_format = 'tuple' + + def __init__(self, db_conf, use_geometry_columns_table=unknown): self.db_conf = db_conf self.srid = int(db_conf['proj'].split(':')[1]) + self._insert_stmts = {} self._connection = None self._cur = None + if use_geometry_columns_table is unknown: + if self.is_postgis_2(): + use_geometry_columns_table = False + else: + use_geometry_columns_table = True + self.use_geometry_columns_table = use_geometry_columns_table + @property def table_prefix(self): return self.db_conf.prefix.rstrip('_') + '_' @@ -38,6 +52,12 @@ class PostGISDB(object): def to_tablename(self, name): return self.table_prefix + name.lower() + def is_postgis_2(self): + cur = self.connection.cursor() + cur.execute('SELECT postgis_version()') + version_string = cur.fetchone()[0] + return version_string.strip()[0] == '2' + @property def connection(self): if not self._connection: @@ -55,16 +75,27 @@ class PostGISDB(object): self._connection.set_isolation_level( psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED) return self._connection - + def commit(self): self.connection.commit() - + @property def cur(self): if self._cur is None: self._cur = self.connection.cursor() return self._cur - + + @contextmanager + def savepoint(self, cur, raise_errors=False): + savepoint_name = 'savepoint' + uuid.uuid4().get_hex() + try: + cur.execute('SAVEPOINT %s' % savepoint_name) + yield + except psycopg2.ProgrammingError: + cur.execute('ROLLBACK TO SAVEPOINT %s' % savepoint_name) + if raise_errors: + raise + def insert(self, mapping, insert_data, tries=0): insert_stmt = self.insert_stmt(mapping) try: @@ -91,10 +122,24 @@ class PostGISDB(object): self.connection.commit() self.connection.commit() - + + def post_insert(self, mappings): + mappings = [m for m in mappings.values() if isinstance(m, (GeneralizedTable, Mapping))] + for mapping in mappings: + table_name = self.to_tablename(mapping.name) + self.create_geom_index(table_name) + + def create_geom_index(self, table_name): + idx_name = '%s_geom' % table_name + cur = self.connection.cursor() + cur.execute(""" + CREATE INDEX "%s" ON "%s" USING GIST (geometry) + """ % (idx_name,table_name)) + self.connection.commit() + def geom_wrapper(self, geom): return psycopg2.Binary(geom.wkb) - + def reconnect(self): if self._connection: try: @@ -128,14 +173,17 @@ class PostGISDB(object): for mapping in mappings: self.create_table(mapping) + def drop_table_or_view(self, cur, name): + with self.savepoint(cur): + cur.execute('DROP TABLE "' + name + '" CASCADE') + with self.savepoint(cur): + cur.execute('DROP VIEW "' + name + '" CASCADE') + def create_table(self, mapping): tablename = self.table_prefix + mapping.name cur = self.connection.cursor() - cur.execute('SAVEPOINT pre_drop_tables') - try: - cur.execute('DROP TABLE "' + tablename + '" CASCADE') - except psycopg2.ProgrammingError: - cur.execute('ROLLBACK TO SAVEPOINT pre_drop_tables') + + self.drop_table_or_view(cur, tablename) extra_fields = '' for n, t in mapping.fields: @@ -145,7 +193,7 @@ class PostGISDB(object): serial_column = "id SERIAL PRIMARY KEY," else: serial_column = "" - + cur.execute(""" CREATE TABLE "%s" ( %s @@ -153,27 +201,42 @@ class PostGISDB(object): %s ); """ % (tablename, serial_column, extra_fields)) - cur.execute(""" - SELECT AddGeometryColumn ('', '%(tablename)s', 'geometry', - %(srid)s, '%(pg_geometry_type)s', 2) - """ % dict(tablename=tablename, srid=self.srid, - pg_geometry_type=mapping.geom_type)) + self.create_geometry_column(cur, tablename, mapping) + + self.create_field_indices(cur=cur, mapping=mapping, tablename=tablename) + + + def create_geometry_column(self, cur, tablename, mapping): + if self.use_geometry_columns_table: + cur.execute(""" + SELECT AddGeometryColumn ('', '%(tablename)s', 'geometry', + %(srid)s, '%(pg_geometry_type)s', 2) + """ % dict(tablename=tablename, srid=self.srid, + pg_geometry_type=mapping.geom_type)) + else: + cur.execute(""" + ALTER TABLE %(tablename)s ADD COLUMN geometry geometry(%(pg_geometry_type)s, %(srid)s); + """ % dict(tablename=tablename, srid=self.srid, + pg_geometry_type=mapping.geom_type)) + + def create_field_indices(self, cur, mapping, tablename): for n, t in mapping.fields: if isinstance(t, TrigramIndex): cur.execute(""" CREATE INDEX "%(tablename)s_trgm_idx_%(column)s" ON "%(tablename)s" USING GIST ("%(column)s" gist_trgm_ops) """ % dict(tablename=tablename, column=n)) + if isinstance(t, (StringIndex, Index)): + cur.execute(""" + CREATE INDEX "%(tablename)s_idx_%(column)s" ON "%(tablename)s" ("%(column)s") + """ % dict(tablename=tablename, column=n)) - cur.execute(""" - CREATE INDEX "%(tablename)s_geom" ON "%(tablename)s" USING GIST (geometry) - """ % dict(tablename=tablename)) - def swap_tables(self, new_prefix, existing_prefix, backup_prefix): cur = self.connection.cursor() self.remove_tables(backup_prefix) - + self.remove_views(backup_prefix) + cur.execute('SELECT tablename FROM pg_tables WHERE tablename like %s', (existing_prefix + '%', )) existing_tables = [] for row in cur: @@ -182,6 +245,14 @@ class PostGISDB(object): # check for overlapping prefixes: osm_ but not osm_new_ or osm_backup_ existing_tables.append(table_name) + cur.execute('SELECT viewname FROM pg_views WHERE viewname like %s', (existing_prefix + '%', )) + existing_views = [] + for row in cur: + view_name = row[0] + if view_name.startswith(existing_prefix) and not view_name.startswith((new_prefix, backup_prefix)): + # check for overlapping prefixes: osm_ but not osm_new_ or osm_backup_ + existing_views.append(view_name) + cur.execute('SELECT indexname FROM pg_indexes WHERE indexname like %s', (existing_prefix + '%', )) existing_indexes = set() for row in cur: @@ -189,7 +260,7 @@ class PostGISDB(object): if index_name.startswith(existing_prefix) and not index_name.startswith((new_prefix, backup_prefix)): # check for overlapping prefixes: osm_ but not osm_new_ or osm_backup_ existing_indexes.add(index_name) - + cur.execute('SELECT relname FROM pg_class WHERE relname like %s', (existing_prefix + '%_id_seq', )) existing_seq = set() for row in cur: @@ -204,73 +275,100 @@ class PostGISDB(object): table_name = row[0] new_tables.append(table_name) + cur.execute('SELECT viewname FROM pg_views WHERE viewname like %s', (new_prefix + '%', )) + new_views = [] + for row in cur: + view_name = row[0] + new_views.append(view_name) + cur.execute('SELECT indexname FROM pg_indexes WHERE indexname like %s', (new_prefix + '%', )) new_indexes = set() for row in cur: index_name = row[0] new_indexes.add(index_name) - + cur.execute('SELECT relname FROM pg_class WHERE relname like %s', (new_prefix + '%_id_seq', )) new_seq = [] for row in cur: seq_name = row[0] new_seq.append(seq_name) - + if not new_tables: raise RuntimeError('did not found tables to swap') - + # rename existing tables (osm_) to backup_prefix (osm_backup_) for table_name in existing_tables: rename_to = table_name.replace(existing_prefix, backup_prefix) cur.execute('ALTER TABLE "%s" RENAME TO "%s"' % (table_name, rename_to)) for idx in existing_indexes: - if idx in (table_name + '_geom', table_name + '_pkey') or idx.startswith(table_name + '_trgm_idx_'): + if idx in (table_name + '_geom', table_name + '_pkey') or idx.startswith(table_name + '_trgm_idx_') or idx.startswith(table_name + '_idx_'): new_idx = idx.replace(table_name, rename_to, 1) cur.execute('ALTER INDEX "%s" RENAME TO "%s"' % (idx, new_idx)) if table_name + '_id_seq' in existing_seq: cur.execute('ALTER SEQUENCE "%s" RENAME TO "%s"' % (table_name + '_id_seq', rename_to + '_id_seq')) - cur.execute('UPDATE geometry_columns SET f_table_name = %s WHERE f_table_name = %s', (rename_to, table_name)) - + if self.use_geometry_columns_table: + cur.execute('UPDATE geometry_columns SET f_table_name = %s WHERE f_table_name = %s', (rename_to, table_name)) + + # rename existing views (osm_) to backup_prefix (osm_backup_) + for view_name in existing_views: + rename_to = view_name.replace(existing_prefix, backup_prefix) + cur.execute('ALTER VIEW "%s" RENAME TO "%s"' % (view_name, rename_to)) + + if self.use_geometry_columns_table: + cur.execute('UPDATE geometry_columns SET f_table_name = %s WHERE f_table_name = %s', (rename_to, view_name)) + # rename new tables (osm_new_) to existing_prefix (osm_) for table_name in new_tables: rename_to = table_name.replace(new_prefix, existing_prefix) cur.execute('ALTER TABLE "%s" RENAME TO "%s"' % (table_name, rename_to)) for idx in new_indexes: - if idx in (table_name + '_geom', table_name + '_pkey') or idx.startswith(table_name + '_trgm_idx_'): + if idx in (table_name + '_geom', table_name + '_pkey') or idx.startswith(table_name + '_trgm_idx_') or idx.startswith(table_name + '_idx_'): new_idx = idx.replace(table_name, rename_to, 1) cur.execute('ALTER INDEX "%s" RENAME TO "%s"' % (idx, new_idx)) if table_name + '_id_seq' in new_seq: cur.execute('ALTER SEQUENCE "%s" RENAME TO "%s"' % (table_name + '_id_seq', rename_to + '_id_seq')) - cur.execute('UPDATE geometry_columns SET f_table_name = %s WHERE f_table_name = %s', (rename_to, table_name)) - + if self.use_geometry_columns_table: + cur.execute('UPDATE geometry_columns SET f_table_name = %s WHERE f_table_name = %s', (rename_to, table_name)) + + # rename new views (osm_new_) to existing_prefix (osm_) + for view_name in new_views: + rename_to = view_name.replace(new_prefix, existing_prefix) + cur.execute('ALTER VIEW "%s" RENAME TO "%s"' % (view_name, rename_to)) + + if self.use_geometry_columns_table: + cur.execute('UPDATE geometry_columns SET f_table_name = %s WHERE f_table_name = %s', (rename_to, view_name)) + + def remove_tables(self, prefix): cur = self.connection.cursor() cur.execute('SELECT tablename FROM pg_tables WHERE tablename like %s', (prefix + '%', )) remove_tables = [row[0] for row in cur] - + for table_name in remove_tables: cur.execute("DROP TABLE %s CASCADE" % (table_name, )) - cur.execute("DELETE FROM geometry_columns WHERE f_table_name = %s", (table_name, )) - + if self.use_geometry_columns_table: + cur.execute("DELETE FROM geometry_columns WHERE f_table_name = %s", (table_name, )) + def remove_views(self, prefix): cur = self.connection.cursor() cur.execute('SELECT viewname FROM pg_views WHERE viewname like %s', (prefix + '%', )) remove_views = [row[0] for row in cur] - + for view_name in remove_views: cur.execute('DROP VIEW "%s" CASCADE' % (view_name, )) - cur.execute("DELETE FROM geometry_columns WHERE f_table_name = %s", (view_name, )) - - + if self.use_geometry_columns_table: + cur.execute("DELETE FROM geometry_columns WHERE f_table_name = %s", (view_name, )) + + def create_views(self, mappings, ignore_errors=False): for mapping in mappings.values(): if isinstance(mapping, UnionView): PostGISUnionView(self, mapping).create(ignore_errors=ignore_errors) - + def create_generalized_tables(self, mappings): mappings = [m for m in mappings.values() if isinstance(m, GeneralizedTable)] for mapping in sorted(mappings, key=lambda x: x.name, reverse=True): @@ -298,7 +396,6 @@ class PostGISDB(object): cur.execute("VACUUM ANALYZE") self.connection.set_isolation_level(old_isolation_level) - class PostGISUnionView(object): def __init__(self, db, mapping): self.mapping = mapping @@ -322,10 +419,11 @@ class PostGISUnionView(object): selects = '\nUNION ALL\n'.join(selects) stmt = 'CREATE VIEW "%s" as (\n%s\n)' % (self.view_name, selects) - + return stmt def _geom_table_stmt(self): + assert self.db.use_geometry_columns_table stmt = "insert into geometry_columns values ('', 'public', '%s', 'geometry', 2, %d, 'GEOMETRY')" % ( self.view_name, self.db.srid) return stmt @@ -349,22 +447,18 @@ class PostGISUnionView(object): def create(self, ignore_errors): cur = self.db.connection.cursor() cur.execute('BEGIN') - try: - cur.execute('SAVEPOINT pre_create_view') - cur.execute('SELECT * FROM pg_views WHERE viewname = %s', (self.view_name, )) - if cur.fetchall(): - cur.execute('DROP VIEW %s' % (self.view_name, )) + + self.db.drop_table_or_view(cur, self.view_name) + + with self.db.savepoint(cur, raise_errors=not ignore_errors): cur.execute(self._view_stmt()) - except psycopg2.ProgrammingError: - cur.execute('ROLLBACK TO SAVEPOINT pre_create_view') - if not ignore_errors: - raise - cur.execute('SELECT * FROM geometry_columns WHERE f_table_name = %s', (self.view_name, )) - if cur.fetchall(): - # drop old entry to handle changes of SRID - cur.execute('DELETE FROM geometry_columns WHERE f_table_name = %s', (self.view_name, )) - cur.execute(self._geom_table_stmt()) + if self.db.use_geometry_columns_table: + cur.execute('SELECT * FROM geometry_columns WHERE f_table_name = %s', (self.view_name, )) + if cur.fetchall(): + # drop old entry to handle changes of SRID + cur.execute('DELETE FROM geometry_columns WHERE f_table_name = %s', (self.view_name, )) + cur.execute(self._geom_table_stmt()) class PostGISGeneralizedTable(object): @@ -373,11 +467,8 @@ class PostGISGeneralizedTable(object): self.mapping = mapping self.table_name = db.to_tablename(mapping.name) - def _idx_stmt(self): - return 'CREATE INDEX "%s_geom" ON "%s" USING GIST (geometry)' % ( - self.table_name, self.table_name) - def _geom_table_stmt(self): + assert self.db.use_geometry_columns_table stmt = "insert into geometry_columns values ('', 'public', '%s', 'geometry', 2, %d, 'GEOMETRY')" % ( self.table_name, self.db.srid) return stmt @@ -386,18 +477,17 @@ class PostGISGeneralizedTable(object): fields = ', '.join([n for n, t in self.mapping.fields]) if fields: fields += ',' + where = ' WHERE ST_IsValid(ST_SimplifyPreserveTopology(geometry, %f))' % (self.mapping.tolerance,) if self.mapping.where: - where = ' WHERE ' + self.mapping.where - else: - where = '' - + where = ' AND '.join([where, self.mapping.where]) + if config.imposm_pg_serial_id: serial_column = "id, " else: serial_column = "" - + return """CREATE TABLE "%s" AS (SELECT %s osm_id, %s - ST_Simplify(geometry, %f) as geometry from "%s"%s)""" % ( + ST_SimplifyPreserveTopology(geometry, %f) as geometry from "%s"%s)""" % ( self.table_name, serial_column, fields, self.mapping.tolerance, self.db.to_tablename(self.mapping.origin.name), where) @@ -405,20 +495,23 @@ class PostGISGeneralizedTable(object): def create(self): cur = self.db.connection.cursor() cur.execute('BEGIN') - try: - cur.execute('SAVEPOINT pre_drop_table') - cur.execute('DROP TABLE "%s" CASCADE' % (self.table_name, )) - except psycopg2.ProgrammingError: - cur.execute('ROLLBACK TO SAVEPOINT pre_drop_table') - + + self.db.drop_table_or_view(cur, self.table_name) + cur.execute(self._stmt()) - cur.execute(self._idx_stmt()) - cur.execute('SELECT * FROM geometry_columns WHERE f_table_name = %s', (self.table_name, )) - if cur.fetchall(): - # drop old entry to handle changes of SRID - cur.execute('DELETE FROM geometry_columns WHERE f_table_name = %s', (self.table_name, )) - cur.execute(self._geom_table_stmt()) + if self.db.use_geometry_columns_table: + cur.execute('SELECT * FROM geometry_columns WHERE f_table_name = %s', (self.table_name, )) + if cur.fetchall(): + # drop old entry to handle changes of SRID + cur.execute('DELETE FROM geometry_columns WHERE f_table_name = %s', (self.table_name, )) + cur.execute(self._geom_table_stmt()) class TrigramIndex(object): pass + +class StringIndex(object): + pass + +class Index(object): + pass diff --git a/imposm/dbimporter.py b/imposm/dbimporter.py index 652c5c2..3f00732 100644 --- a/imposm/dbimporter.py +++ b/imposm/dbimporter.py @@ -1,11 +1,11 @@ -# Copyright 2011 Omniscale (http://omniscale.com) -# +# Copyright 2011, 2012 Omniscale (http://omniscale.com) +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,7 +16,7 @@ from collections import defaultdict from multiprocessing import Process import threading -from Queue import Queue +from Queue import Queue from imposm.base import OSMElem from imposm.geom import IncompletePolygonError @@ -41,6 +41,7 @@ class ImporterProcess(Process): self.osm_cache = osm_cache self.db = db self.dry_run = dry_run + self.db_queue = Queue(256) def run(self): self.setup() @@ -49,8 +50,9 @@ class ImporterProcess(Process): self.teardown() def setup(self): - self.db_queue = Queue(256) - self.db_importer = DBImporter(self.db_queue, self.db, dry_run=self.dry_run) + self.db_importer = threading.Thread(target=self.db_importer, + args=(self.db_queue, self.db), + kwargs=dict(dry_run=self.dry_run)) self.db_importer.start() def doit(self): @@ -61,6 +63,35 @@ class ImporterProcess(Process): self.db_queue.put(None) self.db_importer.join() +class TupleBasedImporter(ImporterProcess): + def db_importer(self, queue, db, dry_run=False): + db.reconnect() + mappings = defaultdict(list) + + while True: + data = queue.get() + if data is None: + break + + mapping, osm_id, osm_elem, extra_args = data + insert_data = mappings[mapping] + + if isinstance(osm_elem.geom, (list)): + for geom in osm_elem.geom: + insert_data.append((osm_id, db.geom_wrapper(geom)) + tuple(extra_args)) + else: + insert_data.append((osm_id, db.geom_wrapper(osm_elem.geom)) + tuple(extra_args)) + + if len(insert_data) >= 128: + if not dry_run: + db.insert(mapping, insert_data) + del mappings[mapping] + + # flush + for mapping, insert_data in mappings.iteritems(): + if not dry_run: + db.insert(mapping, insert_data) + def insert(self, mappings, osm_id, geom, tags): inserted = False for type, ms in mappings: @@ -76,15 +107,81 @@ class ImporterProcess(Process): pass return inserted +class DictBasedImporter(ImporterProcess): + def db_importer(self, queue, db, dry_run=False): + db.reconnect() + insert_data = [] + + while True: + data = queue.get() + if data is None: + break + + data['geometry'] = db.geom_wrapper(data['geometry']) + insert_data.append(data) + + if len(insert_data) >= 128: + if not dry_run: + db.insert(insert_data) + insert_data = [] + # flush + if not dry_run: + db.insert(insert_data) + + + def insert(self, mappings, osm_id, geom, tags): + inserted = False + osm_objects = {} + + for type, ms in mappings: + for m in ms: + osm_elem = OSMElem(osm_id, geom, type, tags) + try: + m.filter(osm_elem) + except DropElem: + continue + + if m.geom_type in osm_objects: + obj = osm_objects[m.geom_type] + obj['fields'].update(m.field_dict(osm_elem)) + obj['fields'][type[0]] = type[1] + obj['mapping_names'].append(m.name) + else: + try: + m.build_geom(osm_elem) + except DropElem: + continue + obj = {} + obj['fields'] = m.field_dict(osm_elem) + obj['fields'][type[0]] = type[1] + obj['osm_id'] = osm_id + obj['geometry'] = osm_elem.geom + obj['mapping_names'] = [m.name] + osm_objects[m.geom_type] = obj + + inserted = True + + for obj in osm_objects.itervalues(): + if isinstance(obj['geometry'], (list, )): + for geom in obj['geometry']: + obj_part = obj.copy() + obj_part['geometry'] = geom + self.db_queue.put(obj_part) + else: + self.db_queue.put(obj) + + return inserted + + class NodeProcess(ImporterProcess): name = 'node' - def doit(self): + def doit(self): while True: nodes = self.in_queue.get() if nodes is None: break - + for node in nodes: mappings = self.mapper.for_nodes(node.tags) if not mappings: @@ -92,6 +189,12 @@ class NodeProcess(ImporterProcess): self.insert(mappings, node.osm_id, node.coord, node.tags) +class NodeProcessDict(NodeProcess, DictBasedImporter): + pass + +class NodeProcessTuple(NodeProcess, TupleBasedImporter): + pass + class WayProcess(ImporterProcess): name = 'way' @@ -118,7 +221,7 @@ class WayProcess(ImporterProcess): skip_id = inserted_ways.next() except StopIteration: skip_id = 2**64 - + if skip_id == way.osm_id: continue @@ -129,17 +232,22 @@ class WayProcess(ImporterProcess): coords = coords_cache.get_coords(way.refs) if not coords: - print 'missing coords for way %s' % (way.osm_id, ) + log.debug('missing coords for way %s', way.osm_id) continue self.insert(mappings, way.osm_id, coords, way.tags) +class WayProcessDict(WayProcess, DictBasedImporter): + pass + +class WayProcessTuple(WayProcess, TupleBasedImporter): + pass class RelationProcess(ImporterProcess): name = 'relation' def __init__(self, in_queue, db, mapper, osm_cache, dry_run, inserted_way_queue): - ImporterProcess.__init__(self, in_queue, db, mapper, osm_cache, dry_run) + super(RelationProcess, self).__init__(in_queue, db, mapper, osm_cache, dry_run) self.inserted_way_queue = inserted_way_queue def doit(self): @@ -150,7 +258,7 @@ class RelationProcess(ImporterProcess): relations = self.in_queue.get() if relations is None: break - + for relation in relations: builder = RelationBuilder(relation, ways_cache, coords_cache) try: @@ -165,41 +273,8 @@ class RelationProcess(ImporterProcess): if inserted: builder.mark_inserted_ways(self.inserted_way_queue) +class RelationProcessDict(RelationProcess, DictBasedImporter): + pass -class DBImporter(threading.Thread): - def __init__(self, queue, db, dry_run=False): - threading.Thread.__init__(self) - self.db = db - self.db.reconnect() - self.queue = queue - self.cur = None - self.dry_run = dry_run - self.mappings = defaultdict(list) - - def run(self): - # cProfile.runctx('self.doit()', globals(), locals()) - # def doit(self): - while True: - data = self.queue.get() - if data is None: - break - - mapping, osm_id, osm_elem, extra_args = data - insert_data = self.mappings[mapping] - - if isinstance(osm_elem.geom, (list)): - for geom in osm_elem.geom: - insert_data.append((osm_id, self.db.geom_wrapper(geom)) + tuple(extra_args)) - else: - insert_data.append((osm_id, self.db.geom_wrapper(osm_elem.geom)) + tuple(extra_args)) - - if len(insert_data) >= 128: - if not self.dry_run: - self.db.insert(mapping, insert_data) - del self.mappings[mapping] - - # flush - for mapping, insert_data in self.mappings.iteritems(): - if not self.dry_run: - self.db.insert(mapping, insert_data) - +class RelationProcessTuple(RelationProcess, TupleBasedImporter): + pass diff --git a/imposm/geom.py b/imposm/geom.py index aa374ff..42a5715 100644 --- a/imposm/geom.py +++ b/imposm/geom.py @@ -1,11 +1,11 @@ -# Copyright 2011 Omniscale (http://omniscale.com) -# +# Copyright 2011, 2012 Omniscale (http://omniscale.com) +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,13 +15,24 @@ from __future__ import division import math - +import codecs +import os +import shapely.geometry import shapely.geos +import shapely.prepared from shapely.geometry.base import BaseGeometry from shapely import geometry from shapely import wkt +from shapely.ops import cascaded_union, linemerge +from shapely.topology import TopologicalError + +try: + import rtree +except ImportError: + rtree = None from imposm import config +from imposm.util.geom import load_polygons, load_datasource, build_multipolygon import logging log = logging.getLogger(__name__) @@ -42,12 +53,12 @@ SHAPELY_SUPPORTS_BUFFER = shapely.geos.geos_capi_version >= (1, 6, 0) def validate_and_simplify(geom, meter_units=False): if SHAPELY_SUPPORTS_BUFFER: try: - # buffer(0) is nearly fast as is_valid + # buffer(0) is nearly fast as is_valid return geom.buffer(0) except ValueError: # shapely raises ValueError if buffer(0) result is empty raise InvalidGeometryError('geometry is empty') - + orig_geom = geom if not geom.is_valid: tolerance = TOLERANCE_METERS if meter_units else TOLERANCE_DEEGREES @@ -77,8 +88,8 @@ class GeomBuilder(object): if geom_wkt is None or geom is None: # unable to build valid wkt (non closed polygon, etc) raise InvalidGeometryError() - return geom - + return geom + def check_geom_type(self, geom): return @@ -96,23 +107,23 @@ class GeomBuilder(object): if geom is None: # unable to build valid wkt (non closed polygon, etc) raise InvalidGeometryError() - return geom + return geom class PointBuilder(GeomBuilder): def to_wkt(self, data): if len(data) != 2: return None return 'POINT(%f %f)' % data - + def to_geom(self, data): if len(data) != 2: return None return geometry.Point(*data) - + def check_geom_type(self, geom): if geom.type != 'Point': raise InvalidGeometryError('expected Point, got %s' % geom.type) - + def build_checked_geom(self, osm_elem, validate=False): geom = self.build_geom(osm_elem) if not validate or geom.is_valid: @@ -126,16 +137,16 @@ class PolygonBuilder(GeomBuilder): if len(data) >= 4 and data[0] == data[-1]: return 'POLYGON((' + ', '.join('%f %f' % p for p in data) + '))' return None - + def to_geom(self, data): if len(data) >= 4 and data[0] == data[-1]: return geometry.Polygon(data) return None - + def check_geom_type(self, geom): if geom.type not in ('Polygon', 'MultiPolygon'): raise InvalidGeometryError('expected Polygon or MultiPolygon, got %s' % geom.type) - + def build_checked_geom(self, osm_elem, validate=False): geom = self.build_geom(osm_elem) if not validate: @@ -181,3 +192,232 @@ class LineStringBuilder(GeomBuilder): else: raise InvalidGeometryError('invalid geometry for %s: %s, %s' % (osm_elem.osm_id, geom, osm_elem.coords)) + +def tile_bbox(bbox, grid_width): + """ + Tile bbox into multiple sub-boxes, each of `grid_width` size. + + >>> list(tile_bbox((-1, 1, 0.49, 1.51), 0.5)) #doctest: +NORMALIZE_WHITESPACE + [(-1.0, 1.0, -0.5, 1.5), + (-1.0, 1.5, -0.5, 2.0), + (-0.5, 1.0, 0.0, 1.5), + (-0.5, 1.5, 0.0, 2.0), + (0.0, 1.0, 0.5, 1.5), + (0.0, 1.5, 0.5, 2.0)] + """ + min_x = math.floor(bbox[0]/grid_width) * grid_width + min_y = math.floor(bbox[1]/grid_width) * grid_width + max_x = math.ceil(bbox[2]/grid_width) * grid_width + max_y = math.ceil(bbox[3]/grid_width) * grid_width + + x_steps = (max_x - min_x) / grid_width + y_steps = (max_y - min_y) / grid_width + + for x in xrange(int(x_steps)): + for y in xrange(int(y_steps)): + yield ( + min_x + x * grid_width, + min_y + y * grid_width, + min_x + (x + 1) * grid_width, + min_y + (y + 1)* grid_width, + ) + +def split_polygon_at_grid(geom, grid_width=0.1): + """ + >>> p = list(split_polygon_at_grid(geometry.box(-0.5, 1, 0.2, 2), 1)) + >>> p[0].contains(geometry.box(-0.5, 1, 0, 2)) + True + >>> p[0].area == geometry.box(-0.5, 1, 0, 2).area + True + >>> p[1].contains(geometry.box(0, 1, 0.2, 2)) + True + >>> p[1].area == geometry.box(0, 1, 0.2, 2).area + True + """ + if not geom.is_valid: + geom = geom.buffer(0) + for split_box in tile_bbox(geom.bounds, grid_width): + try: + polygon_part = geom.intersection(shapely.geometry.box(*split_box)) + except TopologicalError: + continue + if not polygon_part.is_empty: + yield polygon_part + +def load_geom(source): + geom = None + # source is a wkt file + if os.path.exists(os.path.abspath(source)): + data = None + with open(os.path.abspath(source), 'r') as fp: + data = fp.read(50) + # load WKT geometry and remove leading whitespaces + if data.lower().lstrip().startswith(('polygon', 'multipolygon')): + geom = load_polygons(source) + # source is an OGR datasource + if geom is None: + geom = load_datasource(source) + + if geom: + # get the first and maybe only geometry + if not check_wgs84_srs(geom[0]): + log.error('Geometry is not in EPSG:4326') + return None + if rtree: + return LimitRTreeGeometry(geom) + else: + log.info('You should install RTree for large --limit-to polygons') + return LimitPolygonGeometry(build_multipolygon(geom)[1]) + return None + +def check_wgs84_srs(geom): + bbox = geom.bounds + if bbox[0] >= -180 and bbox[1] >= -90 and bbox[2] <= 180 and bbox[3] <= 90: + return True + return False + +class EmtpyGeometryError(Exception): + pass + +class LimitPolygonGeometry(object): + def __init__(self, shapely_geom): + self._geom = shapely_geom + self._prepared_geom = None + self._prepared_counter = 0 + self._prepared_max = 100000 + + @property + def geom(self): + # GEOS internal data structure for prepared geometries grows over time, + # recreate to limit memory consumption + if not self._prepared_geom or self._prepared_counter > self._prepared_max: + print 'create prepared' + self._prepared_geom = shapely.prepared.prep(self._geom) + self._prepared_counter = 0 + self._prepared_counter += 1 + return self._prepared_geom + + def intersection(self, geom): + if self.geom.contains_properly(geom): + # no need to limit contained geometries + return geom + + new_geom = None + if self.geom.intersects(geom): + try: + # can not use intersection with prepared geom + new_geom = self._geom.intersection(geom) + except TopologicalError: + pass + + if not new_geom or new_geom.is_empty: + raise EmtpyGeometryError('No intersection or empty geometry') + + new_geom = filter_geometry_by_type(new_geom, geom.type) + if new_geom: + return new_geom + + raise EmtpyGeometryError('No intersection or empty geometry') + +def filter_geometry_by_type(geometry, geom_type): + """ + Filter (multi)geometry for compatible `geom_type`, + because we can't insert points into linestring tables for example + """ + if geometry.type == geom_type: + # same type is fine + return geometry + + if geometry.type == 'MultiPolygon' and geom_type == 'Polygon': + # polygon mappings should also support multipolygons + return geometry + + if hasattr(geometry, 'geoms'): + # GeometryCollection or MultiLineString? return list of geometries + geoms = [] + for part in geometry.geoms: + # only parts with same type + if part.type == geom_type: + geoms.append(part) + + if geoms: + return geoms + + return None + +def flatten_polygons(polygons): + for polygon in polygons: + if polygon.type == 'MultiPolygon': + for p in polygon.geoms: + yield p + else: + yield polygon + +def flatten_linestrings(linestrings): + for linestring in linestrings: + if linestring.type == 'MultiLineString': + for ls in linestring.geoms: + yield ls + else: + yield linestring + +class LimitRTreeGeometry(object): + def __init__(self, polygons): + index = rtree.index.Index() + sub_polygons = [] + part_idx = 0 + for polygon in polygons: + for part in split_polygon_at_grid(polygon): + sub_polygons.append(part) + index.insert(part_idx, part.bounds) + part_idx += 1 + + self.polygons = sub_polygons + self.index = index + + def intersection(self, geom): + intersection_ids = list(self.index.intersection(geom.bounds)) + + if not intersection_ids: + raise EmtpyGeometryError('No intersection or empty geometry') + + intersections = [] + for i in intersection_ids: + + polygon = self.polygons[i] + + if polygon.contains(geom): + return geom + + if polygon.intersects(geom): + try: + new_geom_part = polygon.intersection(geom) + new_geom_part = filter_geometry_by_type(new_geom_part, geom.type) + if new_geom_part: + if len(intersection_ids) == 1: + return new_geom_part + + if isinstance(new_geom_part, list): + intersections.extend(new_geom_part) + else: + intersections.append(new_geom_part) + except TopologicalError: + pass + + if not intersections: + raise EmtpyGeometryError('No intersection or empty geometry') + + # intersections from multiple sub-polygons + # try to merge them back to a single geometry + if geom.type.endswith('Polygon'): + union = cascaded_union(list(flatten_polygons(intersections))) + elif geom.type.endswith('LineString'): + union = linemerge(list(flatten_linestrings(intersections))) + if union.type == 'MultiLineString': + union = list(union.geoms) + elif geom.type == 'Point': + union = intersections[0] + else: + log.warn('unexpexted geometry type %s', geom.type) + raise EmtpyGeometryError() + return union diff --git a/imposm/mapping.py b/imposm/mapping.py index e07732e..7ef7c87 100644 --- a/imposm/mapping.py +++ b/imposm/mapping.py @@ -1,12 +1,12 @@ # -:- encoding: UTF8 -:- # Copyright 2011 Omniscale (http://omniscale.com) -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -45,11 +45,11 @@ default_name_field = None def set_default_name_type(type, column_name='name'): """ Set new default type for 'name' field. - + :: - + set_default_name_type(LocalizedName(['name:en', 'int_name', 'name'])) - + """ global default_name_field default_name_field = column_name, type @@ -90,11 +90,11 @@ class Mapping(object): self.name = name self.mapping = mapping self.fields = fields or tuple(self.fields) + self.limit_to_polygon = None if with_type_field is not None: # allow subclass to define other default by setting it as class variable self.with_type_field = with_type_field - if self.with_type_field: - self._add_type_field() + self._add_type_field() self._add_name_field() if field_filter: self.field_filter = field_filter @@ -108,11 +108,18 @@ class Mapping(object): self.fields = (default_name_field,) + self.fields else: self.fields = (('name', Name()),) + self.fields - + def _add_type_field(self): """ Add type field. """ + if not self.with_type_field: + return + + for name, type_ in self.fields: + if name == 'type': + # do not add type field if already present + return self.fields = (('type', Type()), ) + self.fields @property @@ -120,12 +127,12 @@ class Mapping(object): if not self._insert_stmt: self._insert_stmt = self.table('osm_' + self.name, self).insert_stmt return self._insert_stmt - + def extra_field_names(self): extra_field_names = [] for field_name, field_filter in self.field_filter: extra_field_names.append(field_name) - + for field_name, field in self.fields: field_names = field.extra_fields() if field_names is not None: @@ -133,27 +140,39 @@ class Mapping(object): else: extra_field_names.append(field_name) return extra_field_names - + def build_geom(self, osm_elem): try: geom = self.geom_builder.build_checked_geom(osm_elem) + if self.limit_to_polygon is not None: + geom = self.limit_to_polygon.intersection(geom) osm_elem.geom = geom except imposm.geom.InvalidGeometryError, ex: raise DropElem('invalid geometry: %s' % (ex, )) - + except imposm.geom.EmtpyGeometryError, ex: + raise DropElem(ex) + def field_values(self, osm_elem): return [t.value(osm_elem.tags.get(n), osm_elem) for n, t in self.fields] - + + def field_dict(self, osm_elem): + result = dict((n, t.value(osm_elem.tags.get(n), osm_elem)) for n, t in self.fields) + if self.with_type_field: + del result['type'] + result[osm_elem.cls] = osm_elem.type + return result + def filter(self, osm_elem): [t.filter(osm_elem.tags.get(n), osm_elem) for n, t in self.field_filter] - + def __repr__(self): return '<Mapping for %s>' % self.name class TagMapper(object): - def __init__(self, mappings): + def __init__(self, mappings, limit_to=None): self.mappings = mappings + self.limit_to_polygon = limit_to self._init_map() def _init_map(self): @@ -174,7 +193,7 @@ class TagMapper(object): elif mapping.table is PolygonTable: tags = self.polygon_tags add_to = self.polygon_mappings - + for extra in mapping.extra_field_names(): tags.setdefault(extra, set()).add('__any__') @@ -184,11 +203,14 @@ class TagMapper(object): tags.setdefault(tag, set()).add(type) add_to[tag].setdefault(type, []).append(mapping) + # add limit_to polygon to each mapping + mapping.limit_to_polygon = self.limit_to_polygon + def for_nodes(self, tags): return self._mapping_for_tags(self.point_mappings, tags) def for_ways(self, tags): - return (self._mapping_for_tags(self.line_mappings, tags) + + return (self._mapping_for_tags(self.line_mappings, tags) + self._mapping_for_tags(self.polygon_mappings, tags)) def for_relations(self, tags): @@ -218,7 +240,7 @@ class TagMapper(object): tags = dict() for k, v in self.line_tags.iteritems(): tags.setdefault(k, set()).update(v) - + for k, v in self.polygon_tags.iteritems(): tags.setdefault(k, set()).update(v) return self._tag_filter(tags) @@ -272,7 +294,7 @@ class TagMapper(object): new_mappings.append(proc) if new_mappings: result.append(((tag_name, tag_value), tuple(new_mappings))) - + return result @@ -287,7 +309,7 @@ class PolygonTable(object): class Points(Mapping): """ Table class for point features. - + :PostGIS datatype: POINT """ table = PointTable @@ -297,7 +319,7 @@ class Points(Mapping): class LineStrings(Mapping): """ Table class for line string features. - + :PostGIS datatype: LINESTRING """ table = LineStringTable @@ -307,7 +329,7 @@ class LineStrings(Mapping): class Polygons(Mapping): """ Table class for polygon features. - + :PostGIS datatype: GEOMETRY (POLYGON does not support multi-polygons) """ table = PolygonTable @@ -350,7 +372,7 @@ class UnionView(object): if 'type' not in self.fields and any(m.with_type_field for m in self.mappings): self.fields += (('type', None), ) - + class DropElem(Exception): pass @@ -360,12 +382,12 @@ class FieldType(object): """ List with names of fields (keys) that should be processed during read-phase. - + Return ``None`` to use the field name from the mapping. Return ``[]`` if no extra fields (keys) are required. """ return None - + def value(self, val, osm_elem): return val @@ -393,7 +415,7 @@ class Type(FieldType): ('road_class', Type(),), ), ) - + :PostgreSQL datatype: VARCHAR(255) .. versionadded:: 2.4.0 @@ -411,11 +433,11 @@ class Class(FieldType): """ Field for class values (i.e. the *key* of the mapped key/value). - Use this if you want to store the key that was used for + Use this if you want to store the key that was used for this mapping. For example, the following mapping will create a column ``class`` that will have the value ``landuse`` or ``natural``, depending on the feature. - + :: landusages = Polygons( @@ -434,7 +456,7 @@ class Class(FieldType): ) :PostgreSQL datatype: VARCHAR(255) - + .. versionadded:: 2.4.0 """ @@ -449,7 +471,7 @@ class Class(FieldType): class String(FieldType): """ Field for string values. - + :PostgreSQL datatype: VARCHAR(255) """ column_type = "VARCHAR(255)" @@ -457,14 +479,14 @@ class String(FieldType): class Name(String): """ Field for name values. - + Filters out common FixMe values. - + :PostgreSQL datatype: VARCHAR(255) .. versionadded:: 2.3.0 """ - + filter_out_names = set(( 'fixme', 'fix me', 'fix-me!', '0', 'none', 'n/a', 's/n', @@ -486,18 +508,18 @@ class LocalizedName(Name): Field for localized name values. Checks different name keys and uses the first key with a valid value. - + :param coalesce: list of name keys to check :PostgreSQL datatype: VARCHAR(255) - + .. versionadded:: 2.3.0 """ def __init__(self, coalesce=['name', 'int_name']): self.coalesce_keys = coalesce - + def extra_fields(self): return self.coalesce_keys - + def value(self, val, osm_elem): for key in self.coalesce_keys: val = osm_elem.tags.get(key) @@ -512,7 +534,7 @@ class Bool(FieldType): """ Field for boolean values. Converts false, no, 0 to False and true, yes, 1 to True. - + :PostgreSQL datatype: SMALLINT """ # there was a reason this is not BOOL @@ -543,7 +565,7 @@ class Direction(FieldType): Converts `yes`, `true` and `1` to ``1`` for one ways in the direction of the way, `-1` to ``-1`` for one ways against the direction of the way and ``0`` for all other values. - + :PostgreSQL datatype: SMALLINT """ column_type = "SMALLINT" @@ -559,36 +581,36 @@ class Direction(FieldType): class PseudoArea(FieldType): """ Field for the (pseudo) area of a polygon in square meters. - + The value is just an approximation since the geometries are in EPSG:4326 and not in a equal-area projection. The approximation is good for smaller polygons (<1%) and should be precise enough to compare geometries for rendering order (larger below smaller). - + The area of the geometry is multiplied by the cosine of the mid-latitude to compensate the reduced size towards the poles. - + :PostgreSQL datatype: REAL - + .. versionadded:: 2.3.0 """ - + column_type = "REAL" - + def value(self, val, osm_elem): area = osm_elem.geom.area if not area: return None - + extent = osm_elem.geom.bounds mid_lat = extent[1] + (abs(extent[3] - extent[1]) / 2) sqr_deg = area * math.cos(math.radians(mid_lat)) - + # convert deg^2 to m^2 sqr_m = (math.sqrt(sqr_deg) * (40075160 / 360))**2 return sqr_m - + def extra_fields(self): return [] @@ -597,7 +619,7 @@ class OneOfInt(FieldType): Field type for integer values. Converts values to integers, drops element if is not included in ``values``. - + :PostgreSQL datatype: SMALLINT """ column_type = "SMALLINT" @@ -614,7 +636,7 @@ class Integer(FieldType): """ Field type for integer values. Converts values to integers, defaults to ``NULL``. - + :PostgreSQL datatype: INTEGER """ column_type = "INTEGER" @@ -628,12 +650,12 @@ class Integer(FieldType): class ZOrder(FieldType): """ Field type for z-ordering based on the feature type. - + :param types: list of mapped feature types, from highest to lowest ranking :PostgreSQL datatype: SMALLINT """ - + column_type = "SMALLINT" def __init__(self, types): @@ -651,12 +673,12 @@ class ZOrder(FieldType): class WayZOrder(FieldType): """ Field type for z-ordered based on highway types. - + Ordering based on the osm2pgsql z-ordering: From ``roads`` = 3 to ``motorways`` = 9, ``railway`` = 7 and unknown = 0. Ordering changes with ``tunnels`` by -10, ``bridges`` by +10 and ``layer`` by 10 * ``layer``. - + :PostgreSQL datatype: SMALLINT """ @@ -680,7 +702,7 @@ class WayZOrder(FieldType): } brunnel_bool = Bool() - + def extra_fields(self): return [] diff --git a/imposm/psqldb.py b/imposm/psqldb.py index 6d5846d..fdfff42 100644 --- a/imposm/psqldb.py +++ b/imposm/psqldb.py @@ -1,11 +1,11 @@ # Copyright 2011 Omniscale (http://omniscale.com) -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,27 +18,38 @@ from os.path import join, dirname, exists db_create_template = """ # run this as postgres user, eg: -# imposm-pgsql > create_db.sh; sudo su postgres; sh ./create_db.sh +# imposm-psqldb > create_db.sh; sudo su postgres; sh ./create_db.sh set -xe createuser --no-superuser --no-createrole --createdb ${user} createdb -E UTF8 -O ${user} ${dbname} createlang plpgsql ${dbname} -psql -d ${dbname} -f ${postgis_sql} -psql -d ${dbname} -f ${spatial_ref_sys_sql} -psql -d ${dbname} -f ${epsg900913_sql} -echo "ALTER TABLE geometry_columns OWNER TO ${user}; ALTER TABLE spatial_ref_sys OWNER TO ${user};" | psql -d ${dbname} +${postgis} +echo "ALTER TABLE spatial_ref_sys OWNER TO ${user};" | psql -d ${dbname} echo "ALTER USER ${user} WITH PASSWORD '${password}';" |psql -d ${dbname} echo "host\t${dbname}\t${user}\t127.0.0.1/32\tmd5" >> ${pg_hba} set +x echo "Done. Don't forget to restart postgresql!" """.strip() -def find_sql_files(version, mapping): - - pg_hba = '/path/to/pg_hba.conf \t\t# <- CHANGE THIS PATH' - postgis_sql = '/path/to/postgis.sql \t\t# <- CHANGE THIS PATH' - spatial_ref_sys_sql = '/path/to/spatial_ref_sys.sql \t# <- CHANGE THIS PATH' - +postgis_create_template_15 = """ +psql -d ${dbname} -f ${postgis_sql} +psql -d ${dbname} -f ${spatial_ref_sys_sql} +psql -d ${dbname} -f ${epsg900913_sql} +echo "ALTER TABLE geometry_columns OWNER TO ${user};" | psql -d ${dbname} +""".strip() + +postgis_create_template_20 = """ +echo "CREATE EXTENSION postgis;" | psql -d ${dbname} +echo "ALTER TABLE geometry_columns OWNER TO ${user};" | psql -d ${dbname} +psql -d ${dbname} -f ${epsg900913_sql} +""".strip() + +def find_sql_files(version, postgis_version, mapping): + + pg_hba = '/path/to/pg_hba.conf \t# <- CHANGE THIS PATH' + postgis_sql = '/path/to/postgis.sql \t\t\t\t# <- CHANGE THIS PATH' + spatial_ref_sys_sql = '/path/to/spatial_ref_sys.sql \t\t\t# <- CHANGE THIS PATH' + if version in ('8.3', 'auto'): p = '/usr/share/postgresql-8.3-postgis/lwpostgis.sql' if exists(p): @@ -49,7 +60,7 @@ def find_sql_files(version, mapping): p = '/etc/postgresql/8.3/main/pg_hba.conf' if exists(p): pg_hba = p - + if version in ('8.4', 'auto'): p = '/usr/share/postgresql/8.4/contrib/postgis.sql' if exists(p): @@ -66,7 +77,7 @@ def find_sql_files(version, mapping): p = '/etc/postgresql/8.4/main/pg_hba.conf' if exists(p): pg_hba = p - + if version in ('9.1', 'auto'): p = '/usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql' if exists(p): @@ -77,10 +88,15 @@ def find_sql_files(version, mapping): p = '/etc/postgresql/9.1/main/pg_hba.conf' if exists(p): pg_hba = p - + + if postgis_version == '2.0': + postgis_sql = None + spatial_ref_sys_sql = None + mapping['postgis_sql'] = postgis_sql mapping['spatial_ref_sys_sql'] = spatial_ref_sys_sql mapping['pg_hba'] = pg_hba + mapping['pg_version'] = postgis_version def main(): usage = '%prog [options]' @@ -89,8 +105,9 @@ def main(): parser.add_option('--database', dest='dbname', metavar='osm', default='osm') parser.add_option('--user', dest='user', metavar='osm', default='osm') parser.add_option('--password', dest='password', metavar='osm', default='osm') - parser.add_option('--pg-version', dest='pg_version', metavar='8.3|8.4|auto', default='auto') - + parser.add_option('--pg-version', dest='pg_version', metavar='8.3|8.4|9.1|auto', default='auto') + parser.add_option('--postgis-version', dest='postgis_version', metavar='1.5|2.0', default='1.5') + (options, args) = parser.parse_args() mapping = { @@ -98,10 +115,15 @@ def main(): 'dbname': options.dbname, 'password': options.password, } - + mapping['epsg900913_sql'] = join(dirname(__file__), '900913.sql') - find_sql_files(options.pg_version, mapping) - + find_sql_files(options.pg_version, options.postgis_version, mapping) + + if options.postgis_version == '2.0': + mapping['postgis'] = string.Template(postgis_create_template_20).substitute(mapping) + else: + mapping['postgis'] = string.Template(postgis_create_template_15).substitute(mapping) + template = string.Template(db_create_template) print template.substitute(mapping) diff --git a/imposm/reader.py b/imposm/reader.py index 9768571..0a06974 100644 --- a/imposm/reader.py +++ b/imposm/reader.py @@ -1,11 +1,11 @@ # Copyright 2011 Omniscale (http://omniscale.com) -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,7 +16,7 @@ from functools import partial from multiprocessing import Process, JoinableQueue from imposm.parser import OSMParser -from imposm.util import ParserProgress, setproctitle +from imposm.util import setproctitle class ImposmReader(object): def __init__(self, mapping, cache, pool_size=2, merge=False, logger=None): @@ -33,22 +33,22 @@ class ImposmReader(object): coords_queue = JoinableQueue(512) ways_queue = JoinableQueue(128) relations_queue = JoinableQueue(128) - - log_proc = ParserProgress() + + log_proc = self.logger() log_proc.start() - + marshal = True if self.merge: # merging needs access to unmarshaled data marshal = False - + estimates = { 'coords': self.estimated_coords, 'nodes': self.estimated_coords//50, 'ways': self.estimated_coords//7, 'relations': self.estimated_coords//1000, } - + coords_writer = CacheWriterProcess(coords_queue, self.cache.coords_cache, estimates['coords'], log=partial(log_proc.log, 'coords'), marshaled_data=marshal) @@ -64,27 +64,27 @@ class ImposmReader(object): estimates['ways'], merge=self.merge, log=partial(log_proc.log, 'ways'), marshaled_data=marshal) ways_writer.start() - + relations_writer = CacheWriterProcess(relations_queue, self.cache.relations_cache, estimates['relations'], merge=self.merge, log=partial(log_proc.log, 'relations'), marshaled_data=marshal) relations_writer.start() - + log_proc.message('coords: %dk nodes: %dk ways: %dk relations: %dk (estimated)' % ( estimates['coords']/1000, estimates['nodes']/1000, estimates['ways']/1000, estimates['relations']/1000) ) - + # keep one CPU free for writer proc on hosts with 4 or more CPUs pool_size = self.pool_size if self.pool_size < 4 else self.pool_size - 1 - + parser = OSMParser(pool_size, nodes_callback=nodes_queue.put, coords_callback=coords_queue.put, ways_callback=ways_queue.put, relations_callback=relations_queue.put, marshal_elem_data=marshal) parser.nodes_tag_filter = self.mapper.tag_filter_for_nodes() parser.ways_tag_filter = self.mapper.tag_filter_for_ways() parser.relations_tag_filter = self.mapper.tag_filter_for_relations() - + parser.parse(filename) coords_queue.put(None) @@ -111,7 +111,7 @@ class CacheWriterProcess(Process): self.log = log self.marshaled_data = marshaled_data self.estimated_records = estimated_records - + def run(self): # print 'creating %s (%d)' % (self.filename, self.estimated_records or 0) cache = self.cache(mode='w', estimated_records=self.estimated_records) @@ -121,7 +121,7 @@ class CacheWriterProcess(Process): cache_put = cache.put while True: data = self.queue.get() - if data is None: + if data is None: self.queue.task_done() break if self.merge: diff --git a/imposm/test/test_dbimporter.py b/imposm/test/test_dbimporter.py new file mode 100644 index 0000000..3a10cb8 --- /dev/null +++ b/imposm/test/test_dbimporter.py @@ -0,0 +1,100 @@ +# Copyright 2012 Omniscale (http://omniscale.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from imposm.dbimporter import DictBasedImporter, TupleBasedImporter +from imposm import defaultmapping + +from nose.tools import eq_, assert_almost_equal + +class TestDictBasedImporter(object): + def setup(self): + dummy_db = None + mapper = None + self.importer = DictBasedImporter(None, dummy_db, mapper, None, + dry_run=False) + + def test_insert(self): + mappings = [ + (('highway', 'secondary'), [defaultmapping.mainroads]), + (('railway', 'tram'), [defaultmapping.railways]), + (('landusage', 'grass'), [defaultmapping.landusages]), + ] + assert self.importer.insert(mappings, 1234, [(0, 0), (1, 0), (1, 1), (0, 0)], + {'highway': 'secondary', 'railway': 'tram', 'oneway': '1', + 'name': 'roundabout', + } + ) + + # get items, sort by mapping_names so that landusages comes first + queue_items = [self.importer.db_queue.get(), self.importer.db_queue.get()] + queue_items.sort(key=lambda x: x['mapping_names']) + + polygon_item = queue_items[0] + linestring_item = queue_items[1] + + eq_(linestring_item['mapping_names'], ['mainroads', 'railways']) + eq_(linestring_item['osm_id'], 1234) + eq_(linestring_item['fields'], { + 'highway': 'secondary', + 'railway': 'tram', + 'oneway': 1, + 'z_order': 7, + 'bridge': 0, + 'tunnel': 0, + 'name': 'roundabout', + 'ref': None + }) + eq_(polygon_item['mapping_names'], ['landusages']) + eq_(polygon_item['osm_id'], 1234) + assert_almost_equal(polygon_item['fields']['area'], 6195822904.182782) + del polygon_item['fields']['area'] + eq_(polygon_item['fields'], { + 'z_order': 27, + 'landusage': 'grass', + 'name': 'roundabout', + }) + +class TestTupleBasedImporter(object): + def setup(self): + dummy_db = None + mapper = None + self.importer = TupleBasedImporter(None, dummy_db, mapper, None, + dry_run=False) + + def test_insert(self): + mappings = [ + (('highway', 'secondary'), [defaultmapping.mainroads]), + (('railway', 'tram'), [defaultmapping.railways]), + (('landusage', 'grass'), [defaultmapping.landusages]), + ] + assert self.importer.insert(mappings, 1234, [(0, 0), (1, 0), (1, 1), (0, 0)], + {'highway': 'secondary', 'railway': 'tram', 'oneway': '1', + 'name': 'roundabout', + } + ) + + mainroads_item = self.importer.db_queue.get() + eq_(mainroads_item[0], defaultmapping.mainroads) + eq_(mainroads_item[1], 1234) + eq_(mainroads_item[3], ['roundabout', 'secondary', 0, 0, 1, None, 5]) + + railways_item = self.importer.db_queue.get() + eq_(railways_item[0], defaultmapping.railways) + eq_(railways_item[1], 1234) + eq_(railways_item[3], ['roundabout', 'tram', 0, 0, 7]) + + landusages_item = self.importer.db_queue.get() + eq_(landusages_item[0], defaultmapping.landusages) + eq_(landusages_item[1], 1234) + eq_(landusages_item[3], ['roundabout', 'grass', 6195822904.182782, 27]) diff --git a/imposm/test/test_geom.py b/imposm/test/test_geom.py deleted file mode 100644 index de56b56..0000000 --- a/imposm/test/test_geom.py +++ /dev/null @@ -1,33 +0,0 @@ -from imposm.geom import LineStringBuilder -from shapely.ops import linemerge -from nose.tools import eq_ - -class TestLineStringBuilder(object): - def test_split(self): - builder = LineStringBuilder() - geom = builder.to_geom([(0, 0), (1, 1), (2, 2), (3, 3)], max_length=3) - assert isinstance(geom, list) - eq_(len(geom), 2) - - merged = linemerge(geom) - eq_(merged.type, 'LineString') - - def test_split(self): - builder = LineStringBuilder() - geom = builder.to_geom([(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)], max_length=2) - assert isinstance(geom, list) - eq_(len(geom), 3) - - merged = linemerge(geom) - eq_(merged.type, 'LineString') - - def test_split_same_length(self): - builder = LineStringBuilder() - geom = builder.to_geom([(0, 0), (1, 1), (2, 2), (3, 3)], max_length=4) - eq_(geom.type, 'LineString') - - - def test_split_short(self): - builder = LineStringBuilder() - geom = builder.to_geom([(0, 0), (1, 1), (2, 2), (3, 3)], max_length=10) - eq_(geom.type, 'LineString') diff --git a/imposm/test/test_imported.py b/imposm/test/test_imported.py index 41c9948..13788cf 100644 --- a/imposm/test/test_imported.py +++ b/imposm/test/test_imported.py @@ -1,11 +1,11 @@ # Copyright 2011 Omniscale (http://omniscale.com) -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,36 +42,38 @@ def setup_module(): os.chdir(temp_dir) test_osm_file = os.path.join(os.path.dirname(__file__), 'test.out.osm') with capture_out(): - imposm.app.main(['--read', test_osm_file, '--write', '-d', db_conf.db, '--host', db_conf.host, - '--proj', db_conf.proj]) + print db_conf.password + imposm.app.main(['--read', test_osm_file, '--write', + '--proj', db_conf.proj, '--table-prefix', db_conf.prefix, + '--connect', 'postgis://%(user)s:%(password)s@%(host)s:%(port)s/%(db)s' % db_conf]) class TestImported(object): def __init__(self): self.db = imposm.db.config.DB(db_conf) - + def test_point(self): cur = self.db.cur - cur.execute('select osm_id, name, ST_AsText(geometry) from osm_new_places where osm_id = 1') + cur.execute('select osm_id, name, ST_AsText(geometry) from %splaces where osm_id = 1' % db_conf.prefix) results = cur.fetchall() eq_(len(results), 1) eq_(results[0], (1, 'Foo', 'POINT(13 47.5)')) def test_way(self): cur = self.db.cur - cur.execute('select osm_id, name, ST_AsText(geometry) from osm_new_landusages where osm_id = 1001') + cur.execute('select osm_id, name, ST_AsText(geometry) from %slandusages where osm_id = 1001' % db_conf.prefix) results = cur.fetchall() eq_(len(results), 1) eq_(results[0][:-1], (1001, 'way 1001',)) eq_(roundwkt(results[0][-1]), 'POLYGON((13.0 47.5,14.5 50.0,16.5 49.0,17.0 47.0,14.5 45.5,13.0 47.5),(14.0 47.5,15.0 47.0,15.5 48.0,14.5 48.5,14.0 47.5))') - cur.execute('select osm_id, name, ST_AsText(geometry) from osm_new_landusages where osm_id = 2001') + cur.execute('select osm_id, name, ST_AsText(geometry) from %slandusages where osm_id = 2001' % db_conf.prefix) results = cur.fetchall() eq_(len(results), 1) eq_(results[0][:-1], (2001, 'way 2001',)) eq_(roundwkt(results[0][-1]), 'POLYGON((23.0 47.5,24.5 50.0,26.5 49.0,27.0 47.0,24.5 45.5,23.0 47.5),(24.5 47.0,25.5 46.5,26.0 47.5,25.0 47.5,24.5 47.0),(24.2 48.25,25.25 47.7,25.7 48.8,24.7 49.25,24.2 48.25))') - cur.execute('select osm_id, name, ST_AsText(geometry) from osm_new_landusages where osm_id = 3001') + cur.execute('select osm_id, name, ST_AsText(geometry) from %slandusages where osm_id = 3001' % db_conf.prefix) results = cur.fetchall() eq_(len(results), 1) eq_(results[0][:-1], (3001, 'way 3002',)) @@ -86,10 +88,10 @@ def roundwkt(wkt): def teardown_module(): if old_cwd: os.chdir(old_cwd) - + if temp_dir and os.path.exists(temp_dir): shutil.rmtree(temp_dir) - + @contextmanager def capture_out(): @@ -105,4 +107,3 @@ def capture_out(): finally: sys.stdout = old_stdout sys.stderr = old_stderr - \ No newline at end of file diff --git a/imposm/test/test_limit_to.py b/imposm/test/test_limit_to.py new file mode 100644 index 0000000..e99babe --- /dev/null +++ b/imposm/test/test_limit_to.py @@ -0,0 +1,85 @@ +# Copyright 2012 Omniscale (http://omniscale.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from imposm.geom import LimitPolygonGeometry, EmtpyGeometryError +from shapely.wkt import loads + +from nose.tools import raises + +class TestLimitPolygonGeometry(object): + + @raises(EmtpyGeometryError) + def test_linestring_no_intersection(self): + geom = 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))' + limit_to = LimitPolygonGeometry(loads(geom)) + limit_to.intersection(loads('LINESTRING(-100 -100, -50 -50)')) + + def test_linestring(self): + geom = 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))' + limit_to = LimitPolygonGeometry(loads(geom)) + geom = limit_to.intersection(loads('LINESTRING(-10 -10, 20 20)')) + assert geom.almost_equals(loads('LINESTRING(0 0, 10 10)')) + + def test_linestring_contained(self): + geom = 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))' + limit_to = LimitPolygonGeometry(loads(geom)) + test_geom = loads('LINESTRING(1 1, 9 9)') + geom = limit_to.intersection(test_geom) + # should return unmodified input geometry + assert geom is test_geom + + def test_linestring_multilinestring_result(self): + geom = 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))' + limit_to = LimitPolygonGeometry(loads(geom)) + geom = limit_to.intersection(loads('LINESTRING(-10 -20, 5 10, 20 -20)')) + assert isinstance(geom, list) + assert geom[0].almost_equals(loads('LINESTRING(0 0, 5 10)')) + assert geom[1].almost_equals(loads('LINESTRING(5 10, 10 0)')) + + @raises(EmtpyGeometryError) + def test_linestring_point_result(self): + geom = 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))' + limit_to = LimitPolygonGeometry(loads(geom)) + geom = limit_to.intersection(loads('LINESTRING(-10 -10, 0 0)')) + + def test_linestring_mixed_result(self): + geom = 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))' + limit_to = LimitPolygonGeometry(loads(geom)) + geom = limit_to.intersection(loads('LINESTRING(0 0, 5 -10, 5 10)')) + # point and linestring, point not returned + assert isinstance(geom, list) + assert len(geom) == 1 + assert geom[0].almost_equals(loads('LINESTRING(5 0, 5 10)')) + + def test_polygon_mixed_result(self): + geom = 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))' + limit_to = LimitPolygonGeometry(loads(geom)) + test_geom = loads('POLYGON((0 -10, 0 5, 2.5 -5, 5 0, 7.5 -5, 10 5, 10 -10, 0 -10))') + geom = limit_to.intersection(test_geom) + # point and two polygons, point not returned + assert isinstance(geom, list) + assert len(geom) == 2 + assert geom[0].almost_equals(loads('POLYGON((1.25 0, 0 0, 0 5, 1.25 0))')) + assert geom[1].almost_equals(loads('POLYGON((10 0, 8.75 0, 10 5, 10 0))')) + + def test_polygon_multipolygon_result(self): + geom = 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))' + limit_to = LimitPolygonGeometry(loads(geom)) + test_geom = loads('POLYGON((0 -10, 0 5, 2.5 -5, 5 -1, 7.5 -5, 10 5, 10 -10, 0 -10))') + geom = limit_to.intersection(test_geom) + # similar to above, but point does not touch the box, so we should get + # a single multipolygon + assert geom.almost_equals(loads( + 'MULTIPOLYGON(((1.25 0, 0 0, 0 5, 1.25 0)),' + '((10 0, 8.75 0, 10 5, 10 0)))')) diff --git a/imposm/util.py b/imposm/util/__init__.py similarity index 95% rename from imposm/util.py rename to imposm/util/__init__.py index 968f917..b0fa0dc 100644 --- a/imposm/util.py +++ b/imposm/util/__init__.py @@ -1,11 +1,11 @@ # Copyright 2011 Omniscale (http://omniscale.com) -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -41,10 +41,12 @@ class Timer(object): self.logger.message('%s took %s' % (self.title, format_total_time(seconds))) class ParserProgress(multiprocessing.Process): + log_every_seconds = 0.2 + def __init__(self): self.queue = multiprocessing.Queue() multiprocessing.Process.__init__(self) - + def run(self): last_log = time.time() counters = {'coords': 0, 'nodes':0, 'ways':0, 'relations':0} @@ -52,13 +54,13 @@ class ParserProgress(multiprocessing.Process): log_statement = self.queue.get() if log_statement is None: break - + log_type, incr = log_statement counters[log_type] += incr - if time.time() - last_log > 0.2: + if time.time() - last_log > self.log_every_seconds: last_log = time.time() self.print_log(counters) - + @staticmethod def message(msg): print >>sys.stderr, "[%s] %s" % (timestamp(), msg) @@ -73,16 +75,21 @@ class ParserProgress(multiprocessing.Process): int(counters['relations']/1000) ), sys.stderr.flush() - + def log(self, log_type, incr): self.queue.put((log_type, incr)) - + def stop(self): sys.stderr.write('\n') sys.stderr.flush() self.queue.put(None) +class QuietParserProgress(ParserProgress): + log_every_seconds = 60 + class ProgressLog(object): + log_every_seconds = 0.2 + def __init__(self, title=None, total=None): self.count = 0 self.total = total @@ -91,12 +98,12 @@ class ProgressLog(object): self.start_time = time.time() self.last_log = time.time() self.print_log() - + @staticmethod def message(msg): print >>sys.stderr, "[%s] %s" % (timestamp(), msg) sys.stderr.flush() - + def log(self, value=None, step=1): before = self.count//1000 if value: @@ -105,16 +112,16 @@ class ProgressLog(object): self.count += step if self.count//1000 > before: self.print_log() - + def print_log(self): - if time.time() - self.last_log > 0.2: + if time.time() - self.last_log > self.log_every_seconds: self.last_log = time.time() print >>sys.stderr, "[%s] %s: %dk%s\r" % ( timestamp(), self.title, int(self.count/1000), self._total, ), sys.stderr.flush() - + def stop(self): print >>sys.stderr seconds = time.time() - self.start_time @@ -123,9 +130,12 @@ class ProgressLog(object): timestamp(), self.title, total_time, self.count, self.count/seconds) sys.stderr.flush() +class QuietProgressLog(ProgressLog): + log_every_seconds = 60 + def timestamp(): return datetime.datetime.now().strftime('%H:%M:%S') - + def format_total_time(seconds): h, m, s = seconds_to_hms(seconds) res = '%-02ds' % s @@ -144,11 +154,11 @@ class NullLog(object): def log_node(self): pass node = log_node - + def log_way(self): pass way = log_way - + def log_relation(self): pass relation = log_relation @@ -159,14 +169,14 @@ class MMapReader(object): self.m = m self.m.seek(0) self.size = size - + def read(self, size=None): if size is None: size = self.size - self.m.tell() else: size = min(self.size - self.m.tell(), size) return self.m.read(size) - + def readline(self): cur_pos = self.m.tell() if cur_pos >= self.size: @@ -174,7 +184,7 @@ class MMapReader(object): nl_pos = self.m.find('\n') self.m.seek(cur_pos) return self.m.read(nl_pos-cur_pos) - + def seek(self, n): self.m.seek(n) @@ -185,7 +195,7 @@ class MMapPool(object): self.pool = [mmap.mmap(-1, mmap_size) for _ in range(n)] self.free_mmaps = set(range(n)) self.free_queue = JoinableQueue() - + def new(self): if not self.free_mmaps: self.free_mmaps.add(self.free_queue.get()) @@ -198,7 +208,7 @@ class MMapPool(object): break mmap_idx = self.free_mmaps.pop() return mmap_idx, self.pool[mmap_idx] - + def join(self): while len(self.free_mmaps) < self.n: self.free_mmaps.add(self.free_queue.get()) @@ -238,6 +248,5 @@ def estimate_records(files): if f.endswith('.pbf'): fsize *= 15 # observed pbf compression factor on osm data records += fsize/200 - + return int(records) - \ No newline at end of file diff --git a/imposm/util/geom.py b/imposm/util/geom.py new file mode 100644 index 0000000..494bf29 --- /dev/null +++ b/imposm/util/geom.py @@ -0,0 +1,144 @@ +# This file is part of the MapProxy project. +# Copyright (C) 2010 Omniscale <http://omniscale.de> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division, with_statement + +import codecs +from functools import partial + +import logging +log = logging.getLogger(__name__) + +try: + import shapely.wkt + import shapely.geometry + import shapely.ops + import shapely.prepared + geom_support = True +except ImportError: + geom_support = False + +def require_geom_support(): + if not geom_support: + raise ImportError('Shapely required for geometry support') + + +def load_datasource(datasource, where=None): + """ + Loads polygons from any OGR datasource. + + Returns the bbox and a Shapely MultiPolygon with + the loaded geometries. + """ + from imposm.util.ogr import OGRShapeReader + + polygons = [] + for wkt in OGRShapeReader(datasource).wkts(where): + geom = shapely.wkt.loads(wkt) + if geom.type == 'Polygon': + polygons.append(geom) + elif geom.type == 'MultiPolygon': + for p in geom: + polygons.append(p) + else: + log.info('skipping %s geometry from %s: not a Polygon/MultiPolygon', + geom.type, datasource) + return polygons + +def load_polygons(geom_files): + """ + Loads WKT polygons from one or more text files. + + Returns the bbox and a Shapely MultiPolygon with + the loaded geometries. + """ + polygons = [] + if isinstance(geom_files, basestring): + geom_files = [geom_files] + + for geom_file in geom_files: + # open with utf-8-sig encoding to get rid of UTF8 BOM from MS Notepad + with codecs.open(geom_file, encoding='utf-8-sig') as f: + polygons.extend(load_polygon_lines(f, source=geom_files)) + + return polygons + +def load_polygon_lines(line_iter, source='<string>'): + polygons = [] + for line in line_iter: + if not line.strip(): + continue + geom = shapely.wkt.loads(line) + if geom.type == 'Polygon': + polygons.append(geom) + elif geom.type == 'MultiPolygon': + for p in geom: + polygons.append(p) + else: + log.info('ignoring non-polygon geometry (%s) from %s', + geom.type, source) + + return polygons + +def build_multipolygon(polygons, simplify=False): + if polygons: + mp = shapely.geometry.MultiPolygon(polygons) + if simplify: + mp = simplify_geom(mp) + else: + mp = shapely.geometry.Polygon() + return mp.bounds, mp + +def simplify_geom(geom): + bounds = geom.bounds + w, h = bounds[2] - bounds[0], bounds[3] - bounds[1] + tolerance = min((w/1e6, h/1e6)) + return geom.simplify(tolerance, preserve_topology=True) + +def bbox_polygon(bbox): + """ + Create Polygon that covers the given bbox. + """ + return shapely.geometry.Polygon(( + (bbox[0], bbox[1]), + (bbox[2], bbox[1]), + (bbox[2], bbox[3]), + (bbox[0], bbox[3]), + )) + +def transform_geometry(from_srs, to_srs, geometry): + transf = partial(transform_xy, from_srs, to_srs) + + if geometry.type == 'Polygon': + return transform_polygon(transf, geometry) + + if geometry.type == 'MultiPolygon': + return transform_multipolygon(transf, geometry) + + raise ValueError('cannot transform %s' % geometry.type) + +def transform_polygon(transf, polygon): + ext = transf(polygon.exterior.xy) + ints = [transf(ring.xy) for ring in polygon.interiors] + return shapely.geometry.Polygon(ext, ints) + +def transform_multipolygon(transf, multipolygon): + transformed_polygons = [] + for polygon in multipolygon: + transformed_polygons.append(transform_polygon(transf, polygon)) + return shapely.geometry.MultiPolygon(transformed_polygons) + +def transform_xy(from_srs, to_srs, xy): + return list(from_srs.transform_to(to_srs, zip(*xy))) diff --git a/imposm/util/lib.py b/imposm/util/lib.py new file mode 100644 index 0000000..9458e53 --- /dev/null +++ b/imposm/util/lib.py @@ -0,0 +1,107 @@ +# This file is part of the MapProxy project. +# Copyright (C) 2010 Omniscale <http://omniscale.de> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +ctypes utilities. +""" + +import sys +import os + +from ctypes import CDLL +from ctypes.util import find_library as _find_library + + +default_locations = dict( + darwin=dict( + paths = ['/opt/local/lib'], + exts = ['.dylib'], + ), + win32=dict( + paths = [os.path.dirname(os.__file__) + '/../../../DLLs'], + exts = ['.dll'] + ), + other=dict( + paths = [], # MAPPROXY_LIB_PATH will add paths here + exts = ['.so'] + ), +) + +# TODO does this work for imposm too? + +additional_lib_path = os.environ.get('MAPPROXY_LIB_PATH') +if additional_lib_path: + additional_lib_path = additional_lib_path.split(os.pathsep) + additional_lib_path.reverse() + for locs in default_locations.values(): + for path in additional_lib_path: + locs['paths'].insert(0, path) + +def load_library(lib_names, locations_conf=default_locations): + """ + Load the `lib_name` library with ctypes. + If ctypes.util.find_library does not find the library, + different path and filename extensions will be tried. + + Retruns the loaded library or None. + """ + if isinstance(lib_names, basestring): + lib_names = [lib_names] + + for lib_name in lib_names: + lib = load_library_(lib_name, locations_conf) + if lib is not None: return lib + +def load_library_(lib_name, locations_conf=default_locations): + lib_path = find_library(lib_name) + + if lib_path: + return CDLL(lib_path) + + if sys.platform in locations_conf: + paths = locations_conf[sys.platform]['paths'] + exts = locations_conf[sys.platform]['exts'] + lib_path = find_library(lib_name, paths, exts) + else: + paths = locations_conf['other']['paths'] + exts = locations_conf['other']['exts'] + lib_path = find_library(lib_name, paths, exts) + + if lib_path: + return CDLL(lib_path) + + +def find_library(lib_name, paths=None, exts=None): + """ + Search for library in all permutations of `paths` and `exts`. + If nothing is found None is returned. + """ + if not paths or not exts: + lib = _find_library(lib_name) + if lib is None and lib_name.startswith('lib'): + lib = _find_library(lib_name[3:]) + return lib + + for lib_name in [lib_name] + ([lib_name[3:]] if lib_name.startswith('lib') else []): + for path in paths: + for ext in exts: + lib_path = os.path.join(path, lib_name + ext) + if os.path.exists(lib_path): + return lib_path + + return None + +if __name__ == '__main__': + print load_library(sys.argv[1]) diff --git a/imposm/util/ogr.py b/imposm/util/ogr.py new file mode 100644 index 0000000..140c214 --- /dev/null +++ b/imposm/util/ogr.py @@ -0,0 +1,130 @@ +# This file is part of the MapProxy project. +# Copyright (C) 2010 Omniscale <http://omniscale.de> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from imposm.util.lib import load_library +import ctypes +from ctypes import c_void_p, c_char_p, c_int + +def init_libgdal(): + libgdal = load_library(['libgdal', 'libgdal1']) + + if not libgdal: return + + libgdal.OGROpen.argtypes = [c_char_p, c_int, c_void_p] + libgdal.OGROpen.restype = c_void_p + + libgdal.CPLGetLastErrorMsg.argtypes = [] + libgdal.CPLGetLastErrorMsg.restype = c_char_p + + libgdal.OGR_DS_GetLayer.argtypes = [c_void_p, c_int] + libgdal.OGR_DS_GetLayer.restype = c_void_p + + libgdal.OGR_FD_GetName.argtypes = [c_void_p] + libgdal.OGR_FD_GetName.restype = c_char_p + + libgdal.OGR_L_GetLayerDefn.argtypes = [c_void_p] + libgdal.OGR_L_GetLayerDefn.restype = c_void_p + + libgdal.OGR_DS_Destroy.argtypes = [c_void_p] + + libgdal.OGR_DS_ExecuteSQL.argtypes = [c_void_p, c_char_p, c_void_p, c_char_p] + libgdal.OGR_DS_ExecuteSQL.restype = c_void_p + libgdal.OGR_DS_ReleaseResultSet.argtypes = [c_void_p, c_void_p] + + libgdal.OGR_L_ResetReading.argtypes = [c_void_p] + libgdal.OGR_L_GetNextFeature.argtypes = [c_void_p] + libgdal.OGR_L_GetNextFeature.restype = c_void_p + + libgdal.OGR_F_Destroy.argtypes = [c_void_p] + + libgdal.OGR_F_GetGeometryRef.argtypes = [c_void_p] + libgdal.OGR_F_GetGeometryRef.restype = c_void_p + + libgdal.OGR_G_ExportToWkt.argtypes = [c_void_p, ctypes.POINTER(c_char_p)] + libgdal.OGR_G_ExportToWkt.restype = c_void_p + + libgdal.VSIFree.argtypes = [c_void_p] + + libgdal.OGRRegisterAll() + + return libgdal + +libgdal = init_libgdal() + +class OGRShapeReaderError(Exception): + pass + +class OGRShapeReader(object): + def __init__(self, datasource): + self.datasource = datasource + self.opened = False + self._ds = None + + def open(self): + if self.opened: return + self._ds = libgdal.OGROpen(self.datasource, False, None) + if self._ds is None: + msg = libgdal.CPLGetLastErrorMsg() + if not msg: + msg = 'failed to open %s' % self.datasource + raise OGRShapeReaderError(msg) + + def wkts(self, where=None): + if not self.opened: self.open() + + if where: + if not where.lower().startswith('select'): + layer = libgdal.OGR_DS_GetLayer(self._ds, 0) + layer_def = libgdal.OGR_L_GetLayerDefn(layer) + name = libgdal.OGR_FD_GetName(layer_def) + where = 'select * from %s where %s' % (name, where) + layer = libgdal.OGR_DS_ExecuteSQL(self._ds, where, None, None) + else: + layer = libgdal.OGR_DS_GetLayer(self._ds, 0) + if layer is None: + msg = libgdal.CPLGetLastErrorMsg() + raise OGRShapeReaderError(msg) + + libgdal.OGR_L_ResetReading(layer) + while True: + feature = libgdal.OGR_L_GetNextFeature(layer) + if feature is None: + break + geom = libgdal.OGR_F_GetGeometryRef(feature) + res = c_char_p() + libgdal.OGR_G_ExportToWkt(geom, ctypes.byref(res)) + yield res.value + libgdal.VSIFree(res) + libgdal.OGR_F_Destroy(feature) + + if where: + libgdal.OGR_DS_ReleaseResultSet(self._ds, layer) + + def close(self): + if self.opened: + libgdal.OGR_DS_Destroy(self._ds) + self.opened = False + + def __del__(self): + self.close() + +if __name__ == '__main__': + import sys + reader = OGRShapeReader(sys.argv[1]) + where = None + if len(sys.argv) == 3: + where = sys.argv[2] + for wkt in reader.wkts(where): + print wkt \ No newline at end of file diff --git a/imposm/version.py b/imposm/version.py index 9ef161f..ed295c8 100644 --- a/imposm/version.py +++ b/imposm/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '2.4.0' +__version__ = '2.5.0' diff --git a/imposm/writer.py b/imposm/writer.py index 5b9067e..9fb9db1 100644 --- a/imposm/writer.py +++ b/imposm/writer.py @@ -14,9 +14,23 @@ from multiprocessing import Process, JoinableQueue -from imposm.dbimporter import NodeProcess, WayProcess, RelationProcess +from imposm.dbimporter import NodeProcessTuple, WayProcessTuple, RelationProcessTuple +from imposm.dbimporter import NodeProcessDict, WayProcessDict, RelationProcessDict from imposm.util import create_pool, shutdown_pool +import_processes = { + 'tuple': { + 'node': NodeProcessTuple, + 'way': WayProcessTuple, + 'relation': RelationProcessTuple, + }, + 'dict': { + 'node': NodeProcessDict, + 'way': WayProcessDict, + 'relation': RelationProcessDict, + } +} + class ImposmWriter(object): def __init__(self, mapping, db, cache, pool_size=2, logger=None, dry_run=False): self.mapping = mapping @@ -55,7 +69,8 @@ class ImposmWriter(object): way_marker = WayMarkerProcess(inserted_way_queue, self.cache, self.logger) way_marker.start() - self._write_elem(RelationProcess, cache, log, self.pool_size, [inserted_way_queue]) + self._write_elem(import_processes[self.db.insert_data_format]['relation'], + cache, log, self.pool_size, [inserted_way_queue]) inserted_way_queue.put(None) way_marker.join() @@ -63,13 +78,15 @@ class ImposmWriter(object): def ways(self): cache = self.cache.ways_cache() log = self.logger('ways', len(cache)) - self._write_elem(WayProcess, cache, log, self.pool_size) + self._write_elem(import_processes[self.db.insert_data_format]['way'], + cache, log, self.pool_size) self.cache.remove_inserted_way_cache() def nodes(self): cache = self.cache.nodes_cache() log = self.logger('nodes', len(cache)) - self._write_elem(NodeProcess, cache, log, self.pool_size) + self._write_elem(import_processes[self.db.insert_data_format]['node'], + cache, log, self.pool_size) class WayMarkerProcess(Process): diff --git a/setup.py b/setup.py index e9f7d04..84386da 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ setup( name = "imposm", version = version, description='OpenStreetMap importer for PostGIS.', - long_description=open('README').read() + open('CHANGES').read(), + long_description=open('README.rst').read() + open('CHANGES').read(), author = "Oliver Tonnhofer", author_email = "o...@omniscale.de", url='http://imposm.org/', -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/imposm.git _______________________________________________ Pkg-grass-devel mailing list Pkg-grass-devel@lists.alioth.debian.org http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-grass-devel