This is an automated email from the git hooks/post-receive script. sebastic pushed a commit to branch master in repository mapnik-vector-tile.
commit 6730b43b86a8a25feb951a6151e357a265664f0e Author: Bas Couwenberg <sebas...@xs4all.nl> Date: Thu Feb 8 16:21:04 2018 +0100 New upstream version 1.6.0+dfsg --- .travis.yml | 9 + CHANGELOG.md | 6 + Makefile | 4 +- README.md | 6 + bench/vtile-encode.cpp | 2 +- bench/vtile-transform.cpp | 5 +- gyp/build.gyp | 43 ++++- install_mason.sh | 12 +- package.json | 2 +- src/vector_tile_composite.hpp | 2 +- src/vector_tile_datasource_pbf.ipp | 5 +- src/vector_tile_douglas_peucker.hpp | 280 ++++++++++++++++++++++++-------- src/vector_tile_geometry_decoder.ipp | 37 ++++- src/vector_tile_geometry_simplifier.hpp | 8 +- src/vector_tile_layer.hpp | 4 +- src/vector_tile_load_tile.hpp | 8 +- src/vector_tile_merc_tile.hpp | 8 +- src/vector_tile_projection.cpp | 2 - src/vector_tile_projection.hpp | 38 ++--- src/vector_tile_projection.ipp | 62 ------- test/raster_tile.cpp | 6 +- test/unit/decoding/linestring.cpp | 28 ++++ test/unit/decoding/point.cpp | 23 +++ test/unit/decoding/polygon.cpp | 40 +++++ test/vector_tile_pbf.cpp | 5 +- test/vector_tile_projection.cpp | 15 +- 26 files changed, 441 insertions(+), 219 deletions(-) diff --git a/.travis.yml b/.travis.yml index 42f2bce..f56235c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,15 @@ matrix: apt: sources: ['ubuntu-toolchain-r-test'] packages: [ 'libstdc++-5-dev', 'xutils-dev'] + # test a build with sse disabled + - os: linux + sudo: false + compiler: ": clang" + env: JOBS=10 SSE_MATH=false CXX="ccache clang++-4.0 -Qunused-arguments" + addons: + apt: + sources: ['ubuntu-toolchain-r-test'] + packages: [ 'libstdc++-5-dev', 'xutils-dev'] - os: linux sudo: false compiler: ": clang-coverage" diff --git a/CHANGELOG.md b/CHANGELOG.md index 93eae1b..2fad2b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 1.6.0 + +- Bug fix for possible massive allocations in invalid vector tiles +- Fixed mercator bounding box code so that it does not require tile size, removed spherical mercator class +- Added the ability to use SSE code to improve the performance of simplification prior to vector tile creation. + ## 1.5.0 - Added back ability to build against external mapnik (see docs for instructions) diff --git a/Makefile b/Makefile index 669d90f..95096ee 100755 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ GYP_REVISION=3464008 +SSE_MATH ?= true + default: release mason_packages/.link/bin: @@ -16,7 +18,7 @@ pre_build_check: git clone https://chromium.googlesource.com/external/gyp.git ./deps/gyp && cd ./deps/gyp && git checkout $(GYP_REVISION) build/Makefile: pre_build_check ./deps/gyp gyp/build.gyp test/* - deps/gyp/gyp gyp/build.gyp --depth=. -DMAPNIK_PLUGINDIR=\"$(shell mapnik-config --input-plugins)\" -Goutput_dir=. --generator-output=./build -f make + deps/gyp/gyp gyp/build.gyp -Denable_sse=$(SSE_MATH) --depth=. -DMAPNIK_PLUGINDIR=\"$(shell mapnik-config --input-plugins)\" -Goutput_dir=. --generator-output=./build -f make $(MAKE) -C build/ V=$(V) release: mason_packages/.link/bin/mapnik-config Makefile diff --git a/README.md b/README.md index 5e8354a..df515ba 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,12 @@ If you have Mapnik, libprotobuf, and all the Mapnik deps already installed on yo make release_base +Note: SSE optimizations are enabled by default. If you want to turn them off do: + +``` +SSE_MATH=false make +``` + If building against an external Mapnik please know that Mapnik Vector Tile does not currently support Mapnik 3.1.x. - mapnik-vector-tile >=1.4.x depends on Mapnik >=v3.0.14 diff --git a/bench/vtile-encode.cpp b/bench/vtile-encode.cpp index bc15088..67284a0 100644 --- a/bench/vtile-encode.cpp +++ b/bench/vtile-encode.cpp @@ -133,7 +133,7 @@ int main(int argc, char** argv) unsigned tile_size = layer_extent; int buffer_size = 0; - mapnik::box2d<double> bbox = mapnik::vector_tile_impl::merc_extent(tile_size, x, y, z); + mapnik::box2d<double> bbox = mapnik::vector_tile_impl::tile_mercator_bbox(x, y, z); // Create a fresh map to render into a tile mapnik::Map map(tile_size,tile_size,"+init=epsg:3857"); diff --git a/bench/vtile-transform.cpp b/bench/vtile-transform.cpp index 5f24990..378d205 100644 --- a/bench/vtile-transform.cpp +++ b/bench/vtile-transform.cpp @@ -37,10 +37,7 @@ int main() mapnik::proj_transform prj_trans(merc,merc); // no-op mapnik::proj_transform prj_trans2(merc2,merc); // op unsigned tile_size = 256; - mapnik::vector_tile_impl::spherical_mercator merc_tiler(tile_size); - double minx,miny,maxx,maxy; - merc_tiler.xyz(9664,20435,15,minx,miny,maxx,maxy); - mapnik::box2d<double> z15_extent(minx,miny,maxx,maxy); + mapnik::box2d<double> z15_extent = mapnik::vector_tile_impl::tile_mercator_bbox(9664,20435,15); unsigned path_multiplier = 16; mapnik::view_transform tr(tile_size * path_multiplier, tile_size * path_multiplier, diff --git a/gyp/build.gyp b/gyp/build.gyp index 7ab392a..a74cf3f 100644 --- a/gyp/build.gyp +++ b/gyp/build.gyp @@ -4,8 +4,9 @@ ], 'variables': { 'MAPNIK_PLUGINDIR%': '', + 'enable_sse%':'true', 'common_defines' : [ - 'MAPNIK_VECTOR_TILE_LIBRARY=1', + 'MAPNIK_VECTOR_TILE_LIBRARY=1' ] }, "targets": [ @@ -39,6 +40,11 @@ 'include_dirs': [ '<(SHARED_INTERMEDIATE_DIR)/' ], + 'conditions': [ + ['enable_sse == "true"', { + 'defines' : [ 'SSE_MATH' ] + }] + ], 'cflags_cc' : [ '-D_THREAD_SAFE', '<!@(mapnik-config --cflags)', @@ -81,6 +87,11 @@ 'defines' : [ "<@(common_defines)" ], + 'conditions': [ + ['enable_sse == "true"', { + 'defines' : [ 'SSE_MATH' ] + }] + ], 'cflags_cc' : [ '<!@(mapnik-config --cflags)' ], @@ -123,6 +134,11 @@ "<@(common_defines)", "MAPNIK_PLUGINDIR=<(MAPNIK_PLUGINDIR)" ], + 'conditions': [ + ['enable_sse == "true"', { + 'defines' : [ 'SSE_MATH' ] + }] + ], "sources": [ "<!@(find ../test/ -name '*.cpp')" ], @@ -140,6 +156,11 @@ "<@(common_defines)", "MAPNIK_PLUGINDIR=<(MAPNIK_PLUGINDIR)" ], + 'conditions': [ + ['enable_sse == "true"', { + 'defines' : [ 'SSE_MATH' ] + }] + ], "sources": [ "../bench/vtile-transform.cpp" ], @@ -155,6 +176,11 @@ "<@(common_defines)", "MAPNIK_PLUGINDIR=<(MAPNIK_PLUGINDIR)" ], + 'conditions': [ + ['enable_sse == "true"', { + 'defines' : [ 'SSE_MATH' ] + }] + ], "sources": [ "../bench/vtile-decode.cpp" ], @@ -170,6 +196,11 @@ "<@(common_defines)", "MAPNIK_PLUGINDIR=<(MAPNIK_PLUGINDIR)" ], + 'conditions': [ + ['enable_sse == "true"', { + 'defines' : [ 'SSE_MATH' ] + }] + ], "sources": [ "../bench/vtile-encode.cpp" ], @@ -185,6 +216,11 @@ "<@(common_defines)", "MAPNIK_PLUGINDIR=<(MAPNIK_PLUGINDIR)" ], + 'conditions': [ + ['enable_sse == "true"', { + 'defines' : [ 'SSE_MATH' ] + }] + ], "sources": [ "../bin/vtile-edit.cpp" ], @@ -199,6 +235,11 @@ "sources": [ "../examples/c++/tileinfo.cpp" ], + 'conditions': [ + ['enable_sse == "true"', { + 'defines' : [ 'SSE_MATH' ] + }] + ], "include_dirs": [ "../src" ], diff --git a/install_mason.sh b/install_mason.sh index f20aea3..8b69fa7 100755 --- a/install_mason.sh +++ b/install_mason.sh @@ -9,7 +9,7 @@ function install() { } ICU_VERSION="57.1" -MASON_VERSION="v0.14.2" +MASON_VERSION="9a3f4d195614bb30d9ca54dd6070739de605cb14" if [ ! -f ./mason/mason.sh ]; then mkdir -p ./mason @@ -35,14 +35,14 @@ if [[ ${SKIP_MAPNIK_INSTALL:-} != 'YES' ]] && [[ ! -f ./mason_packages/.link/bin install cairo 1.14.8 install webp 0.6.0 install libgdal 2.1.3 - install boost 1.63.0 - install boost_libsystem 1.63.0 - install boost_libfilesystem 1.63.0 - install boost_libregex_icu57 1.63.0 + install boost 1.65.1 + install boost_libsystem 1.65.1 + install boost_libfilesystem 1.65.1 + install boost_libregex_icu57 1.65.1 install freetype 2.7.1 install harfbuzz 1.4.2-ft # mapnik - install mapnik 3.0.14 + install mapnik 3.0.18 fi diff --git a/package.json b/package.json index e19a8d9..144ee69 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mapnik-vector-tile", - "version": "1.5.0", + "version": "1.6.0", "description": "Mapnik Vector Tile API", "main": "include_dirs.js", "repository" : { diff --git a/src/vector_tile_composite.hpp b/src/vector_tile_composite.hpp index c58b763..a6f1e95 100644 --- a/src/vector_tile_composite.hpp +++ b/src/vector_tile_composite.hpp @@ -19,7 +19,7 @@ namespace mapnik namespace vector_tile_impl { -void composite(merc_tile & target_vt, +inline void composite(merc_tile & target_vt, std::vector<merc_tile_ptr> const& vtiles, mapnik::Map & map, processor & ren, // processor should be on map object provided diff --git a/src/vector_tile_datasource_pbf.ipp b/src/vector_tile_datasource_pbf.ipp index b9800e8..3a85a69 100644 --- a/src/vector_tile_datasource_pbf.ipp +++ b/src/vector_tile_datasource_pbf.ipp @@ -219,10 +219,7 @@ void tile_datasource_pbf::set_envelope(box2d<double> const& bbox) box2d<double> tile_datasource_pbf::get_tile_extent() const { - spherical_mercator merc(tile_size_); - double minx,miny,maxx,maxy; - merc.xyz(x_,y_,z_,minx,miny,maxx,maxy); - return box2d<double>(minx,miny,maxx,maxy); + return tile_mercator_bbox(x_, y_, z_); } box2d<double> tile_datasource_pbf::envelope() const diff --git a/src/vector_tile_douglas_peucker.hpp b/src/vector_tile_douglas_peucker.hpp index 8e3010d..b3e4e8c 100644 --- a/src/vector_tile_douglas_peucker.hpp +++ b/src/vector_tile_douglas_peucker.hpp @@ -40,6 +40,13 @@ // std #include <vector> +#include <limits> + +#ifdef SSE_MATH +// simd +#include <xmmintrin.h> +#include <emmintrin.h> +#endif namespace mapnik { @@ -67,14 +74,14 @@ struct douglas_peucker_point } }; -template <typename value_type, typename calc_type> -inline void consider(typename std::vector<douglas_peucker_point<value_type> >::iterator begin, - typename std::vector<douglas_peucker_point<value_type> >::iterator end, - calc_type const& max_dist) +template <typename Range> +inline void consider(Range const& vec, + std::vector<bool> & included, + std::size_t begin_idx, + std::size_t last_idx, + double max_dist) { - typedef typename std::vector<douglas_peucker_point<value_type> >::iterator iterator_type; - - std::size_t size = end - begin; + std::size_t size = last_idx - begin_idx + 1; // size must be at least 3 // because we want to consider a candidate point in between @@ -83,107 +90,236 @@ inline void consider(typename std::vector<douglas_peucker_point<value_type> >::i return; } - iterator_type last = end - 1; - // Find most far point, compare to the current segment - calc_type md(-1.0); // any value < 0 - iterator_type candidate; + double md = -1.0; // any value < 0 + std::size_t candidate = begin_idx + 1; + /* + Algorithm [p: (px,py), p1: (x1,y1), p2: (x2,y2)] + VECTOR v(x2 - x1, y2 - y1) + VECTOR w(px - x1, py - y1) + c1 = w . v + c2 = v . v + b = c1 / c2 + RETURN POINT(x1 + b * vx, y1 + b * vy) + */ + auto const& last = vec[last_idx]; + auto const& begin = vec[begin_idx]; + double const v_x = last.x - begin.x; + double const v_y = last.y - begin.y; + double const c2 = v_x * v_x + v_y * v_y; + std::size_t i = begin_idx + 1; + + #ifdef SSE_MATH + if (size > 4 && size < std::numeric_limits<std::uint32_t>::max()) { - /* - Algorithm [p: (px,py), p1: (x1,y1), p2: (x2,y2)] - VECTOR v(x2 - x1, y2 - y1) - VECTOR w(px - x1, py - y1) - c1 = w . v - c2 = v . v - b = c1 / c2 - RETURN POINT(x1 + b * vx, y1 + b * vy) - */ - calc_type const v_x = last->p.x - begin->p.x; - calc_type const v_y = last->p.y - begin->p.y; - calc_type const c2 = v_x * v_x + v_y * v_y; - for(iterator_type it = begin + 1; it != last; ++it) + __m128 begin_x_4 = _mm_set1_ps(begin.x); + __m128 begin_y_4 = _mm_set1_ps(begin.y); + __m128 last_x_4 = _mm_set1_ps(last.x); + __m128 last_y_4 = _mm_set1_ps(last.y); + __m128 v_x_4 = _mm_set1_ps(v_x); + __m128 v_y_4 = _mm_set1_ps(v_y); + __m128 c2_4 = _mm_set1_ps(c2); + __m128 zero_4 = _mm_set1_ps(0.0); + __m128 true_4 = _mm_castsi128_ps(_mm_set1_epi32(std::numeric_limits<std::uint32_t>::max())); + __m128 md_4 = _mm_set1_ps(md); + __m128 md_idx = _mm_castsi128_ps(_mm_set1_epi32(i)); + + for (; i + 3 < last_idx; i += 4) { - calc_type const w_x = it->p.x - begin->p.x; - calc_type const w_y = it->p.y - begin->p.y; - calc_type const c1 = w_x * v_x + w_y * v_y; - calc_type dist; - if (c1 <= 0) // calc_type() should be 0 of the proper calc type format - { - calc_type const dx = it->p.x - begin->p.x; - calc_type const dy = it->p.y - begin->p.y; - dist = dx * dx + dy * dy; - } - else if (c2 <= c1) - { - calc_type const dx = it->p.x - last->p.x; - calc_type const dy = it->p.y - last->p.y; - dist = dx * dx + dy * dy; - } - else + // Setup loop values + __m128 it_x_4 = _mm_set_ps(vec[i+3].x, vec[i+2].x, vec[i+1].x, vec[i].x); + __m128 it_y_4 = _mm_set_ps(vec[i+3].y, vec[i+2].y, vec[i+1].y, vec[i].y); + __m128 it_idx = _mm_castsi128_ps(_mm_set_epi32(i+3, i+2, i+1, i)); + + // Calculate c1 + __m128 w_x_4 = _mm_sub_ps(it_x_4, begin_x_4); + __m128 w_y_4 = _mm_sub_ps(it_y_4, begin_y_4); + __m128 u_x_4 = _mm_sub_ps(it_x_4, last_x_4); + __m128 u_y_4 = _mm_sub_ps(it_y_4, last_y_4); + __m128 c1_4 = _mm_add_ps(_mm_mul_ps(w_x_4, v_x_4), _mm_mul_ps(w_y_4, v_y_4)); + + // Calculate two optional distance results + __m128 dist_1 = _mm_add_ps(_mm_mul_ps(w_x_4, w_x_4), _mm_mul_ps(w_y_4, w_y_4)); + __m128 dist_2 = _mm_add_ps(_mm_mul_ps(u_x_4, u_x_4), _mm_mul_ps(u_y_4, u_y_4)); + __m128 bool_dist_1 = _mm_cmple_ps(c1_4, zero_4); + __m128 bool_dist_2 = _mm_cmple_ps(c2_4, c1_4); + + // Calculate the default distance solution now + __m128 b_4 = _mm_div_ps(c1_4, c2_4); + __m128 dx_4 = _mm_sub_ps(w_x_4, _mm_mul_ps(b_4, v_x_4)); + __m128 dy_4 = _mm_sub_ps(w_y_4, _mm_mul_ps(b_4, v_y_4)); + __m128 dist_3 = _mm_add_ps(_mm_mul_ps(dx_4, dx_4), _mm_mul_ps(dy_4, dy_4)); + + __m128 dist = _mm_and_ps(dist_1, bool_dist_1); + dist = _mm_or_ps(dist, _mm_and_ps(_mm_and_ps(_mm_andnot_ps(bool_dist_1, true_4), bool_dist_2), dist_2)); + dist = _mm_or_ps(dist, _mm_and_ps(_mm_andnot_ps(_mm_or_ps(bool_dist_1, bool_dist_2), true_4), dist_3)); + __m128 bool_md = _mm_cmplt_ps(md_4, dist); + md_4 = _mm_or_ps(_mm_andnot_ps(bool_md, md_4), _mm_and_ps(bool_md, dist)); + md_idx = _mm_or_ps(_mm_andnot_ps(bool_md, md_idx), _mm_and_ps(bool_md, it_idx)); + } + + float md_out [4]; + std::uint32_t md_idx_out [4]; + _mm_store_ps(md_out, md_4); + _mm_store_ps(reinterpret_cast<float*>(md_idx_out), md_idx); + + md = md_out[0]; + candidate = md_idx_out[0]; + for (std::size_t j = 1; j < 4; ++j) + { + if (md < md_out[j]) { - // See above, c1 > 0 AND c2 > c1 so: c2 != 0 - calc_type const b = c1 / c2; - calc_type const p_x = begin->p.x + b * v_x; - calc_type const p_y = begin->p.y + b * v_y; - calc_type const dx = it->p.x - p_x; - calc_type const dy = it->p.y - p_y; - dist = dx * dx + dy * dy; + md = md_out[j]; + candidate = md_idx_out[j]; } - if (md < dist) + else if (md == md_out[j] && md_idx_out[j] < candidate) { - md = dist; - candidate = it; + candidate = md_idx_out[j]; } } + } + else if (size > 2) + { + __m128d begin_x_2 = _mm_set1_pd(begin.x); + __m128d begin_y_2 = _mm_set1_pd(begin.y); + __m128d last_x_2 = _mm_set1_pd(last.x); + __m128d last_y_2 = _mm_set1_pd(last.y); + __m128d v_x_2 = _mm_set1_pd(v_x); + __m128d v_y_2 = _mm_set1_pd(v_y); + __m128d c2_2 = _mm_set1_pd(c2); + __m128d zero_2 = _mm_set1_pd(0.0); + __m128d true_2 = _mm_castsi128_pd(_mm_set1_epi64(reinterpret_cast<__m64>(std::numeric_limits<std::uint64_t>::max()))); + __m128d md_2 = _mm_set1_pd(md); + __m128d md_idx = _mm_castsi128_pd(_mm_set1_epi64(reinterpret_cast<__m64>(i))); + + for (; i + 1 < last_idx; i += 2) + { + // Setup loop values + __m128d it_x_2 = _mm_set_pd(vec[i+1].x, vec[i].x); + __m128d it_y_2 = _mm_set_pd(vec[i+1].y, vec[i].y); + __m128d it_idx = _mm_castsi128_pd(_mm_set_epi64(reinterpret_cast<__m64>(i+1), reinterpret_cast<__m64>(i))); + + // Calculate c1 + __m128d w_x_2 = _mm_sub_pd(it_x_2, begin_x_2); + __m128d w_y_2 = _mm_sub_pd(it_y_2, begin_y_2); + __m128d u_x_2 = _mm_sub_pd(it_x_2, last_x_2); + __m128d u_y_2 = _mm_sub_pd(it_y_2, last_y_2); + __m128d c1_2 = _mm_add_pd(_mm_mul_pd(w_x_2, v_x_2), _mm_mul_pd(w_y_2, v_y_2)); + + // Calculate two optional distance results + __m128d dist_1 = _mm_add_pd(_mm_mul_pd(w_x_2, w_x_2), _mm_mul_pd(w_y_2, w_y_2)); + __m128d dist_2 = _mm_add_pd(_mm_mul_pd(u_x_2, u_x_2), _mm_mul_pd(u_y_2, u_y_2)); + __m128d bool_dist_1 = _mm_cmple_pd(c1_2, zero_2); + __m128d bool_dist_2 = _mm_cmple_pd(c2_2, c1_2); + + // Calculate the default distance solution now + __m128d b_2 = _mm_div_pd(c1_2, c2_2); + __m128d dx_2 = _mm_sub_pd(w_x_2, _mm_mul_pd(b_2, v_x_2)); + __m128d dy_2 = _mm_sub_pd(w_y_2, _mm_mul_pd(b_2, v_y_2)); + __m128d dist_3 = _mm_add_pd(_mm_mul_pd(dx_2, dx_2), _mm_mul_pd(dy_2, dy_2)); + + __m128d dist = _mm_and_pd(dist_1, bool_dist_1); + dist = _mm_or_pd(dist, _mm_and_pd(_mm_and_pd(_mm_andnot_pd(bool_dist_1, true_2), bool_dist_2), dist_2)); + dist = _mm_or_pd(dist, _mm_and_pd(_mm_andnot_pd(_mm_or_pd(bool_dist_1, bool_dist_2), true_2), dist_3)); + __m128d bool_md = _mm_cmplt_pd(md_2, dist); + md_2 = _mm_or_pd(_mm_andnot_pd(bool_md, md_2), _mm_and_pd(bool_md, dist)); + md_idx = _mm_or_pd(_mm_andnot_pd(bool_md, md_idx), _mm_and_pd(bool_md, it_idx)); + } + + double md_out [2]; + std::size_t md_idx_out [2]; + _mm_store_pd(md_out, md_2); + _mm_store_pd(reinterpret_cast<double*>(md_idx_out), md_idx); + + md = md_out[0]; + candidate = md_idx_out[0]; + if (md < md_out[1]) + { + md = md_out[1]; + candidate = md_idx_out[1]; + } + else if (md == md_out[1] && md_idx_out[1] < candidate) + { + candidate = md_idx_out[1]; + } + } + #endif + + for (; i < last_idx; ++i) + { + auto const& it = vec[i]; + double const w_x = it.x - begin.x; + double const w_y = it.y - begin.y; + double const c1 = w_x * v_x + w_y * v_y; + double dist; + if (c1 <= 0) // calc_type() should be 0 of the proper calc type format + { + dist = w_x * w_x + w_y * w_y; + } + else if (c2 <= c1) + { + double const dx = it.x - last.x; + double const dy = it.y - last.y; + dist = dx * dx + dy * dy; + } + else + { + // See above, c1 > 0 AND c2 > c1 so: c2 != 0 + double const b = c1 / c2; + double const dx = w_x - b * v_x; + double const dy = w_y - b * v_y; + dist = dx * dx + dy * dy; + } + if (md < dist) + { + md = dist; + candidate = i; + } } // If a point is found, set the include flag // and handle segments in between recursively if (max_dist < md) { - candidate->included = true; - consider<value_type>(begin, candidate + 1, max_dist); - consider<value_type>(candidate, end, max_dist); + included[candidate] = true; + consider<Range>(vec, included, begin_idx, candidate, max_dist); + consider<Range>(vec, included, candidate, last_idx, max_dist); } } } // end ns detail -template <typename value_type, typename calc_type, typename Range, typename OutputIterator> +template <typename Range, typename OutputIterator> inline void douglas_peucker(Range const& range, OutputIterator out, - calc_type max_distance) + double max_distance) { - // Copy coordinates, a vector of references to all points - std::vector<detail::douglas_peucker_point<value_type> > ref_candidates(std::begin(range), - std::end(range)); + std::vector<bool> included(range.size()); // Include first and last point of line, // they are always part of the line - ref_candidates.front().included = true; - ref_candidates.back().included = true; + included.front() = true; + included.back() = true; // We will compare to squared of distance so we don't have to do a sqrt - calc_type const max_sqrd = max_distance * max_distance; + double const max_sqrd = max_distance * max_distance; - // Get points, recursively, including them if they are further away - // than the specified distance - detail::consider<value_type, calc_type>(std::begin(ref_candidates), std::end(ref_candidates), max_sqrd); + detail::consider<Range>(range, included, 0, range.size() - 1, max_sqrd); // Copy included elements to the output - for(typename std::vector<detail::douglas_peucker_point<value_type> >::const_iterator it - = std::begin(ref_candidates); - it != std::end(ref_candidates); - ++it) + auto data = std::begin(range); + auto end_data = std::end(range); + auto inc = included.begin(); + for (; data != end_data; ++data) { - if (it->included) + if (*inc) { // copy-coordinates does not work because OutputIterator // does not model Point (??) //geometry::convert(it->p, *out); - *out = it->p; - out++; + *out++ = *data; } + ++inc; } } diff --git a/src/vector_tile_geometry_decoder.ipp b/src/vector_tile_geometry_decoder.ipp index d4bc43e..9245563 100644 --- a/src/vector_tile_geometry_decoder.ipp +++ b/src/vector_tile_geometry_decoder.ipp @@ -63,6 +63,13 @@ inline double get_point_value<double>(double const val, return tile_loc + (val / scale_val); } +constexpr std::size_t max_reserve() +{ + // Based on int64_t geometry being 16 bytes in size and + // maximum allocation size of 1 MB. + return (1024 * 1024) / 16; +} + template <typename geom_value_type> void decode_point(mapnik::geometry::geometry<geom_value_type> & geom, GeometryPBF & paths, @@ -100,7 +107,15 @@ void decode_point(mapnik::geometry::geometry<geom_value_type> & geom, } previous_len = paths.get_length(); #endif - mp.reserve(paths.get_length() + 1); + constexpr std::size_t max_size = max_reserve(); + if (paths.get_length() + 1 < max_size) + { + mp.reserve(paths.get_length() + 1); + } + else + { + mp.reserve(max_size); + } mp.emplace_back(x1_, y1_); break; } @@ -216,7 +231,15 @@ void decode_linestring(mapnik::geometry::geometry<geom_value_type> & geom, multi_line.emplace_back(); auto & line = multi_line.back(); // reserve prior - line.reserve(paths.get_length() + 2); + constexpr std::size_t max_size = max_reserve(); + if (paths.get_length() + 2 < max_size) + { + line.reserve(paths.get_length() + 2); + } + else + { + line.reserve(max_size); + } // add moveto command position x0_ = get_point_value<geom_value_type>(x0, scale_x, tile_x); y0_ = get_point_value<geom_value_type>(y0, scale_y, tile_y); @@ -390,7 +413,15 @@ void decode_polygon(mapnik::geometry::geometry<geom_value_type> & geom, rings.emplace_back(); auto & ring = rings.back(); // reserve prior - ring.reserve(paths.get_length() + 4); + constexpr std::size_t max_size = max_reserve(); + if (paths.get_length() + 4 < max_size) + { + ring.reserve(paths.get_length() + 4); + } + else + { + ring.reserve(max_size); + } // add moveto command position x0_ = get_point_value<geom_value_type>(x0, scale_x, tile_x); y0_ = get_point_value<geom_value_type>(y0, scale_y, tile_y); diff --git a/src/vector_tile_geometry_simplifier.hpp b/src/vector_tile_geometry_simplifier.hpp index 50e6a5b..ea7d42f 100644 --- a/src/vector_tile_geometry_simplifier.hpp +++ b/src/vector_tile_geometry_simplifier.hpp @@ -41,7 +41,7 @@ struct geometry_simplifier else { mapbox::geometry::line_string<std::int64_t> simplified; - douglas_peucker<std::int64_t>(geom, std::back_inserter(simplified), simplify_distance_); + douglas_peucker(geom, std::back_inserter(simplified), simplify_distance_); next_(simplified); } } @@ -58,7 +58,7 @@ struct geometry_simplifier else { mapbox::geometry::line_string<std::int64_t> simplified_line; - douglas_peucker<std::int64_t>(g, std::back_inserter(simplified_line), simplify_distance_); + douglas_peucker(g, std::back_inserter(simplified_line), simplify_distance_); simplified.push_back(simplified_line); } } @@ -77,7 +77,7 @@ struct geometry_simplifier else { mapbox::geometry::linear_ring<std::int64_t> simplified_ring; - douglas_peucker<std::int64_t>(g, std::back_inserter(simplified_ring), simplify_distance_); + douglas_peucker(g, std::back_inserter(simplified_ring), simplify_distance_); simplified.push_back(simplified_ring); } } @@ -99,7 +99,7 @@ struct geometry_simplifier else { mapbox::geometry::linear_ring<std::int64_t> simplified_ring; - douglas_peucker<std::int64_t>(g, std::back_inserter(simplified_ring), simplify_distance_); + douglas_peucker(g, std::back_inserter(simplified_ring), simplify_distance_); simplified.push_back(simplified_ring); } } diff --git a/src/vector_tile_layer.hpp b/src/vector_tile_layer.hpp index 532b0f6..af512e6 100644 --- a/src/vector_tile_layer.hpp +++ b/src/vector_tile_layer.hpp @@ -243,7 +243,7 @@ public: { if (!prj_trans_.forward(unbuffered_query_extent, PROJ_ENVELOPE_POINTS)) { - throw std::runtime_error("vector_tile_processor: unbuffered query extent did not repoject back to map projection"); + throw std::runtime_error("vector_tile_processor: unbuffered query extent did not reproject back to map projection"); } } @@ -265,7 +265,7 @@ public: // forward project layer extent back into native projection if (!prj_trans_.forward(query_extent, PROJ_ENVELOPE_POINTS)) { - throw std::runtime_error("vector_tile_processor: query extent did not repoject back to map projection"); + throw std::runtime_error("vector_tile_processor: query extent did not reproject back to map projection"); } } else diff --git a/src/vector_tile_load_tile.hpp b/src/vector_tile_load_tile.hpp index db9d3ed..9b92c11 100644 --- a/src/vector_tile_load_tile.hpp +++ b/src/vector_tile_load_tile.hpp @@ -29,7 +29,7 @@ namespace mapnik namespace vector_tile_impl { -std::pair<std::string,std::uint32_t> get_layer_name_and_version(protozero::pbf_reader & layer_msg) +inline std::pair<std::string,std::uint32_t> get_layer_name_and_version(protozero::pbf_reader & layer_msg) { std::string name; std::uint32_t version = 1; @@ -51,7 +51,7 @@ std::pair<std::string,std::uint32_t> get_layer_name_and_version(protozero::pbf_r return std::make_pair(name,version); } -void merge_from_buffer(merc_tile & t, const char * data, std::size_t size, bool validate = false, bool upgrade = false) +inline void merge_from_buffer(merc_tile & t, const char * data, std::size_t size, bool validate = false, bool upgrade = false) { using ds_ptr = std::shared_ptr<mapnik::vector_tile_impl::tile_datasource_pbf>; protozero::pbf_reader tile_msg(data, size); @@ -137,7 +137,7 @@ void merge_from_buffer(merc_tile & t, const char * data, std::size_t size, bool t.buffer_size(prev_buffer_size); } -void merge_from_compressed_buffer(merc_tile & t, const char * data, std::size_t size, bool validate = false, bool upgrade = false) +inline void merge_from_compressed_buffer(merc_tile & t, const char * data, std::size_t size, bool validate = false, bool upgrade = false) { if (mapnik::vector_tile_impl::is_gzip_compressed(data,size) || mapnik::vector_tile_impl::is_zlib_compressed(data,size)) @@ -152,7 +152,7 @@ void merge_from_compressed_buffer(merc_tile & t, const char * data, std::size_t } } -void add_image_buffer_as_tile_layer(merc_tile & t, std::string const& layer_name, const char * data, std::size_t size) +inline void add_image_buffer_as_tile_layer(merc_tile & t, std::string const& layer_name, const char * data, std::size_t size) { std::string layer_buffer; protozero::pbf_writer layer_writer(layer_buffer); diff --git a/src/vector_tile_merc_tile.hpp b/src/vector_tile_merc_tile.hpp index c86fabd..6377021 100644 --- a/src/vector_tile_merc_tile.hpp +++ b/src/vector_tile_merc_tile.hpp @@ -23,7 +23,7 @@ public: std::uint64_t z, std::uint32_t tile_size = 4096, std::int32_t buffer_size = 128) - : tile(merc_extent(tile_size, x, y, z), tile_size, buffer_size), + : tile(tile_mercator_bbox(x, y, z), tile_size, buffer_size), x_(x), y_(y), z_(z) {} @@ -47,7 +47,7 @@ public: void x(std::uint64_t x) { x_ = x; - extent_ = merc_extent(tile_size_, x_, y_, z_); + extent_ = tile_mercator_bbox(x_, y_, z_); } std::uint64_t y() const @@ -58,7 +58,7 @@ public: void y(std::uint64_t y) { y_ = y; - extent_ = merc_extent(tile_size_, x_, y_, z_); + extent_ = tile_mercator_bbox(x_, y_, z_); } std::uint64_t z() const @@ -69,7 +69,7 @@ public: void z(std::uint64_t z) { z_ = z; - extent_ = merc_extent(tile_size_, x_, y_, z_); + extent_ = tile_mercator_bbox(x_, y_, z_); } }; diff --git a/src/vector_tile_projection.cpp b/src/vector_tile_projection.cpp deleted file mode 100644 index 87bb291..0000000 --- a/src/vector_tile_projection.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#include "vector_tile_projection.hpp" -#include "vector_tile_projection.ipp" diff --git a/src/vector_tile_projection.hpp b/src/vector_tile_projection.hpp index 88d86fb..b48d98c 100644 --- a/src/vector_tile_projection.hpp +++ b/src/vector_tile_projection.hpp @@ -6,6 +6,7 @@ // mapnik #include <mapnik/box2d.hpp> +#include <mapnik/well_known_srs.hpp> namespace mapnik { @@ -13,36 +14,21 @@ namespace mapnik namespace vector_tile_impl { -class spherical_mercator +inline mapnik::box2d<double> tile_mercator_bbox(std::uint64_t x, + std::uint64_t y, + std::uint64_t z) { -private: - double tile_size_; -public: - spherical_mercator(unsigned tile_size) - : tile_size_(static_cast<double>(tile_size)) {} - - MAPNIK_VECTOR_INLINE void from_pixels(double shift, double & x, double & y); - - MAPNIK_VECTOR_INLINE void xyz(std::uint64_t x, - std::uint64_t y, - std::uint64_t z, - double & minx, - double & miny, - double & maxx, - double & maxy); -}; - -MAPNIK_VECTOR_INLINE mapnik::box2d<double> merc_extent(std::uint32_t tile_size, - std::uint64_t x, - std::uint64_t y, - std::uint64_t z); + const double half_of_equator = M_PI * EARTH_RADIUS; + const double tile_size = 2.0 * half_of_equator / (1ull << z); + double minx = -half_of_equator + x * tile_size; + double miny = half_of_equator - (y + 1.0) * tile_size; + double maxx = -half_of_equator + (x + 1.0) * tile_size; + double maxy = half_of_equator - y * tile_size; + return mapnik::box2d<double>(minx,miny,maxx,maxy); +} } // end vector_tile_impl ns } // end mapnik ns -#if !defined(MAPNIK_VECTOR_TILE_LIBRARY) -#include "vector_tile_projection.ipp" -#endif - #endif // __MAPNIK_VECTOR_TILE_PROJECTION_H__ diff --git a/src/vector_tile_projection.ipp b/src/vector_tile_projection.ipp deleted file mode 100644 index d357682..0000000 --- a/src/vector_tile_projection.ipp +++ /dev/null @@ -1,62 +0,0 @@ -// mapnik-vector-tile -#include "vector_tile_config.hpp" - -// mapnik -#include <mapnik/box2d.hpp> -#include <mapnik/well_known_srs.hpp> - -// std -#include <cmath> -#include <cstdint> - -#ifndef M_PI -#define M_PI 3.141592653589793238462643 -#endif - -namespace mapnik -{ - -namespace vector_tile_impl -{ - -MAPNIK_VECTOR_INLINE void spherical_mercator::from_pixels(double shift, double & x, double & y) -{ - double b = shift/2.0; - x = (x - b)/(shift/360.0); - double g = (y - b)/-(shift/(2 * M_PI)); - y = R2D * (2.0 * std::atan(std::exp(g)) - M_PI_by2); -} - -MAPNIK_VECTOR_INLINE void spherical_mercator::xyz(std::uint64_t x, - std::uint64_t y, - std::uint64_t z, - double & minx, - double & miny, - double & maxx, - double & maxy) -{ - minx = x * tile_size_; - miny = (y + 1.0) * tile_size_; - maxx = (x + 1.0) * tile_size_; - maxy = y * tile_size_; - double shift = std::pow(2.0,z) * tile_size_; - from_pixels(shift,minx,miny); - from_pixels(shift,maxx,maxy); - lonlat2merc(&minx,&miny,1); - lonlat2merc(&maxx,&maxy,1); -} - -MAPNIK_VECTOR_INLINE mapnik::box2d<double> merc_extent(std::uint32_t tile_size, - std::uint64_t x, - std::uint64_t y, - std::uint64_t z) -{ - spherical_mercator merc(tile_size); - double minx,miny,maxx,maxy; - merc.xyz(x, y, z, minx, miny, maxx, maxy); - return mapnik::box2d<double>(minx,miny,maxx,maxy); -} - -} // end vector_tile_impl ns - -} // end mapnik ns diff --git a/test/raster_tile.cpp b/test/raster_tile.cpp index 466773b..fc50119 100644 --- a/test/raster_tile.cpp +++ b/test/raster_tile.cpp @@ -245,11 +245,7 @@ TEST_CASE("raster tile output 2") // now read back and render image at larger size // and zoomed in - double minx,miny,maxx,maxy; - mapnik::vector_tile_impl::spherical_mercator merc(256); - // 2/0/1.png - merc.xyz(0,1,2,minx,miny,maxx,maxy); - mapnik::box2d<double> bbox(minx,miny,maxx,maxy); + mapnik::box2d<double> bbox = mapnik::vector_tile_impl::tile_mercator_bbox(0,1,2); mapnik::Map map2(256,256,"+init=epsg:3857"); map2.set_buffer_size(1024); mapnik::layer lyr2("layer",map2.srs()); diff --git a/test/unit/decoding/linestring.cpp b/test/unit/decoding/linestring.cpp index fe2b5a5..b6ff02d 100644 --- a/test/unit/decoding/linestring.cpp +++ b/test/unit/decoding/linestring.cpp @@ -552,3 +552,31 @@ TEST_CASE("decode linestring that begins with two moveto commands") CHECK_THROWS(mapnik::vector_tile_impl::decode_geometry<double>(geoms, vector_tile::Tile_GeomType_LINESTRING, 2, 0.0, 0.0, 1.0, 1.0)); } } + +TEST_CASE("decode malicious linestring") +{ + // This should work with v1 but throw with v2 + vector_tile::Tile_Feature feature; + feature.set_type(vector_tile::Tile_GeomType_LINESTRING); + // MoveTo(1,1) + feature.add_geometry((1 << 3u) | 1u); + feature.add_geometry(protozero::encode_zigzag32(1)); + feature.add_geometry(protozero::encode_zigzag32(1)); + // LineTo(2,2) + feature.add_geometry((((1 << 29) - 1u) << 3u) | 2u); + feature.add_geometry(protozero::encode_zigzag32(1)); + feature.add_geometry(protozero::encode_zigzag32(1)); + + std::string feature_string = feature.SerializeAsString(); + mapnik::vector_tile_impl::GeometryPBF geoms = feature_to_pbf_geometry(feature_string); + + SECTION("VT Spec v1") + { + CHECK_THROWS(mapnik::vector_tile_impl::decode_geometry<double>(geoms, vector_tile::Tile_GeomType_LINESTRING, 1, 0.0, 0.0, 1.0, 1.0)); + } + + SECTION("VT Spec v2") + { + CHECK_THROWS(mapnik::vector_tile_impl::decode_geometry<double>(geoms, vector_tile::Tile_GeomType_LINESTRING, 2, 0.0, 0.0, 1.0, 1.0)); + } +} diff --git a/test/unit/decoding/point.cpp b/test/unit/decoding/point.cpp index db210e7..e2ca39d 100644 --- a/test/unit/decoding/point.cpp +++ b/test/unit/decoding/point.cpp @@ -363,3 +363,26 @@ TEST_CASE("multipoint with three movetos with command count 1") CHECK( geom.is<mapnik::geometry::multi_point<double> >() ); } } + +TEST_CASE("decode malicious point") +{ + vector_tile::Tile_Feature feature; + feature.set_type(vector_tile::Tile_GeomType_POINT); + // MoveTo(1,1) + feature.add_geometry((((1 << 29) - 1u) << 3u) | 1u); + feature.add_geometry(protozero::encode_zigzag32(1)); + feature.add_geometry(protozero::encode_zigzag32(1)); + + std::string feature_string = feature.SerializeAsString(); + mapnik::vector_tile_impl::GeometryPBF geoms = feature_to_pbf_geometry(feature_string); + + SECTION("VT Spec v1") + { + CHECK_THROWS(mapnik::vector_tile_impl::decode_geometry<double>(geoms, vector_tile::Tile_GeomType_POINT, 1, 0.0, 0.0, 1.0, 1.0)); + } + + SECTION("VT Spec v2") + { + CHECK_THROWS(mapnik::vector_tile_impl::decode_geometry<double>(geoms, vector_tile::Tile_GeomType_POINT, 2, 0.0, 0.0, 1.0, 1.0)); + } +} diff --git a/test/unit/decoding/polygon.cpp b/test/unit/decoding/polygon.cpp index 643daf3..2322ea3 100644 --- a/test/unit/decoding/polygon.cpp +++ b/test/unit/decoding/polygon.cpp @@ -1619,3 +1619,43 @@ TEST_CASE("decode polygon -- has invalid command first") CHECK_THROWS(mapnik::vector_tile_impl::decode_geometry<double>(geoms, vector_tile::Tile_GeomType_POLYGON, 2, 0.0, 0.0, 1.0, 1.0)); } } + +TEST_CASE("decode malicious polygon") +{ + vector_tile::Tile_Feature feature; + feature.set_type(vector_tile::Tile_GeomType_POLYGON); + // MoveTo(0,0) + feature.add_geometry(9); // move_to | (1 << 3) + feature.add_geometry(protozero::encode_zigzag32(0)); + feature.add_geometry(protozero::encode_zigzag32(0)); + + // We are going to massively increase the number of + // command counts here to produce a very large + // LineTo command. + + // LineTo(0,10) + feature.add_geometry((((1 << 29) - 1u) << 3u) | 2u); + feature.add_geometry(protozero::encode_zigzag32(0)); + feature.add_geometry(protozero::encode_zigzag32(10)); + // LineTo(-10,10) + feature.add_geometry(protozero::encode_zigzag32(-10)); + feature.add_geometry(protozero::encode_zigzag32(0)); + // LineTo(-10,0) + feature.add_geometry(protozero::encode_zigzag32(0)); + feature.add_geometry(protozero::encode_zigzag32(-10)); + // Close + feature.add_geometry(15); + + std::string feature_string = feature.SerializeAsString(); + mapnik::vector_tile_impl::GeometryPBF geoms = feature_to_pbf_geometry(feature_string); + + SECTION("VT Spec v1") + { + REQUIRE_THROWS(mapnik::vector_tile_impl::decode_geometry<std::int64_t>(geoms, feature.type(), 1, 0.0, 0.0, 1.0, 1.0)); + } + + SECTION("VT Spec v2") + { + REQUIRE_THROWS(mapnik::vector_tile_impl::decode_geometry<std::int64_t>(geoms, feature.type(), 2, 0.0, 0.0, 1.0, 1.0)); + } +} diff --git a/test/vector_tile_pbf.cpp b/test/vector_tile_pbf.cpp index 188a103..fac62c7 100644 --- a/test/vector_tile_pbf.cpp +++ b/test/vector_tile_pbf.cpp @@ -509,11 +509,8 @@ TEST_CASE("pbf raster tile output -- should be able to overzoom raster") // now read back and render image at larger size // and zoomed in - double minx,miny,maxx,maxy; - mapnik::vector_tile_impl::spherical_mercator merc(256); // 2/0/1.png - merc.xyz(0,1,2,minx,miny,maxx,maxy); - mapnik::box2d<double> bbox(minx,miny,maxx,maxy); + mapnik::box2d<double> bbox = mapnik::vector_tile_impl::tile_mercator_bbox(0,1,2); mapnik::Map map2(256,256,"+init=epsg:3857"); map2.set_buffer_size(1024); mapnik::layer lyr2("layer",map2.srs()); diff --git a/test/vector_tile_projection.cpp b/test/vector_tile_projection.cpp index 3fcbcf4..aada025 100644 --- a/test/vector_tile_projection.cpp +++ b/test/vector_tile_projection.cpp @@ -54,10 +54,7 @@ std::vector<unsigned> bboxToXYZ(mapnik::box2d<double> const& bboxCoords) } TEST_CASE( "vector tile projection 1", "should support z/x/y to bbox conversion at 0/0/0" ) { - mapnik::vector_tile_impl::spherical_mercator merc(256); - double minx,miny,maxx,maxy; - merc.xyz(0,0,0,minx,miny,maxx,maxy); - mapnik::box2d<double> map_extent(minx,miny,maxx,maxy); + mapnik::box2d<double> map_extent = mapnik::vector_tile_impl::tile_mercator_bbox(0,0,0); mapnik::box2d<double> e(-20037508.342789,-20037508.342789,20037508.342789,20037508.342789); double epsilon = 0.000001; CHECK(std::fabs(map_extent.minx() - e.minx()) < epsilon); @@ -73,13 +70,10 @@ TEST_CASE( "vector tile projection 1", "should support z/x/y to bbox conversion } TEST_CASE( "vector tile projection 2", "should support z/x/y to bbox conversion up to z33" ) { - mapnik::vector_tile_impl::spherical_mercator merc(256); int x = 2145960701; int y = 1428172928; int z = 32; - double minx,miny,maxx,maxy; - merc.xyz(x,y,z,minx,miny,maxx,maxy); - mapnik::box2d<double> map_extent(minx,miny,maxx,maxy); + mapnik::box2d<double> map_extent = mapnik::vector_tile_impl::tile_mercator_bbox(x,y,z); mapnik::box2d<double> e(-14210.1492817168364127,6711666.7204630710184574,-14210.1399510249066225,6711666.7297937674447894); double epsilon = 0.00000001; CHECK(std::fabs(map_extent.minx() - e.minx()) < epsilon); @@ -95,13 +89,10 @@ TEST_CASE( "vector tile projection 2", "should support z/x/y to bbox conversion } TEST_CASE( "vector tile projection 3", "should support z/x/y to bbox conversion for z3" ) { - mapnik::vector_tile_impl::spherical_mercator merc(256); int x = 3; int y = 3; int z = 3; - double minx,miny,maxx,maxy; - merc.xyz(x,y,z,minx,miny,maxx,maxy); - mapnik::box2d<double> map_extent(minx,miny,maxx,maxy); + mapnik::box2d<double> map_extent = mapnik::vector_tile_impl::tile_mercator_bbox(x,y,z); mapnik::box2d<double> e(-5009377.085697311,0.0,0.0,5009377.085697311); double epsilon = 0.00000001; CHECK(std::fabs(map_extent.minx() - e.minx()) < epsilon); -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/mapnik-vector-tile.git _______________________________________________ Pkg-grass-devel mailing list Pkg-grass-devel@lists.alioth.debian.org http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-grass-devel