This is an automated email from the git hooks/post-receive script. sebastic pushed a commit to branch master in repository osmcoastline.
commit 6ce404e77652872dca08580ae0c7918578d7bbe1 Author: Bas Couwenberg <[email protected]> Date: Thu Mar 31 07:42:46 2016 +0200 Imported Upstream version 2.1.3 --- CHANGELOG.md | 21 ++++++++++++++++++- CMakeLists.txt | 2 +- include/gdalcpp.hpp | 4 ++++ src/osmcoastline.cpp | 40 +++++++---------------------------- src/osmcoastline_filter.cpp | 49 +++++++++++++++++++++++++++++++++++++------ src/osmcoastline_ways.cpp | 4 +++- src/output_database.cpp | 24 ++++++++++----------- src/verbose_output.hpp | 51 --------------------------------------------- 8 files changed, 90 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1068a7..aa6d6ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,24 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Fixed +## [2.1.3] - 2016-03-30 + +### Added + +- Add verbose option to osmcoastline_filter. +- osmcoastline_filter now shows memory used in verbose mode. + +### Changed + +- Optimized osmcoastline_filter program. +- Use more features from newest libosmium. + +### Fixed + +- Setting the sqlite output to unsynchronized speeds up writing to database. +- Now also works on GDAL 2. Fixes an error in the transaction handling. + + ## [2.1.2] - 2016-01-05 ### Added @@ -59,7 +77,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). - Added man pages -[unreleased]: https://github.com/osmcode/osmium-tool/compare/v2.1.2...HEAD +[unreleased]: https://github.com/osmcode/osmium-tool/compare/v2.1.3...HEAD +[2.1.3]: https://github.com/osmcode/osmium-tool/compare/v2.1.2...v2.1.3 [2.1.2]: https://github.com/osmcode/osmium-tool/compare/v2.1.1...v2.1.2 [2.1.1]: https://github.com/osmcode/osmium-tool/compare/v2.1.0...v2.1.1 [2.1.0]: https://github.com/osmcode/osmium-tool/compare/v2.0.1...v2.1.0 diff --git a/CMakeLists.txt b/CMakeLists.txt index 894b51a..e03ecd3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,7 @@ project(osmcoastline) set(OSMCOASTLINE_VERSION_MAJOR 2) set(OSMCOASTLINE_VERSION_MINOR 1) -set(OSMCOASTLINE_VERSION_PATCH 2) +set(OSMCOASTLINE_VERSION_PATCH 3) set(OSMCOASTLINE_VERSION ${OSMCOASTLINE_VERSION_MAJOR}.${OSMCOASTLINE_VERSION_MINOR}.${OSMCOASTLINE_VERSION_PATCH}) diff --git a/include/gdalcpp.hpp b/include/gdalcpp.hpp index 1502f2f..ac3b104 100644 --- a/include/gdalcpp.hpp +++ b/include/gdalcpp.hpp @@ -347,18 +347,22 @@ namespace gdalcpp { } Layer& start_transaction() { +#if GDAL_VERSION_MAJOR < 2 OGRErr result = m_layer->StartTransaction(); if (result != OGRERR_NONE) { throw gdal_error(std::string("starting transaction on layer '") + name() + "' failed", result, m_dataset.driver_name(), m_dataset.dataset_name(), name()); } +#endif return *this; } Layer& commit_transaction() { +#if GDAL_VERSION_MAJOR < 2 OGRErr result = m_layer->CommitTransaction(); if (result != OGRERR_NONE) { throw gdal_error(std::string("committing transaction on layer '") + name() + "' failed", result, m_dataset.driver_name(), m_dataset.dataset_name(), name()); } +#endif return *this; } diff --git a/src/osmcoastline.cpp b/src/osmcoastline.cpp index 262b81b..1ede948 100644 --- a/src/osmcoastline.cpp +++ b/src/osmcoastline.cpp @@ -25,6 +25,8 @@ #include <unistd.h> #include <osmium/io/any_input.hpp> +#include <osmium/util/memory.hpp> +#include <osmium/util/verbose_output.hpp> #include <osmium/visitor.hpp> #include "return_codes.hpp" @@ -38,7 +40,6 @@ #include "coastline_handlers.hpp" #include "srs.hpp" #include "util.hpp" -#include "verbose_output.hpp" // The global SRS object is used in many places to transform // from WGS84 to the output SRS etc. @@ -104,38 +105,10 @@ polygon_vector_type create_polygons(CoastlineRingCollection& coastline_rings, Ou /* ================================================== */ -std::pair<int, int> get_memory_usage() { - char filename[100]; - sprintf(filename, "/proc/%d/status", getpid()); - std::ifstream status_file(filename); - std::string line; - - int vmpeak = 0; - int vmsize = 0; - if (status_file.is_open()) { - while (! status_file.eof() ) { - std::getline(status_file, line); - if (line.substr(0, 6) == "VmPeak") { - int f = line.find_first_of("0123456789"); - int l = line.find_last_of("0123456789"); - vmpeak = atoi(line.substr(f, l-f+1).c_str()); - } - if (line.substr(0, 6) == "VmSize") { - int f = line.find_first_of("0123456789"); - int l = line.find_last_of("0123456789"); - vmsize = atoi(line.substr(f, l-f+1).c_str()); - } - } - status_file.close(); - } - - return std::make_pair(vmsize / 1024, vmpeak / 1024); -} - std::string memory_usage() { - std::pair<int, int> mem = get_memory_usage(); + osmium::MemoryUsage mem; std::ostringstream s; - s << "Memory used currently: " << mem.first << " MB (Peak was: " << mem.second << " MB).\n"; + s << "Memory used: current: " << mem.current() << " MBytes, peak: " << mem.peak() << " MBytes\n"; return s.str(); } @@ -152,11 +125,12 @@ int main(int argc, char *argv[]) { // The vout object is an output stream we can write to instead of // std::cerr. Nothing is written if we are not in verbose mode. // The running time will be prepended to output lines. - VerboseOutput vout(options.verbose); + osmium::util::VerboseOutput vout(options.verbose); debug = options.debug; CPLSetConfigOption("OGR_ENABLE_PARTIAL_REPROJECTION", "TRUE"); + CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "OFF"); vout << "Using SRS " << options.epsg << " for output. (Change with the --srs/s option.)\n"; if (!srs.set_output(options.epsg)) { std::cerr << "Setting up output transformation failed\n"; @@ -320,7 +294,7 @@ int main(int argc, char *argv[]) { vout << memory_usage(); vout << "Committing database transactions...\n"; - output_database.set_meta(vout.runtime(), get_memory_usage().second, stats); + output_database.set_meta(vout.runtime(), osmium::MemoryUsage{}.peak(), stats); output_database.commit(); vout << "All done.\n"; vout << memory_usage(); diff --git a/src/osmcoastline_filter.cpp b/src/osmcoastline_filter.cpp index 9a3f951..1700769 100644 --- a/src/osmcoastline_filter.cpp +++ b/src/osmcoastline_filter.cpp @@ -20,9 +20,10 @@ */ #include <algorithm> -#include <iostream> -#include <set> #include <getopt.h> +#include <iostream> +#include <string> +#include <vector> #include <osmium/io/any_input.hpp> #include <osmium/io/input_iterator.hpp> @@ -30,6 +31,8 @@ #include <osmium/io/pbf_output.hpp> #include <osmium/handler.hpp> #include <osmium/osm/entity_bits.hpp> +#include <osmium/util/memory.hpp> +#include <osmium/util/verbose_output.hpp> #include "return_codes.hpp" @@ -38,22 +41,25 @@ void print_help() { << "\nOptions:\n" << " -h, --help - This help message\n" << " -o, --output=OSMFILE - Where to write output (default: none)\n" + << " -v, --verbose - Verbose output\n" << " -V, --version - Show version and exit\n" << "\n"; } int main(int argc, char* argv[]) { std::string output_filename; + bool verbose = false; static struct option long_options[] = { {"help", no_argument, 0, 'h'}, {"output", required_argument, 0, 'o'}, + {"verbose", no_argument, 0, 'v'}, {"version", no_argument, 0, 'V'}, {0, 0, 0, 0} }; while (1) { - int c = getopt_long(argc, argv, "ho:V", long_options, 0); + int c = getopt_long(argc, argv, "ho:vV", long_options, 0); if (c == -1) break; @@ -64,6 +70,9 @@ int main(int argc, char* argv[]) { case 'o': output_filename = optarg; break; + case 'v': + verbose = true; + break; case 'V': std::cout << "osmcoastline_filter version " OSMCOASTLINE_VERSION "\n" << "Copyright (C) 2012-2016 Jochen Topf <[email protected]>\n" @@ -76,6 +85,11 @@ int main(int argc, char* argv[]) { } } + // The vout object is an output stream we can write to instead of + // std::cerr. Nothing is written if we are not in verbose mode. + // The running time will be prepended to output lines. + osmium::util::VerboseOutput vout(verbose); + if (output_filename.empty()) { std::cerr << "Missing -o/--output=OSMFILE option\n"; exit(return_code_cmdline); @@ -98,6 +112,7 @@ int main(int argc, char* argv[]) { std::vector<osmium::object_id_type> ids; + vout << "Reading ways (1st pass through input file)...\n"; { osmium::io::Reader reader(infile, osmium::osm_entity_bits::way); auto ways = osmium::io::make_input_iterator_range<const osmium::Way>(reader); @@ -113,17 +128,32 @@ int main(int argc, char* argv[]) { reader.close(); } + vout << "Preparing node ID list...\n"; std::sort(ids.begin(), ids.end()); auto last = std::unique(ids.begin(), ids.end()); + vout << "Reading nodes (2nd pass through input file)...\n"; { osmium::io::Reader reader(infile, osmium::osm_entity_bits::node); auto nodes = osmium::io::make_input_iterator_range<const osmium::Node>(reader); - std::copy_if(nodes.cbegin(), nodes.cend(), output_it, [&ids, &last](const osmium::Node& node){ + + auto first = ids.begin(); + std::copy_if(nodes.cbegin(), nodes.cend(), output_it, [&first, &last](const osmium::Node& node){ + while (*first < node.id() && first != last) { + ++first; + } + + if (node.id() == *first) { + if (first != last) { + ++first; + } + return true; + } + const char* natural = node.get_value_by_key("natural"); - return std::binary_search(ids.begin(), last, node.id()) || - (natural && !strcmp(natural, "coastline")); + return natural && !strcmp(natural, "coastline"); }); + reader.close(); } @@ -132,5 +162,12 @@ int main(int argc, char* argv[]) { std::cerr << "io error: " << e.what() << "'\n"; exit(return_code_fatal); } + + vout << "All done.\n"; + osmium::MemoryUsage mem; + if (mem.current() > 0) { + vout << "Memory used: current: " << mem.current() << " MBytes\n" + << " peak: " << mem.peak() << " MBytes\n"; + } } diff --git a/src/osmcoastline_ways.cpp b/src/osmcoastline_ways.cpp index cd38493..09c8af4 100644 --- a/src/osmcoastline_ways.cpp +++ b/src/osmcoastline_ways.cpp @@ -51,7 +51,7 @@ public: CoastlineWaysHandler(const std::string& db_filename) : m_length(0.0), - m_dataset("SQLite", db_filename, gdalcpp::SRS{}, {"SPATIALITE=TRUE", "OGR_SQLITE_SYNCHRONOUS=OFF", "INIT_WITH_EPSG=no" }), + m_dataset("SQLite", db_filename, gdalcpp::SRS{}, {"SPATIALITE=TRUE", "INIT_WITH_EPSG=no" }), m_layer_ways(m_dataset, "ways", wkbLineString) { m_layer_ways.add_field("way_id", OFTString, 10); @@ -110,6 +110,8 @@ int main(int argc, char* argv[]) { exit(return_code_cmdline); } + CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "OFF"); + std::string input_osm_filename { argv[1] }; std::string output_db_filename { "coastline-ways.db" }; diff --git a/src/output_database.cpp b/src/output_database.cpp index b1b385a..293da90 100644 --- a/src/output_database.cpp +++ b/src/output_database.cpp @@ -33,7 +33,7 @@ OutputDatabase::OutputDatabase(const std::string& outdb, SRS& srs, bool with_index) : m_with_index(with_index), m_srs(srs), - m_dataset("SQLite", outdb, gdalcpp::SRS(*srs.out()), { "SPATIALITE=TRUE", "OGR_SQLITE_SYNCHRONOUS=OFF", "INIT_WITH_EPSG=no" }), + m_dataset("SQLite", outdb, gdalcpp::SRS(*srs.out()), { "SPATIALITE=TRUE", "INIT_WITH_EPSG=no" }), m_layer_error_points(m_dataset, "error_points", wkbPoint, layer_options()), m_layer_error_lines(m_dataset, "error_lines", wkbLineString, layer_options()), m_layer_rings(m_dataset, "rings", wkbPolygon, layer_options()), @@ -43,11 +43,9 @@ OutputDatabase::OutputDatabase(const std::string& outdb, SRS& srs, bool with_ind m_layer_error_points.add_field("osm_id", OFTString, 10); m_layer_error_points.add_field("error", OFTString, 16); - m_layer_error_points.start_transaction(); m_layer_error_lines.add_field("osm_id", OFTString, 10); m_layer_error_lines.add_field("error", OFTString, 16); - m_layer_error_lines.start_transaction(); m_layer_rings.add_field("osm_id", OFTString, 10); m_layer_rings.add_field("nways", OFTInteger, 6); @@ -55,13 +53,6 @@ OutputDatabase::OutputDatabase(const std::string& outdb, SRS& srs, bool with_ind m_layer_rings.add_field("fixed", OFTInteger, 1); m_layer_rings.add_field("land", OFTInteger, 1); m_layer_rings.add_field("valid", OFTInteger, 1); - m_layer_rings.start_transaction(); - - m_layer_land_polygons.start_transaction(); - - m_layer_water_polygons.start_transaction(); - - m_layer_lines.start_transaction(); m_dataset.exec("CREATE TABLE options (overlap REAL, close_distance REAL, max_points_in_polygons INTEGER, split_large_polygons INTEGER)"); m_dataset.exec("CREATE TABLE meta (" @@ -76,6 +67,14 @@ OutputDatabase::OutputDatabase(const std::string& outdb, SRS& srs, bool with_ind "num_rings_turned_around INTEGER, " "num_land_polygons_before_split INTEGER, " "num_land_polygons_after_split INTEGER)"); + + m_dataset.start_transaction(); + m_layer_rings.start_transaction(); + m_layer_land_polygons.start_transaction(); + m_layer_water_polygons.start_transaction(); + m_layer_lines.start_transaction(); + m_layer_error_points.start_transaction(); + m_layer_error_lines.start_transaction(); } void OutputDatabase::set_options(const Options& options) { @@ -119,12 +118,13 @@ void OutputDatabase::set_meta(int runtime, int memory_usage, const Stats& stats) } void OutputDatabase::commit() { + m_layer_error_lines.commit_transaction(); + m_layer_error_points.commit_transaction(); m_layer_lines.commit_transaction(); m_layer_water_polygons.commit_transaction(); m_layer_land_polygons.commit_transaction(); m_layer_rings.commit_transaction(); - m_layer_error_lines.commit_transaction(); - m_layer_error_points.commit_transaction(); + m_dataset.commit_transaction(); } void OutputDatabase::add_error_point(std::unique_ptr<OGRPoint>&& point, const char* error, osmium::object_id_type id) { diff --git a/src/verbose_output.hpp b/src/verbose_output.hpp deleted file mode 100644 index 5284b0b..0000000 --- a/src/verbose_output.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef VERBOSE_OUTPUT_HPP -#define VERBOSE_OUTPUT_HPP - -#include <iostream> -#include <sstream> - -class VerboseOutput { - - time_t m_start; - bool m_verbose; - bool m_newline; - -public: - - VerboseOutput(bool verbose) : - m_start(time(nullptr)), - m_verbose(verbose), - m_newline(true) { - } - - int runtime() const { - return time(nullptr) - m_start; - } - - void start_line() { - if (m_newline) { - int elapsed = time(nullptr) - m_start; - char timestr[20]; - snprintf(timestr, sizeof(timestr)-1, "[%2d:%02d] ", elapsed / 60, elapsed % 60); - std::cerr << timestr; - m_newline = false; - } - } - - template<typename T> - friend VerboseOutput& operator<<(VerboseOutput& out, T t) { - if (out.m_verbose) { - std::ostringstream o; - o << t; - out.start_line(); - std::cerr << o.str(); - if (o.str()[o.str().size()-1] == '\n') { - out.m_newline = true; - } - } - return out; - } - -}; // class VerboseOutput - -#endif // VERBOSE_OUTPUT_HPP -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/osmcoastline.git _______________________________________________ Pkg-grass-devel mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-grass-devel
