From bf4f3f35adfb4485610b6ae7f5601c20f66afff0 Mon Sep 17 00:00:00 2001 From: AlanIWBFT Date: Tue, 13 Jan 2026 22:11:17 +0800 Subject: [PATCH] Fix build with Boost 1.89 * `boost::range::sort` conflicts with `boost::sort`. Implementations are moved into a dedicated .cpp file. * `boost::system` is a header only library now and the stub is removed --- CMakeLists.txt | 21 +++++- include/append_vector.h | 1 + include/output_object.h | 2 - include/tile_data.h | 90 +++-------------------- include/tile_data_base.h | 42 +++++++++++ src/osm_store.cpp | 1 - src/tile_data.cpp | 23 ++---- src/tile_sorting.cpp | 152 +++++++++++++++++++++++++++++++++++++++ src/tilemaker.cpp | 56 ++------------- 9 files changed, 234 insertions(+), 154 deletions(-) create mode 100644 include/tile_data_base.h create mode 100644 src/tile_sorting.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c5c2309d..c93b2598 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,7 +31,20 @@ IF ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") LINK_DIRECTORIES(${Boost_LIBRARY_DIRS}) ENDIF () -find_package(Boost 1.66 REQUIRED COMPONENTS system filesystem program_options) +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + set(Boost_USE_DEBUG_RUNTIME true) +else() + set(Boost_USE_DEBUG_RUNTIME false) +endif() + +set(BOOST_HAS_SYSTEM false) + +find_package(Boost 1.89 QUIET COMPONENTS filesystem program_options) + +if(NOT Boost_FOUND) + find_package(Boost 1.66 REQUIRED COMPONENTS system filesystem program_options) + set(BOOST_HAS_SYSTEM true) +endif() find_package(libshp REQUIRED) @@ -111,6 +124,7 @@ file(GLOB tilemaker_src_files src/tag_map.cpp src/tile_coordinates_set.cpp src/tile_data.cpp + src/tile_sorting.cpp src/tilemaker.cpp src/tile_worker.cpp src/visvalingam.cpp @@ -125,7 +139,10 @@ target_link_libraries(tilemaker shapelib::shp SQLite::SQLite3 Rapidjson::rapidjson - Boost::system Boost::filesystem Boost::program_options) + Boost::filesystem Boost::program_options) +if(BOOST_HAS_SYSTEM) + target_link_libraries(tilemaker Boost::system) +endif() include(CheckCxxAtomic) if(NOT HAVE_CXX11_ATOMIC) diff --git a/include/append_vector.h b/include/append_vector.h index 07531217..c742f704 100644 --- a/include/append_vector.h +++ b/include/append_vector.h @@ -4,6 +4,7 @@ #include "mmap_allocator.h" #include #include +#include // Tilemaker collects OutputObjects in a list that // - spills to disk diff --git a/include/output_object.h b/include/output_object.h index f1f7390f..ee76b4a4 100644 --- a/include/output_object.h +++ b/include/output_object.h @@ -6,10 +6,8 @@ #include #include #include -#include "geom.h" #include "coordinates.h" #include "attribute_store.h" -#include "osm_store.h" #include enum OutputGeometryType : unsigned int { POINT_, LINESTRING_, MULTILINESTRING_, POLYGON_ }; diff --git a/include/tile_data.h b/include/tile_data.h index 80ae3239..012cab93 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -6,51 +6,22 @@ #include #include #include -#include -#include "output_object.h" #include "append_vector.h" #include "clip_cache.h" #include "mmap_allocator.h" #include "tile_coordinates_set.h" - -#define TILE_DATA_ID_SIZE 34 +#include "tile_data_base.h" typedef std::vector SourceList; class TileBbox; -// We cluster output objects by z6 tile -#define CLUSTER_ZOOM 6 -#define CLUSTER_ZOOM_WIDTH (1 << CLUSTER_ZOOM) -#define CLUSTER_ZOOM_AREA (CLUSTER_ZOOM_WIDTH * CLUSTER_ZOOM_WIDTH) - -// TileDataSource indexes which tiles have objects in them. The indexed zoom -// is at most z14; we'll clamp to z14 if the base zoom is higher than z14. -// -// As a result, we need at most 15 bits to store an X/Y coordinate. For efficiency, -// we bucket the world into 4,096 z6 tiles, which each contain some number of -// z14 objects. This lets us use only 8 bits to store an X/Y coordinate. -// -// Because index zoom is lower than base zoom in the case where base zoom is -// z15+, we'll get false positives when looking up objects in the index, -// since, e.g., a single z14 tile covers 4 z15 tiles. -// -// This is OK: when writing the z15 tile, there's a clipping step that will filter -// out the false positives. -typedef uint8_t Z6Offset; - -struct OutputObjectXY { - OutputObject oo; - Z6Offset x; - Z6Offset y; -}; - -struct OutputObjectXYID { - OutputObject oo; - Z6Offset x; - Z6Offset y; - uint64_t id; -}; +template void sortOutputObjects( + const unsigned int indexZoom, + const size_t threadNum, + typename AppendVectorNS::AppendVector::Iterator begin, + typename AppendVectorNS::AppendVector::Iterator end +); template void finalizeObjects( const std::string& name, @@ -88,52 +59,7 @@ template void finalizeObjects( if (objectIt->oo.minZoom < CLUSTER_ZOOM) lowZoom[i].push_back(*objectIt); - // If the user is doing a a small extract, there are few populated - // entries in `object`. - // - // e.g. Colorado has ~9 z6 tiles, 1 of which has 95% of its output - // objects. - // - // This optimizes for the small extract case by doing: - // - for each vector in objects - // - do a multi-threaded sort of vector - // - // For small extracts, this ensures that all threads are used even if - // only a handful of entries in `objects` are non-empty. - // - // For a global extract, this will have some overhead of repeatedly - // setting up/tearing down threads. In that case, it would be - // better to assign chunks of `objects` to each thread. - // - // That's a future performance improvement, so deferring for now. - boost::sort::block_indirect_sort( - it->begin(), - it->end(), - [indexZoom](const OO& a, const OO& b) { - // Cluster by parent zoom, so that a subsequent search - // can find a contiguous range of entries for any tile - // at zoom 6 or higher. - const size_t aX = a.x; - const size_t aY = a.y; - const size_t bX = b.x; - const size_t bY = b.y; - for (size_t z = CLUSTER_ZOOM; z <= indexZoom; z++) { - const auto aXz = aX / (1 << (indexZoom - z)); - const auto bXz = bX / (1 << (indexZoom - z)); - if (aXz != bXz) - return aXz < bXz; - - const auto aYz = aY / (1 << (indexZoom - z)); - const auto bYz = bY / (1 << (indexZoom - z)); - - if (aYz != bYz) - return aYz < bYz; - } - - return false; - }, - threadNum - ); + sortOutputObjects(indexZoom, threadNum, it->begin(), it->end()); } std::cout << std::endl; diff --git a/include/tile_data_base.h b/include/tile_data_base.h new file mode 100644 index 00000000..837582f6 --- /dev/null +++ b/include/tile_data_base.h @@ -0,0 +1,42 @@ +#ifndef _TILE_DATA_BASE_H +#define _TILE_DATA_BASE_H + +#include +#include "output_object.h" + +#define TILE_DATA_ID_SIZE 34 + +// We cluster output objects by z6 tile +#define CLUSTER_ZOOM 6 +#define CLUSTER_ZOOM_WIDTH (1 << CLUSTER_ZOOM) +#define CLUSTER_ZOOM_AREA (CLUSTER_ZOOM_WIDTH * CLUSTER_ZOOM_WIDTH) + +// TileDataSource indexes which tiles have objects in them. The indexed zoom +// is at most z14; we'll clamp to z14 if the base zoom is higher than z14. +// +// As a result, we need at most 15 bits to store an X/Y coordinate. For efficiency, +// we bucket the world into 4,096 z6 tiles, which each contain some number of +// z14 objects. This lets us use only 8 bits to store an X/Y coordinate. +// +// Because index zoom is lower than base zoom in the case where base zoom is +// z15+, we'll get false positives when looking up objects in the index, +// since, e.g., a single z14 tile covers 4 z15 tiles. +// +// This is OK: when writing the z15 tile, there's a clipping step that will filter +// out the false positives. +typedef uint8_t Z6Offset; + +struct OutputObjectXY { + OutputObject oo; + Z6Offset x; + Z6Offset y; +}; + +struct OutputObjectXYID { + OutputObject oo; + Z6Offset x; + Z6Offset y; + uint64_t id; +}; + +#endif //_TILE_DATA_BASE_H \ No newline at end of file diff --git a/src/osm_store.cpp b/src/osm_store.cpp index 5b7366b7..5bd6ef61 100644 --- a/src/osm_store.cpp +++ b/src/osm_store.cpp @@ -6,7 +6,6 @@ #include #include -#include #include "node_store.h" #include "way_store.h" diff --git a/src/tile_data.cpp b/src/tile_data.cpp index db9ffaac..1233a029 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -394,6 +394,11 @@ void populateTilesAtZoom( } } +void sortOutputObjectIDs( + const std::vector& sortOrders, + std::vector& data +); + std::vector TileDataSource::getObjectsForTile( const std::vector& sortOrders, unsigned int zoom, @@ -402,23 +407,7 @@ std::vector TileDataSource::getObjectsForTile( std::vector data; collectObjectsForTile(zoom, coordinates, data); collectLargeObjectsForTile(zoom, coordinates, data); - - // Lexicographic comparison, with the order of: layer, geomType, attributes, and objectID. - // Note that attributes is preferred to objectID. - // It is to arrange objects with the identical attributes continuously. - // Such objects will be merged into one object, to reduce the size of output. - boost::sort::pdqsort(data.begin(), data.end(), [&sortOrders](const OutputObjectID& x, const OutputObjectID& y) -> bool { - if (x.oo.layer < y.oo.layer) return true; - if (x.oo.layer > y.oo.layer) return false; - if (x.oo.z_order < y.oo.z_order) return sortOrders[x.oo.layer]; - if (x.oo.z_order > y.oo.z_order) return !sortOrders[x.oo.layer]; - if (x.oo.geomType < y.oo.geomType) return true; - if (x.oo.geomType > y.oo.geomType) return false; - if (x.oo.attributes < y.oo.attributes) return true; - if (x.oo.attributes > y.oo.attributes) return false; - if (x.oo.objectID < y.oo.objectID) return true; - return false; - }); + sortOutputObjectIDs(sortOrders, data); data.erase(unique(data.begin(), data.end()), data.end()); return data; } diff --git a/src/tile_sorting.cpp b/src/tile_sorting.cpp new file mode 100644 index 00000000..1d9dc58a --- /dev/null +++ b/src/tile_sorting.cpp @@ -0,0 +1,152 @@ +#include +#include +#include +#include "tile_data_base.h" +#include "append_vector.h" +#include + +template void sortOutputObjects( + const unsigned int indexZoom, + const size_t threadNum, + typename AppendVectorNS::AppendVector::Iterator begin, + typename AppendVectorNS::AppendVector::Iterator end +) +{ + // If the user is doing a a small extract, there are few populated + // entries in `object`. + // + // e.g. Colorado has ~9 z6 tiles, 1 of which has 95% of its output + // objects. + // + // This optimizes for the small extract case by doing: + // - for each vector in objects + // - do a multi-threaded sort of vector + // + // For small extracts, this ensures that all threads are used even if + // only a handful of entries in `objects` are non-empty. + // + // For a global extract, this will have some overhead of repeatedly + // setting up/tearing down threads. In that case, it would be + // better to assign chunks of `objects` to each thread. + // + // That's a future performance improvement, so deferring for now. + boost::sort::block_indirect_sort( + begin, end, + [indexZoom](const OO& a, const OO& b) { + // Cluster by parent zoom, so that a subsequent search + // can find a contiguous range of entries for any tile + // at zoom 6 or higher. + const size_t aX = a.x; + const size_t aY = a.y; + const size_t bX = b.x; + const size_t bY = b.y; + for (size_t z = CLUSTER_ZOOM; z <= indexZoom; z++) { + const auto aXz = aX / (1 << (indexZoom - z)); + const auto bXz = bX / (1 << (indexZoom - z)); + if (aXz != bXz) + return aXz < bXz; + + const auto aYz = aY / (1 << (indexZoom - z)); + const auto bYz = bY / (1 << (indexZoom - z)); + + if (aYz != bYz) + return aYz < bYz; + } + + return false; + }, + threadNum + ); +} + +template void sortOutputObjects( + const unsigned int indexZoom, + const size_t threadNum, + typename AppendVectorNS::AppendVector::Iterator begin, + typename AppendVectorNS::AppendVector::Iterator end +); + +template void sortOutputObjects( + const unsigned int indexZoom, + const size_t threadNum, + typename AppendVectorNS::AppendVector::Iterator begin, + typename AppendVectorNS::AppendVector::Iterator end +); + +void sortOutputObjectIDs( + const std::vector& sortOrders, + std::vector& data +) { + // Lexicographic comparison, with the order of: layer, geomType, attributes, and objectID. + // Note that attributes is preferred to objectID. + // It is to arrange objects with the identical attributes continuously. + // Such objects will be merged into one object, to reduce the size of output. + boost::sort::pdqsort(data.begin(), data.end(), [&sortOrders](const OutputObjectID& x, const OutputObjectID& y) -> bool { + if (x.oo.layer < y.oo.layer) return true; + if (x.oo.layer > y.oo.layer) return false; + if (x.oo.z_order < y.oo.z_order) return sortOrders[x.oo.layer]; + if (x.oo.z_order > y.oo.z_order) return !sortOrders[x.oo.layer]; + if (x.oo.geomType < y.oo.geomType) return true; + if (x.oo.geomType > y.oo.geomType) return false; + if (x.oo.attributes < y.oo.attributes) return true; + if (x.oo.attributes > y.oo.attributes) return false; + if (x.oo.objectID < y.oo.objectID) return true; + return false; + }); +} + +void sortTileCoordinates( + const size_t baseZoom, + const size_t threadNum, + std::deque>& tileCoordinates +) +{ + boost::sort::block_indirect_sort( + tileCoordinates.begin(), tileCoordinates.end(), + [baseZoom](auto const &a, auto const &b) { + const auto aZoom = a.first; + const auto bZoom = b.first; + const auto aX = a.second.x; + const auto aY = a.second.y; + const auto bX = b.second.x; + const auto bY = b.second.y; + const bool aLowZoom = aZoom < CLUSTER_ZOOM; + const bool bLowZoom = bZoom < CLUSTER_ZOOM; + + // Breadth-first for z0..5 + if (aLowZoom != bLowZoom) + return aLowZoom; + + if (aLowZoom && bLowZoom) { + if (aZoom != bZoom) + return aZoom < bZoom; + + if (aX != bX) + return aX < bX; + + return aY < bY; + } + + for (size_t z = CLUSTER_ZOOM; z <= baseZoom; z++) { + // Translate both a and b to zoom z, compare. + // First, sanity check: can we translate it to this zoom? + if (aZoom < z || bZoom < z) { + return aZoom < bZoom; + } + + const auto aXz = aX / (1 << (aZoom - z)); + const auto aYz = aY / (1 << (aZoom - z)); + const auto bXz = bX / (1 << (bZoom - z)); + const auto bYz = bY / (1 << (bZoom - z)); + + if (aXz != bXz) + return aXz < bXz; + + if (aYz != bYz) + return aYz < bYz; + } + + return false; + }, + threadNum); +} \ No newline at end of file diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index bcbb4d32..6cbd920a 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include "rapidjson/document.h" #include "rapidjson/writer.h" @@ -75,6 +74,11 @@ namespace geom = boost::geometry; // Global verbose switch bool verbose = false; +void sortTileCoordinates( + const size_t baseZoom, + const size_t threadNum, + std::deque>& tileCoordinates +); /** *\brief The Main function is responsible for command line processing, loading data and starting worker threads. @@ -454,55 +458,7 @@ int main(const int argc, const char* argv[]) { std::cout << std::endl; // Cluster tiles: breadth-first for z0..z5, depth-first for z6 - const size_t baseZoom = config.baseZoom; - boost::sort::block_indirect_sort( - tileCoordinates.begin(), tileCoordinates.end(), - [baseZoom](auto const &a, auto const &b) { - const auto aZoom = a.first; - const auto bZoom = b.first; - const auto aX = a.second.x; - const auto aY = a.second.y; - const auto bX = b.second.x; - const auto bY = b.second.y; - const bool aLowZoom = aZoom < CLUSTER_ZOOM; - const bool bLowZoom = bZoom < CLUSTER_ZOOM; - - // Breadth-first for z0..5 - if (aLowZoom != bLowZoom) - return aLowZoom; - - if (aLowZoom && bLowZoom) { - if (aZoom != bZoom) - return aZoom < bZoom; - - if (aX != bX) - return aX < bX; - - return aY < bY; - } - - for (size_t z = CLUSTER_ZOOM; z <= baseZoom; z++) { - // Translate both a and b to zoom z, compare. - // First, sanity check: can we translate it to this zoom? - if (aZoom < z || bZoom < z) { - return aZoom < bZoom; - } - - const auto aXz = aX / (1 << (aZoom - z)); - const auto aYz = aY / (1 << (aZoom - z)); - const auto bXz = bX / (1 << (bZoom - z)); - const auto bYz = bY / (1 << (bZoom - z)); - - if (aXz != bXz) - return aXz < bXz; - - if (aYz != bYz) - return aYz < bYz; - } - - return false; - }, - options.threadNum); + sortTileCoordinates(config.baseZoom, options.threadNum, tileCoordinates); std::size_t batchSize = 0; for(std::size_t startIndex = 0; startIndex < tileCoordinates.size(); startIndex += batchSize) {