Skip to content

Fix Boost 1.89+ and Lua 5.4 compatibility, update CI infrastructure #872

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
johncarmack1984 wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Fix Boost 1.89+ and Lua 5.4 compatibility, update CI infrastructure #872

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
949feee
Fix Boost 1.89+ compatibility by isolating boost::sort from boost::ge…
johncarmack1984 Jan 14, 2026
81d83fc
Update CI workflow and fix infrastructure issues
johncarmack1984 Jan 14, 2026
c67a3ef
Fix Lua 5.4 compatibility in kaguya.hpp
johncarmack1984 Jan 14, 2026
97c26b4
Replace boost::sort with std::sort in files using boost::geometry
johncarmack1984 Jan 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 15 additions & 15 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ jobs:
runs-on: windows-latest

steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4

- name: Enable vcpkg cache
uses: actions/cache@v2
uses: actions/cache@v4
with:
path: c:\vcpkg\installed
key: windows-vcpkg-x64-0 # Increase the number whenever dependencies are modified
Expand All @@ -36,7 +36,7 @@ jobs:

- name: Build openmaptiles-compatible mbtiles files of Liechtenstein
run: |
Invoke-WebRequest -Uri http://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf -OutFile ${{ env.AREA }}.osm.pbf
Invoke-WebRequest -Uri https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf -OutFile ${{ env.AREA }}.osm.pbf -MaximumRedirection 5
${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.pmtiles --verbose
${{ github.workspace }}\build\RelWithDebInfo\tilemaker.exe ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.mbtiles --store osm_store --verbose

Expand All @@ -62,10 +62,10 @@ jobs:
runs-on: ${{ matrix.os }}

steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4

- name: Enable vcpkg cache
uses: actions/cache@v2
uses: actions/cache@v4
with:
path: ${{ matrix.path }}
key: vcpkg-${{ matrix.triplet }}-0 # Increase the number whenever dependencies are modified
Expand All @@ -85,7 +85,7 @@ jobs:

- name: Build openmaptiles-compatible mbtiles files of Liechtenstein
run: |
curl http://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf -o ${{ env.AREA }}.osm.pbf
curl -L https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf -o ${{ env.AREA }}.osm.pbf
${{ github.workspace }}/build/${{ matrix.executable }} ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.pmtiles --verbose
${{ github.workspace }}/build/${{ matrix.executable }} ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.mbtiles --verbose --store /tmp/store

Expand All @@ -109,7 +109,7 @@ jobs:
runs-on: ${{ matrix.os }}

steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4

- name: Install Linux dependencies
if: ${{ matrix.os == 'ubuntu-22.04' }}
Expand All @@ -129,7 +129,7 @@ jobs:

- name: Build openmaptiles-compatible mbtiles files of Liechtenstein
run: |
curl http://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf -o ${{ env.AREA }}.osm.pbf
curl -L https://download.geofabrik.de/europe/${{ env.AREA }}-latest.osm.pbf -o ${{ env.AREA }}.osm.pbf
./tilemaker ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.pmtiles --verbose
./tilemaker ${{ env.AREA }}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${{ env.AREA }}.mbtiles --verbose --store /tmp/store

Expand All @@ -140,10 +140,10 @@ jobs:

steps:
- name: Check out repository
uses: actions/checkout@v2
uses: actions/checkout@v4

- name: Download PBF file
run: curl http://download.geofabrik.de/europe/${AREA}-latest.osm.pbf -o ${AREA}.osm.pbf
run: curl -L https://download.geofabrik.de/europe/${AREA}-latest.osm.pbf -o ${AREA}.osm.pbf

- name: Build openmaptiles-compatible mbtiles files of given area
uses: ./
Expand All @@ -163,31 +163,31 @@ jobs:

steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Log in to the Container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}

- name: Build Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
uses: docker/build-push-action@v6
if: ${{ github.ref != 'refs/heads/master'}}
with:
context: .
push: false
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
- name: Build and push Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
uses: docker/build-push-action@v6
if: ${{ github.ref == 'refs/heads/master'}}
with:
context: .
Expand Down
16 changes: 14 additions & 2 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,15 @@ IF ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
LINK_DIRECTORIES(${Boost_LIBRARY_DIRS})
ENDIF ()

find_package(Boost 1.66 REQUIRED COMPONENTS system filesystem program_options)
# Try to find Boost 1.89+ first (system is header-only, no need to link)
# Fall back to older Boost with system component if not found
find_package(Boost 1.89 QUIET COMPONENTS filesystem program_options)
if(NOT Boost_FOUND)
find_package(Boost 1.66 REQUIRED COMPONENTS system filesystem program_options)
set(BOOST_HAS_SYSTEM TRUE)
else()
set(BOOST_HAS_SYSTEM FALSE)
endif()

find_package(libshp REQUIRED)

Expand Down Expand Up @@ -111,6 +119,7 @@ file(GLOB tilemaker_src_files
src/tag_map.cpp
src/tile_coordinates_set.cpp
src/tile_data.cpp
src/tile_sorting.cpp
src/tilemaker.cpp
src/tile_worker.cpp
src/visvalingam.cpp
Expand All @@ -125,7 +134,10 @@ target_link_libraries(tilemaker
shapelib::shp
SQLite::SQLite3
Rapidjson::rapidjson
Boost::system Boost::filesystem Boost::program_options)
Boost::filesystem Boost::program_options)
if(BOOST_HAS_SYSTEM)
target_link_libraries(tilemaker Boost::system)
endif()

include(CheckCxxAtomic)
if(NOT HAVE_CXX11_ATOMIC)
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ COPY server ./server
RUN mkdir build && \
cd build && \
if [ -z "$BUILD_DEBUG" ]; then \
cmake -DCMAKE_BUILD_TYPE=Release ..; \
cmake -DCMAKE_BUILD_TYPE=Release -DBoost_USE_DEBUG_RUNTIME=OFF ..; \
else \
cmake -DCMAKE_BUILD_TYPE=Debug ..; \
fi; \
Expand Down
8 changes: 6 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,12 @@ prefix = /usr/local

MANPREFIX := /usr/share/man
TM_VERSION ?= $(shell git describe --tags --abbrev=0)
CXXFLAGS ?= -O3 -Wall -Wno-unknown-pragmas -Wno-sign-compare -std=c++14 -pthread -fPIE -DTM_VERSION=$(TM_VERSION) $(CONFIG)
# Suppress warnings from third-party libraries:
# -Wno-missing-template-arg-list-after-template-kw: Boost.Interprocess compatibility with newer compilers
# -Wno-deprecated-declarations: RapidJSON uses deprecated std::iterator
CXXFLAGS ?= -O3 -Wall -Wno-unknown-pragmas -Wno-sign-compare -Wno-missing-template-arg-list-after-template-kw -Wno-deprecated-declarations -std=c++17 -pthread -fPIE -DTM_VERSION=$(TM_VERSION) $(CONFIG)
CFLAGS ?= -O3 -Wall -Wno-unknown-pragmas -Wno-sign-compare -std=c99 -fPIE -DTM_VERSION=$(TM_VERSION) $(CONFIG)
LIB := -L$(PLATFORM_PATH)/lib $(LUA_LIBS) -lboost_program_options -lsqlite3 -lboost_filesystem -lboost_system -lshp -pthread
LIB := -L$(PLATFORM_PATH)/lib $(LUA_LIBS) -lboost_program_options -lsqlite3 -lboost_filesystem -lshp -pthread
INC := -I$(PLATFORM_PATH)/include -isystem ./include -I./src $(LUA_CFLAGS)

# Targets
Expand Down Expand Up @@ -129,6 +132,7 @@ tilemaker: \
src/tag_map.o \
src/tile_coordinates_set.o \
src/tile_data.o \
src/tile_sorting.o \
src/tilemaker.o \
src/tile_worker.o \
src/visvalingam.o \
Expand Down
11 changes: 10 additions & 1 deletion include/external/kaguya.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,15 @@ extern "C" {
#include <lualib.h>
}

// Lua 5.4 compatibility: Some builds removed LUA_GCSETPAUSE/LUA_GCSETSTEPMUL
// Check if old constants exist, otherwise define them using the new names
#if !defined(LUA_GCSETPAUSE) && defined(LUA_GCPPAUSE)
#define LUA_GCSETPAUSE LUA_GCPPAUSE
#endif
#if !defined(LUA_GCSETSTEPMUL) && defined(LUA_GCPSTEPMUL)
#define LUA_GCSETSTEPMUL LUA_GCPSTEPMUL
#endif

#ifndef KAGUYA_USE_CPP11
#if defined(__cpp_decltype) || __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)
#define KAGUYA_USE_CPP11 1
Expand Down Expand Up @@ -12495,7 +12504,7 @@ namespace kaguya

/// @brief sets arg as the new value for the pause of the collector. Returns the previous value for pause.
int steppause(int value) { return lua_gc(state_, LUA_GCSETPAUSE, value); }

/// @brief sets arg as the new value for the step multiplier of the collector. Returns the previous value for step.
int setstepmul(int value) { return lua_gc(state_, LUA_GCSETSTEPMUL, value); }

Expand Down
137 changes: 22 additions & 115 deletions include/tile_data.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,143 +1,50 @@
/*! \file */
/*! \file */
#ifndef _TILE_DATA_H
#define _TILE_DATA_H

#include <map>
#include <set>
#include <vector>
#include <memory>
#include <boost/sort/sort.hpp>
#include "output_object.h"
#include "tile_data_base.h"
#include "append_vector.h"
#include "clip_cache.h"
#include "mmap_allocator.h"
#include "tile_coordinates_set.h"

#define TILE_DATA_ID_SIZE 34

typedef std::vector<class TileDataSource *> SourceList;

class TileBbox;

// We cluster output objects by z6 tile
#define CLUSTER_ZOOM 6
#define CLUSTER_ZOOM_WIDTH (1 << CLUSTER_ZOOM)
#define CLUSTER_ZOOM_AREA (CLUSTER_ZOOM_WIDTH * CLUSTER_ZOOM_WIDTH)

// TileDataSource indexes which tiles have objects in them. The indexed zoom
// is at most z14; we'll clamp to z14 if the base zoom is higher than z14.
//
// As a result, we need at most 15 bits to store an X/Y coordinate. For efficiency,
// we bucket the world into 4,096 z6 tiles, which each contain some number of
// z14 objects. This lets us use only 8 bits to store an X/Y coordinate.
//
// Because index zoom is lower than base zoom in the case where base zoom is
// z15+, we'll get false positives when looking up objects in the index,
// since, e.g., a single z14 tile covers 4 z15 tiles.
//
// This is OK: when writing the z15 tile, there's a clipping step that will filter
// out the false positives.
typedef uint8_t Z6Offset;

struct OutputObjectXY {
OutputObject oo;
Z6Offset x;
Z6Offset y;
};

struct OutputObjectXYID {
OutputObject oo;
Z6Offset x;
Z6Offset y;
uint64_t id;
};

// Declaration only - implementation in tile_sorting.cpp to avoid
// boost::sort conflicts with boost::geometry in Boost 1.89+
template<typename OO> void finalizeObjects(
const std::string& name,
const size_t& threadNum,
const unsigned int& indexZoom,
typename std::vector<AppendVectorNS::AppendVector<OO>>::iterator begin,
typename std::vector<AppendVectorNS::AppendVector<OO>>::iterator end,
typename std::vector<std::vector<OO>>& lowZoom
) {
size_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1;
#ifdef CLOCK_MONOTONIC
timespec startTs, endTs;
clock_gettime(CLOCK_MONOTONIC, &startTs);
#endif

int i = -1;
for (auto it = begin; it != end; it++) {
i++;
if (it->size() > 0 || i % 50 == 0 || i == 4095) {
std::cout << "\r" << name << ": finalizing z6 tile " << (i + 1) << "/" << CLUSTER_ZOOM_AREA;

#ifdef CLOCK_MONOTONIC
clock_gettime(CLOCK_MONOTONIC, &endTs);
uint64_t elapsedNs = 1e9 * (endTs.tv_sec - startTs.tv_sec) + endTs.tv_nsec - startTs.tv_nsec;
std::cout << " (" << std::to_string((uint32_t)(elapsedNs / 1e6)) << " ms)";
#endif
std::cout << std::flush;
}
if (it->size() == 0)
continue;

// We track a separate copy of low zoom objects to avoid scanning large
// lists of objects that may be on slow disk storage.
for (auto objectIt = it->begin(); objectIt != it->end(); objectIt++)
if (objectIt->oo.minZoom < CLUSTER_ZOOM)
lowZoom[i].push_back(*objectIt);

// If the user is doing a a small extract, there are few populated
// entries in `object`.
//
// e.g. Colorado has ~9 z6 tiles, 1 of which has 95% of its output
// objects.
//
// This optimizes for the small extract case by doing:
// - for each vector in objects
// - do a multi-threaded sort of vector
//
// For small extracts, this ensures that all threads are used even if
// only a handful of entries in `objects` are non-empty.
//
// For a global extract, this will have some overhead of repeatedly
// setting up/tearing down threads. In that case, it would be
// better to assign chunks of `objects` to each thread.
//
// That's a future performance improvement, so deferring for now.
boost::sort::block_indirect_sort(
it->begin(),
it->end(),
[indexZoom](const OO& a, const OO& b) {
// Cluster by parent zoom, so that a subsequent search
// can find a contiguous range of entries for any tile
// at zoom 6 or higher.
const size_t aX = a.x;
const size_t aY = a.y;
const size_t bX = b.x;
const size_t bY = b.y;
for (size_t z = CLUSTER_ZOOM; z <= indexZoom; z++) {
const auto aXz = aX / (1 << (indexZoom - z));
const auto bXz = bX / (1 << (indexZoom - z));
if (aXz != bXz)
return aXz < bXz;

const auto aYz = aY / (1 << (indexZoom - z));
const auto bYz = bY / (1 << (indexZoom - z));

if (aYz != bYz)
return aYz < bYz;
}
);

return false;
},
threadNum
);
}
// Extern template declarations to prevent implicit instantiation
extern template void finalizeObjects<OutputObjectXY>(
const std::string& name,
const size_t& threadNum,
const unsigned int& indexZoom,
std::vector<AppendVectorNS::AppendVector<OutputObjectXY>>::iterator begin,
std::vector<AppendVectorNS::AppendVector<OutputObjectXY>>::iterator end,
std::vector<std::vector<OutputObjectXY>>& lowZoom
);

std::cout << std::endl;
}
extern template void finalizeObjects<OutputObjectXYID>(
const std::string& name,
const size_t& threadNum,
const unsigned int& indexZoom,
std::vector<AppendVectorNS::AppendVector<OutputObjectXYID>>::iterator begin,
std::vector<AppendVectorNS::AppendVector<OutputObjectXYID>>::iterator end,
std::vector<std::vector<OutputObjectXYID>>& lowZoom
);

template<typename OO> void collectTilesWithObjectsAtZoomTemplate(
const unsigned int& indexZoom,
Expand Down
Loading