From cc6ebcef1b93293341c907251c6dd181b632ee63 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Mon, 11 Aug 2025 12:02:28 +0200 Subject: [PATCH] Process untagged ways/relations in stage 1b/c correctly When we introduced the "untagged" callbacks in bc87ea2c8b4ea0bcac09975627e78765fda8927e, we did not call the process_untagged_way/relation callbacks for untagged ways/relations in stage 1b and 1c, but called the normal process_way/relation callbacks. That's not correct. So this is changed in this commit. For stage2 processing the answer which callback to call is a bit more complicated: On first glance we should also call the "untagged" function for untagged objects. But if we call the "untagged" function in that case, we would have to implement that function and it would be called for the millions of untagged objects we are not interested in. So it is arguably better to call the normal processing functions here. After all we explicitly requested the processing function to be called with the select_relation_members() call. So we are keeping this behaviour. --- src/output-flex.cpp | 32 +++++--- src/output-flex.hpp | 2 + tests/bdd/flex/untagged.feature | 94 ++++++++++++++++++++++++ tests/data/test_output_flex_untagged.lua | 53 +++++++++++++ 4 files changed, 170 insertions(+), 11 deletions(-) create mode 100644 tests/bdd/flex/untagged.feature create mode 100644 tests/data/test_output_flex_untagged.lua diff --git a/src/output-flex.cpp b/src/output-flex.cpp index 4c9e052c7..d16624a82 100644 --- a/src/output-flex.cpp +++ b/src/output-flex.cpp @@ -835,7 +835,7 @@ void output_flex_t::get_mutex_and_call_lua_function( void output_flex_t::pending_way(osmid_t id) { - if (!m_process_way) { + if (!m_process_way && !m_process_untagged_way) { return; } @@ -844,8 +844,11 @@ void output_flex_t::pending_way(osmid_t id) } way_delete(id); - - get_mutex_and_call_lua_function(m_process_way, m_way_cache.get()); + auto const &func = m_way_cache.get().tags().empty() ? m_process_untagged_way + : m_process_way; + if (func) { + get_mutex_and_call_lua_function(func, m_way_cache.get()); + } } void output_flex_t::select_relation_members() @@ -892,9 +895,20 @@ void output_flex_t::select_relation_members(osmid_t id) select_relation_members(); } +void output_flex_t::process_relation() +{ + auto const &func = m_relation_cache.get().tags().empty() + ? m_process_untagged_relation + : m_process_relation; + if (func) { + get_mutex_and_call_lua_function(func, m_relation_cache.get()); + } +} + void output_flex_t::pending_relation(osmid_t id) { - if (!m_process_relation && !m_select_relation_members) { + if (!m_process_relation && !m_process_untagged_relation && + !m_select_relation_members) { return; } @@ -904,16 +918,12 @@ void output_flex_t::pending_relation(osmid_t id) select_relation_members(); delete_from_tables(osmium::item_type::relation, id); - - if (m_process_relation) { - get_mutex_and_call_lua_function(m_process_relation, - m_relation_cache.get()); - } + process_relation(); } void output_flex_t::pending_relation_stage1c(osmid_t id) { - if (!m_process_relation) { + if (!m_process_relation && !m_process_untagged_relation) { return; } @@ -922,7 +932,7 @@ void output_flex_t::pending_relation_stage1c(osmid_t id) } m_disable_insert = true; - get_mutex_and_call_lua_function(m_process_relation, m_relation_cache.get()); + process_relation(); m_disable_insert = false; } diff --git a/src/output-flex.hpp b/src/output-flex.hpp index e9d7b6330..1cda3068c 100644 --- a/src/output-flex.hpp +++ b/src/output-flex.hpp @@ -196,6 +196,8 @@ class output_flex_t : public output_t void get_mutex_and_call_lua_function(prepared_lua_function_t func, osmium::OSMObject const &object); + void process_relation(); + void init_lua(std::string const &filename, properties_t const &properties); void check_context_and_state(char const *name, char const *context, diff --git a/tests/bdd/flex/untagged.feature b/tests/bdd/flex/untagged.feature new file mode 100644 index 000000000..86c641a0b --- /dev/null +++ b/tests/bdd/flex/untagged.feature @@ -0,0 +1,94 @@ +Feature: Adding untagged objects to a flex database + + Scenario: Import with normal and "untagged" callbacks + Given the style file 'test_output_flex_untagged.lua' + And the OSM data + """ + n11 v1 dV x1 y1 + n12 v1 dV x2 y2 + n13 v1 dV x3 y3 + n14 v1 dV Tamenity=restaurant x4 y4 + w20 v1 dV Thighway=primary Nn11,n12 + w21 v1 dV Nn13,n14 + r30 v1 dV Mn11@,w20@ + r31 v1 dV Ttype=route Mw20@ + """ + When running osm2pgsql flex + Then table osm2pgsql_test_nodes contains exactly + | node_id | tagged | tags | + | 11 | False | {} | + | 12 | False | {} | + | 13 | False | {} | + | 14 | True | {'amenity': 'restaurant'} | + + Then table osm2pgsql_test_ways contains exactly + | way_id | tagged | tags | + | 20 | True | {'highway': 'primary'} | + | 21 | False | {} | + + Then table osm2pgsql_test_relations contains exactly + | relation_id | tagged | tags | + | 30 | False | {} | + | 31 | True | {'type': 'route'} | + + Scenario: Import then update with normal and "untagged" callbacks + Given the style file 'test_output_flex_untagged.lua' + And the OSM data + """ + n11 v1 dV x1 y1 + n12 v1 dV x2 y2 + n13 v1 dV x3 y3 + n14 v1 dV Tamenity=restaurant x4 y4 + w20 v1 dV Thighway=primary Nn11,n12 + w21 v1 dV Nn13,n14 + w22 v1 dV Nn11,n12 + r30 v1 dV Mn11@,w20@ + r31 v1 dV Ttype=route Mw20@ + """ + When running osm2pgsql flex with parameters + | --slim | + Then table osm2pgsql_test_nodes contains exactly + | node_id | tagged | tags | + | 11 | False | {} | + | 12 | False | {} | + | 13 | False | {} | + | 14 | True | {'amenity': 'restaurant'} | + + Then table osm2pgsql_test_ways contains exactly + | way_id | tagged | tags | + | 20 | True | {'highway': 'primary'} | + | 21 | False | {} | + | 22 | False | {} | + + Then table osm2pgsql_test_relations contains exactly + | relation_id | tagged | tags | + | 30 | False | {} | + | 31 | True | {'type': 'route'} | + + Given the style file 'test_output_flex_untagged.lua' + And the OSM data + """ + n11 v2 dV Tnatural=tree x1 y1 + n14 v2 dV x4 y4 + w21 v2 dV Nn14,n13 + """ + When running osm2pgsql flex with parameters + | --slim | -a | + Then table osm2pgsql_test_nodes contains exactly + | node_id | tagged | tags | + | 11 | True | {'natural': 'tree'} | + | 12 | False | {} | + | 13 | False | {} | + | 14 | False | {} | + + Then table osm2pgsql_test_ways contains exactly + | way_id | tagged | tags | + | 20 | True | {'highway': 'primary'} | + | 21 | False | {} | + | 22 | False | {} | + + Then table osm2pgsql_test_relations contains exactly + | relation_id | tagged | tags | + | 30 | False | {} | + | 31 | True | {'type': 'route'} | + diff --git a/tests/data/test_output_flex_untagged.lua b/tests/data/test_output_flex_untagged.lua new file mode 100644 index 000000000..3192e55fc --- /dev/null +++ b/tests/data/test_output_flex_untagged.lua @@ -0,0 +1,53 @@ + +local tables = {} + +tables.nodes = osm2pgsql.define_node_table('osm2pgsql_test_nodes', { + { column = 'tags', type = 'jsonb' }, + { column = 'tagged', type = 'bool' }, + { column = 'geom', type = 'point', not_null = true }, +}) + +tables.ways = osm2pgsql.define_way_table('osm2pgsql_test_ways', { + { column = 'tags', type = 'jsonb' }, + { column = 'tagged', type = 'bool' }, + { column = 'geom', type = 'linestring', not_null = true }, +}) + +tables.relations = osm2pgsql.define_relation_table('osm2pgsql_test_relations', { + { column = 'tags', type = 'jsonb' }, + { column = 'tagged', type = 'bool' }, + { column = 'geom', type = 'geometry', not_null = true }, +}) + +local function insert(dtable, object, tagged, geom) + tables[dtable]:insert({ + tags = object.tags, + tagged = tagged, + geom = geom, + }) +end + +function osm2pgsql.process_node(object) + insert('nodes', object, true, object:as_point()) +end + +function osm2pgsql.process_way(object) + insert('ways', object, true, object:as_linestring()) +end + +function osm2pgsql.process_relation(object) + insert('relations', object, true, object:as_geometrycollection()) +end + +function osm2pgsql.process_untagged_node(object) + insert('nodes', object, false, object:as_point()) +end + +function osm2pgsql.process_untagged_way(object) + insert('ways', object, false, object:as_linestring()) +end + +function osm2pgsql.process_untagged_relation(object) + insert('relations', object, false, object:as_geometrycollection()) +end +