From a2ddd01bd075ba6d75b33972f66fae8238ec151f Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 16 Sep 2025 12:11:39 +0200 Subject: [PATCH 1/2] allow BDD tests involving multigeometries --- tests/bdd/flex/geometry-collection.feature | 14 +-- tests/bdd/flex/geometry-linestring.feature | 6 +- .../bdd/flex/geometry-multilinestring.feature | 16 ++-- tests/bdd/flex/geometry-multipoint.feature | 10 +- tests/bdd/steps/steps_db.py | 93 ++++++++++++++++--- 5 files changed, 102 insertions(+), 37 deletions(-) diff --git a/tests/bdd/flex/geometry-collection.feature b/tests/bdd/flex/geometry-collection.feature index fbadebe1d..ff3704365 100644 --- a/tests/bdd/flex/geometry-collection.feature +++ b/tests/bdd/flex/geometry-collection.feature @@ -45,11 +45,11 @@ Feature: Create geometry collections from relations | 33 | node | ST_GeometryCollection | 1 | ST_Point | And table osm2pgsql_test_collection contains exactly - | osm_id | ST_AsText(ST_GeometryN(geom, 1)) | ST_AsText(ST_GeometryN(geom, 2)) | - | 30 | 10, 11, 12, 13, 10 | NULL | - | 31 | 10, 11, 12, 13, 10 | 14, 15, 16 | - | 32 | 17 | 14, 15, 16 | - | 33 | 17 | NULL | + | osm_id | ST_AsText(geom) | + | 30 | { 10, 11, 12, 13, 10 } | + | 31 | { 10, 11, 12, 13, 10; 14, 15, 16 } | + | 32 | { 17; 14, 15, 16 } | + | 33 | { 17 } | Examples: | param | @@ -105,6 +105,6 @@ Feature: Create geometry collections from relations When running osm2pgsql flex Then table osm2pgsql_test_collection contains exactly - | osm_id | name | ST_NumGeometries(geom) | ST_AsText(ST_GeometryN(geom, 1)) | ST_AsText(ST_GeometryN(geom, 2)) | - | 30 | three | 2 | 10, 11, 12, 13, 10 | 10, 11, 13 | + | osm_id | name | ST_AsText(geom) | + | 30 | three | { 10, 11, 12, 13, 10; 10, 11, 13 } | diff --git a/tests/bdd/flex/geometry-linestring.feature b/tests/bdd/flex/geometry-linestring.feature index ec3526c68..af4039f88 100644 --- a/tests/bdd/flex/geometry-linestring.feature +++ b/tests/bdd/flex/geometry-linestring.feature @@ -32,9 +32,9 @@ Feature: Creating linestring features from way When running osm2pgsql flex Then table osm2pgsql_test_lines contains exactly - | way_id | ST_AsText(sgeom) | ST_AsText(ST_GeometryN(mgeom, 1)) | ST_AsText(ST_GeometryN(xgeom, 1)) | - | 20 | 1, 2, 3 | 1, 2, 3 | 1, 2, 3 | - | 21 | 4, 5 | 4, 5 | 4, 5 | + | way_id | ST_AsText(sgeom) | ST_AsText(mgeom) | ST_AsText(xgeom) | + | 20 | 1, 2, 3 | [ 1, 2, 3 ] | [ 1, 2, 3 ] | + | 21 | 4, 5 | [ 4, 5 ] | [ 4, 5 ] | Scenario: Given the grid diff --git a/tests/bdd/flex/geometry-multilinestring.feature b/tests/bdd/flex/geometry-multilinestring.feature index be8fd8b2b..6b06aa8d0 100644 --- a/tests/bdd/flex/geometry-multilinestring.feature +++ b/tests/bdd/flex/geometry-multilinestring.feature @@ -41,11 +41,11 @@ Feature: Creating (multi)linestring features from way and relations When running osm2pgsql flex Then table osm2pgsql_test_lines contains exactly - | osm_type | osm_id | ST_GeometryType(geom) | ST_AsText(ST_GeometryN(geom, 1)) | ST_AsText(ST_GeometryN(geom, 2)) | - | W | 20 | ST_LineString | 1, 2, 3 | NULL | - | W | 21 | ST_LineString | 4, 5, 6 | NULL | - | R | 30 | ST_LineString | 1, 2, 3 | NULL | - | R | 31 | ST_MultiLineString | 1, 2, 3 | 4, 5, 6 | + | osm_type | osm_id | ST_AsText(geom) | + | W | 20 | 1, 2, 3 | + | W | 21 | 4, 5, 6 | + | R | 30 | 1, 2, 3 | + | R | 31 | [ 1, 2, 3; 4, 5, 6 ] | Scenario: Given the grid @@ -77,7 +77,7 @@ Feature: Creating (multi)linestring features from way and relations When running osm2pgsql flex Then table osm2pgsql_test_roads contains exactly - | relation_id | ST_GeometryType(geom) | ST_GeometryType(merged) | ST_AsText(ST_GeometryN(merged, 1)) | ST_AsText(ST_GeometryN(merged, 2)) | - | 30 | ST_MultiLineString | ST_MultiLineString | 1, 2, 3 | NULL | - | 31 | ST_MultiLineString | ST_MultiLineString | 1, 2 | 3, 4 | + | relation_id | ST_GeometryType(geom) | ST_AsText(merged) | + | 30 | ST_MultiLineString | [ 1, 2, 3 ] | + | 31 | ST_MultiLineString | [ 1, 2; 3, 4 ] | diff --git a/tests/bdd/flex/geometry-multipoint.feature b/tests/bdd/flex/geometry-multipoint.feature index 650128c31..3e98852c0 100644 --- a/tests/bdd/flex/geometry-multipoint.feature +++ b/tests/bdd/flex/geometry-multipoint.feature @@ -43,9 +43,9 @@ Feature: Creating (multi)point features from nodes and relations When running osm2pgsql flex Then table osm2pgsql_test_points contains exactly - | osm_type | osm_id | ST_GeometryType(geom) | ST_NumGeometries(geom) | ST_AsText(ST_GeometryN(geom, 1)) | ST_AsText(ST_GeometryN(geom, 2)) | - | N | 1 | ST_Point | 1 | 1 | NULL | - | N | 5 | ST_Point | 1 | 5 | NULL | - | R | 30 | ST_Point | 1 | 1 | NULL | - | R | 31 | ST_MultiPoint | 2 | 5 | 1 | + | osm_type | osm_id | ST_AsText(geom) | + | N | 1 | 1 | + | N | 5 | 5 | + | R | 30 | 1 | + | R | 31 | [ 5; 1 ] | diff --git a/tests/bdd/steps/steps_db.py b/tests/bdd/steps/steps_db.py index 5d7a99523..26159c2be 100644 --- a/tests/bdd/steps/steps_db.py +++ b/tests/bdd/steps/steps_db.py @@ -178,15 +178,43 @@ def __init__(self, value, props, factory): self.factory = factory def set_coordinates(self, value): - m = re.fullmatch(r'(POINT|LINESTRING|POLYGON)\((.*)\)', value) + if value.startswith('GEOMETRYCOLLECTION('): + geoms = [] + remain = value[19:-1] + while remain: + _, value, remain = self._parse_simple_wkt(remain) + remain = remain[1:] # delete comma + geoms.append(value) + self.geom_type = 'GEOMETRYCOLLECTION' + self.value = geoms + else: + self.geom_type, self.value, remain = self._parse_simple_wkt(value) + if remain: + raise RuntimeError('trailing content for geometry: ' + value) + + def _parse_simple_wkt(self, value): + m = re.fullmatch(r'(MULTI)?(POINT|LINESTRING|POLYGON)\(([^A-Z]*)\)(.*)', value) if not m: raise RuntimeError(f'Unparsable WKT: {value}') - if m[1] == 'POINT': - self.value = self._parse_wkt_coord(m[2]) - elif m[1] == 'LINESTRING': - self.value = self._parse_wkt_line(m[2]) - elif m[1] == 'POLYGON': - self.value = [self._parse_wkt_line(ln) for ln in m[2][1:-1].split('),(')] + geom_type = (m[1] or '') + m[2] + if m[1] == 'MULTI': + splitup = m[3][1:-1].split('),(') + if m[2] == 'POINT': + value = [self._parse_wkt_coord(c) for c in splitup] + elif m[2] == 'LINESTRING': + value = [self._parse_wkt_line(c) for c in splitup] + elif m[2] == 'POLYGON': + value = [[self._parse_wkt_line(ln) for ln in poly[1:-1].split('),(')] + for poly in splitup] + else: + if m[2] == 'POINT': + value = self._parse_wkt_coord(m[3]) + elif m[2] == 'LINESTRING': + value = self._parse_wkt_line(m[3]) + elif m[2] == 'POLYGON': + value = [self._parse_wkt_line(ln) for ln in m[3][1:-1].split('),(')] + + return geom_type, value, m[4] def _parse_wkt_coord(self, coord): return tuple(DBValueFloat(float(f.strip()), self.precision) for f in coord.split()) @@ -195,14 +223,51 @@ def _parse_wkt_line(self, coords): return [self._parse_wkt_coord(pt) for pt in coords.split(',')] def __eq__(self, other): - if other.find(',') < 0: - geom = self._parse_input_coord(other) - elif other.find('(') < 0: - geom = self._parse_input_line(other) + if other.startswith('[') and other.endswith(']'): + gtype = 'MULTI' + toparse = other[1:-1].split(';') + elif other.startswith('{') and other.endswith('}'): + gtype = 'GEOMETRYCOLLECTION' + toparse = other[1:-1].split(';') else: - geom = [self._parse_input_line(ln) for ln in other.strip()[1:-1].split('),(')] - - return self.value == geom + gtype = None + toparse = [other] + + geoms = [] + for sub in toparse: + sub = sub.strip() + if sub.find(',') < 0: + geoms.append(self._parse_input_coord(sub)) + if gtype is None: + gtype = 'POINT' + elif gtype.startswith('MULTI'): + if gtype == 'MULTI': + gtype = 'MULTIPOINT' + elif gtype != 'MULTIPOINT': + raise RuntimeError('MULTI* geometry with different geometry types is not supported.') + elif sub.find('(') < 0: + geoms.append(self._parse_input_line(sub)) + if gtype is None: + gtype = 'LINESTRING' + elif gtype.startswith('MULTI'): + if gtype == 'MULTI': + gtype = 'MULTILINESTRING' + elif gtype != 'MULTILINESTRING': + raise RuntimeError('MULTI* geometry with different geometry types is not supported.') + else: + geoms.append([self._parse_input_line(ln) for ln in sub.strip()[1:-1].split('),(')]) + if gtype is None: + gtype = 'POLYGON' + elif gtype.startswith('MULTI'): + if gtype == 'MULTI': + gtype = 'MULTIPOLYGON' + elif gtype != 'MULTIPOLYGON': + raise RuntimeError('MULTI* geometry with different geometry types is not supported.') + + if not gtype.startswith('MULTI') and gtype != 'GEOMETRYCOLLECTION': + geoms = geoms[0] + + return gtype == self.geom_type and self.value == geoms def _parse_input_coord(self, other): coords = other.split(' ') From b152dc82149cfa7505a5c5adb757d90dc072863e Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 16 Sep 2025 13:56:29 +0200 Subject: [PATCH 2/2] use PBF for testing Geofabrik doesn't see to have osm.bz2 files anymore. --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 459e474bc..763aa0bd3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -384,7 +384,7 @@ jobs: runs-on: windows-2025 env: - OSMURL: https://download.geofabrik.de/europe/monaco-latest.osm.bz2 + OSMURL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf steps: - uses: actions/checkout@v4 @@ -398,8 +398,9 @@ jobs: & $env:PGBIN\psql -d osm -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis;" shell: pwsh - name: Get test data - run: (new-object net.webclient).DownloadFile($env:OSMURL, "testfile.osm.bz2") + run: Invoke-WebRequest -Uri $env:OSMURL -OutFile "testfile.osm.pbf" + shell: pwsh - name: Execute osm2pgsql - run: ./osm2pgsql-bin/osm2pgsql --slim -d osm testfile.osm.bz2 + run: ./osm2pgsql-bin/osm2pgsql --slim -d osm testfile.osm.pbf shell: bash