From bb6b46bc3dd4b564de599737dd82c7e03f03245c Mon Sep 17 00:00:00 2001 From: strech345 Date: Sat, 21 Feb 2026 14:51:08 +0100 Subject: [PATCH 01/12] feat(): create map component, add area filtering to the job config --- package.json | 1 + ui/src/components/map/Map.jsx | 189 ++++++++++++++++++ ui/src/components/map/Map.less | 41 ++++ ui/src/components/map/MapDrawingExtension.js | 151 ++++++++++++++ ui/src/views/jobs/mutation/JobMutation.jsx | 10 + .../components/areaFilter/AreaFilter.jsx | 29 +++ .../components/areaFilter/AreaFilter.less | 16 ++ ui/src/views/listings/Map.jsx | 171 ++-------------- yarn.lock | 72 ++++++- 9 files changed, 523 insertions(+), 157 deletions(-) create mode 100644 ui/src/components/map/Map.jsx create mode 100644 ui/src/components/map/Map.less create mode 100644 ui/src/components/map/MapDrawingExtension.js create mode 100644 ui/src/views/jobs/mutation/components/areaFilter/AreaFilter.jsx create mode 100644 ui/src/views/jobs/mutation/components/areaFilter/AreaFilter.less diff --git a/package.json b/package.json index 5f2e37fc..72423e2d 100755 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "@douyinfe/semi-icons": "^2.91.0", "@douyinfe/semi-ui": "2.91.0", "@douyinfe/semi-ui-19": "^2.91.0", + "@mapbox/mapbox-gl-draw": "^1.5.1", "@sendgrid/mail": "8.1.6", "@vitejs/plugin-react": "5.1.4", "adm-zip": "^0.5.16", diff --git a/ui/src/components/map/Map.jsx b/ui/src/components/map/Map.jsx new file mode 100644 index 00000000..3a26615a --- /dev/null +++ b/ui/src/components/map/Map.jsx @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +import { useEffect, useRef } from 'react'; +import maplibregl from 'maplibre-gl'; +import 'maplibre-gl/dist/maplibre-gl.css'; +import '@mapbox/mapbox-gl-draw/dist/mapbox-gl-draw.css'; +import { fixMapboxDrawCompatibility, addDrawingControl } from './MapDrawingExtension.js'; +import './Map.less'; + +export const GERMANY_BOUNDS = [ + [5.866, 47.27], // Southwest coordinates + [15.042, 55.059], // Northeast coordinates +]; + +export const STYLES = { + STANDARD: 'https://tiles.openfreemap.org/styles/bright', + SATELLITE: { + version: 8, + sources: { + 'satellite-tiles': { + type: 'raster', + tiles: ['https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}'], + tileSize: 256, + attribution: + 'Tiles © Esri — Source: Esri, i-cubed, USDA, USGS, AEX, GeoEye, Getmapping, Aerogrid, IGN, IGP, UPR-EGP, and the GIS User Community', + }, + 'satellite-labels': { + type: 'raster', + tiles: [ + 'https://server.arcgisonline.com/ArcGIS/rest/services/Reference/World_Boundaries_and_Places/MapServer/tile/{z}/{y}/{x}', + ], + tileSize: 256, + attribution: '© Esri', + }, + }, + layers: [ + { + id: 'satellite-tiles', + type: 'raster', + source: 'satellite-tiles', + minzoom: 0, + maxzoom: 19, + }, + { + id: 'satellite-labels', + type: 'raster', + source: 'satellite-labels', + minzoom: 0, + maxzoom: 19, + }, + ], + }, +}; + +export default function Map({ + mapContainerRef, + style = 'STANDARD', + show3dBuildings = false, + onMapReady = null, + enableDrawing = false, +}) { + const mapRef = useRef(null); + + // Initialize map + useEffect(() => { + if (mapRef.current) return; + + mapRef.current = new maplibregl.Map({ + container: mapContainerRef.current, + style: STYLES[style], + center: [10.4515, 51.1657], // Center of Germany + zoom: 4, + maxBounds: GERMANY_BOUNDS, + antialias: true, + }); + + mapRef.current.addControl( + new maplibregl.NavigationControl({ + showCompass: true, + visualizePitch: true, + visualizeRoll: true, + }), + 'top-right', + ); + + mapRef.current.addControl( + new maplibregl.GeolocateControl({ + positionOptions: { + enableHighAccuracy: true, + }, + trackUserLocation: true, + }), + ); + + // Initialize drawing extension only if enabled + if (enableDrawing) { + fixMapboxDrawCompatibility(); + addDrawingControl(mapRef.current); + } + + // Call onMapReady callback if provided + if (onMapReady) { + onMapReady(mapRef.current); + } + + return () => { + mapRef.current.remove(); + mapRef.current = null; + }; + }, [mapContainerRef, onMapReady, enableDrawing]); + + // Handle style changes + useEffect(() => { + if (mapRef.current) { + mapRef.current.setStyle(STYLES[style]); + } + }, [style]); + + // Handle 3D buildings layer + useEffect(() => { + if (!mapRef.current) return; + + const add3dLayer = () => { + if (!mapRef.current || !mapRef.current.isStyleLoaded()) return; + if (show3dBuildings) { + if (!mapRef.current.getSource('openfreemap')) { + mapRef.current.addSource('openfreemap', { + type: 'vector', + url: 'https://tiles.openfreemap.org/planet', + }); + } + if (!mapRef.current.getLayer('3d-buildings')) { + const layers = mapRef.current.getStyle().layers; + let labelLayerId; + for (let i = 0; i < layers.length; i++) { + if (layers[i].type === 'symbol' && layers[i].layout?.['text-field']) { + labelLayerId = layers[i].id; + break; + } + } + mapRef.current.addLayer( + { + id: '3d-buildings', + source: 'openfreemap', + 'source-layer': 'building', + type: 'fill-extrusion', + minzoom: 15, + filter: ['!=', ['get', 'hide_3d'], true], + paint: { + 'fill-extrusion-color': [ + 'interpolate', + ['linear'], + ['get', 'render_height'], + 0, + 'lightgray', + 200, + 'royalblue', + 400, + 'lightblue', + ], + 'fill-extrusion-height': ['interpolate', ['linear'], ['zoom'], 15, 0, 16, ['get', 'render_height']], + 'fill-extrusion-base': ['case', ['>=', ['get', 'zoom'], 16], ['get', 'render_min_height'], 0], + 'fill-extrusion-opacity': 0.6, + }, + }, + labelLayerId, + ); + } + } else { + if (mapRef.current.getLayer('3d-buildings')) { + mapRef.current.removeLayer('3d-buildings'); + } + } + }; + + add3dLayer(); + }, [show3dBuildings, style]); + + // Handle pitch for 3D + useEffect(() => { + if (!mapRef.current) return; + mapRef.current.setPitch(show3dBuildings ? 45 : 0); + }, [show3dBuildings]); + + return
; +} diff --git a/ui/src/components/map/Map.less b/ui/src/components/map/Map.less new file mode 100644 index 00000000..b2f7a49c --- /dev/null +++ b/ui/src/components/map/Map.less @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +/* Fix Mapbox Draw cursors for MapLibre GL compatibility */ +.maplibregl-map.mouse-pointer .maplibregl-canvas-container.maplibregl-interactive { + cursor: pointer; +} + +.maplibregl-map.mouse-move .maplibregl-canvas-container.maplibregl-interactive { + cursor: move; +} + +.maplibregl-map.mouse-add .maplibregl-canvas-container.maplibregl-interactive { + cursor: crosshair; +} + +.maplibregl-map.mouse-move.mode-direct_select .maplibregl-canvas-container.maplibregl-interactive { + cursor: grab; + cursor: -moz-grab; + cursor: -webkit-grab; +} + +.maplibregl-map.mode-direct_select.feature-vertex.mouse-move .maplibregl-canvas-container.maplibregl-interactive { + cursor: move; +} + +.maplibregl-map.mode-direct_select.feature-midpoint.mouse-pointer .maplibregl-canvas-container.maplibregl-interactive { + cursor: cell; +} + +.maplibregl-map.mode-direct_select.feature-feature.mouse-move .maplibregl-canvas-container.maplibregl-interactive { + cursor: move; +} + +.maplibregl-map.mode-static.mouse-pointer .maplibregl-canvas-container.maplibregl-interactive { + cursor: grab; + cursor: -moz-grab; + cursor: -webkit-grab; +} diff --git a/ui/src/components/map/MapDrawingExtension.js b/ui/src/components/map/MapDrawingExtension.js new file mode 100644 index 00000000..2f06941a --- /dev/null +++ b/ui/src/components/map/MapDrawingExtension.js @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +import MapboxDraw from '@mapbox/mapbox-gl-draw'; + +const drawStyles = [ + { + id: 'gl-draw-polygon-fill-inactive', + type: 'fill', + filter: ['all', ['==', 'active', 'false'], ['==', '$type', 'Polygon'], ['!=', 'mode', 'static']], + paint: { 'fill-color': '#3bb2d0', 'fill-outline-color': '#3bb2d0', 'fill-opacity': 0.1 }, + }, + { + id: 'gl-draw-polygon-fill-active', + type: 'fill', + filter: ['all', ['==', 'active', 'true'], ['==', '$type', 'Polygon']], + paint: { 'fill-color': '#fbb03b', 'fill-outline-color': '#fbb03b', 'fill-opacity': 0.1 }, + }, + { + id: 'gl-draw-polygon-midpoint', + type: 'circle', + filter: ['all', ['==', '$type', 'Point'], ['==', 'meta', 'midpoint']], + paint: { 'circle-radius': 3, 'circle-color': '#fbb03b' }, + }, + { + id: 'gl-draw-polygon-stroke-inactive', + type: 'line', + filter: ['all', ['==', 'active', 'false'], ['==', '$type', 'Polygon'], ['!=', 'mode', 'static']], + layout: { 'line-cap': 'round', 'line-join': 'round' }, + paint: { 'line-color': '#3bb2d0', 'line-width': 2 }, + }, + { + id: 'gl-draw-polygon-stroke-active', + type: 'line', + filter: ['all', ['==', 'active', 'true'], ['==', '$type', 'Polygon']], + layout: { 'line-cap': 'round', 'line-join': 'round' }, + paint: { 'line-color': '#fbb03b', 'line-dasharray': [0.2, 2], 'line-width': 2 }, + }, + { + id: 'gl-draw-line-inactive', + type: 'line', + filter: ['all', ['==', 'active', 'false'], ['==', '$type', 'LineString'], ['!=', 'mode', 'static']], + layout: { 'line-cap': 'round', 'line-join': 'round' }, + paint: { 'line-color': '#3bb2d0', 'line-width': 2 }, + }, + { + id: 'gl-draw-line-active', + type: 'line', + filter: ['all', ['==', '$type', 'LineString'], ['==', 'active', 'true']], + layout: { 'line-cap': 'round', 'line-join': 'round' }, + paint: { 'line-color': '#fbb03b', 'line-dasharray': [0.2, 2], 'line-width': 2 }, + }, + { + id: 'gl-draw-polygon-and-line-vertex-stroke-inactive', + type: 'circle', + filter: ['all', ['==', 'meta', 'vertex'], ['==', '$type', 'Point'], ['!=', 'mode', 'static']], + paint: { 'circle-radius': 5, 'circle-color': '#fff' }, + }, + { + id: 'gl-draw-polygon-and-line-vertex-inactive', + type: 'circle', + filter: ['all', ['==', 'meta', 'vertex'], ['==', '$type', 'Point'], ['!=', 'mode', 'static']], + paint: { 'circle-radius': 3, 'circle-color': '#fbb03b' }, + }, + { + id: 'gl-draw-point-point-stroke-inactive', + type: 'circle', + filter: [ + 'all', + ['==', 'active', 'false'], + ['==', '$type', 'Point'], + ['==', 'meta', 'feature'], + ['!=', 'mode', 'static'], + ], + paint: { 'circle-radius': 5, 'circle-opacity': 1, 'circle-color': '#fff' }, + }, + { + id: 'gl-draw-point-inactive', + type: 'circle', + filter: [ + 'all', + ['==', 'active', 'false'], + ['==', '$type', 'Point'], + ['==', 'meta', 'feature'], + ['!=', 'mode', 'static'], + ], + paint: { 'circle-radius': 3, 'circle-color': '#3bb2d0' }, + }, + { + id: 'gl-draw-point-stroke-active', + type: 'circle', + filter: ['all', ['==', '$type', 'Point'], ['==', 'active', 'true'], ['!=', 'meta', 'midpoint']], + paint: { 'circle-radius': 7, 'circle-color': '#fff' }, + }, + { + id: 'gl-draw-point-active', + type: 'circle', + filter: ['all', ['==', '$type', 'Point'], ['!=', 'meta', 'midpoint'], ['==', 'active', 'true']], + paint: { 'circle-radius': 5, 'circle-color': '#fbb03b' }, + }, + { + id: 'gl-draw-polygon-fill-static', + type: 'fill', + filter: ['all', ['==', 'mode', 'static'], ['==', '$type', 'Polygon']], + paint: { 'fill-color': '#404040', 'fill-outline-color': '#404040', 'fill-opacity': 0.1 }, + }, + { + id: 'gl-draw-polygon-stroke-static', + type: 'line', + filter: ['all', ['==', 'mode', 'static'], ['==', '$type', 'Polygon']], + layout: { 'line-cap': 'round', 'line-join': 'round' }, + paint: { 'line-color': '#404040', 'line-width': 2 }, + }, + { + id: 'gl-draw-line-static', + type: 'line', + filter: ['all', ['==', 'mode', 'static'], ['==', '$type', 'LineString']], + layout: { 'line-cap': 'round', 'line-join': 'round' }, + paint: { 'line-color': '#404040', 'line-width': 2 }, + }, + { + id: 'gl-draw-point-static', + type: 'circle', + filter: ['all', ['==', 'mode', 'static'], ['==', '$type', 'Point']], + paint: { 'circle-radius': 5, 'circle-color': '#404040' }, + }, +]; + +export function fixMapboxDrawCompatibility() { + MapboxDraw.constants.classes.CANVAS = 'maplibregl-canvas'; + MapboxDraw.constants.classes.CONTROL_BASE = 'maplibregl-ctrl'; + MapboxDraw.constants.classes.CONTROL_PREFIX = 'maplibregl-ctrl-'; + MapboxDraw.constants.classes.CONTROL_GROUP = 'maplibregl-ctrl-group'; + MapboxDraw.constants.classes.ATTRIBUTION = 'maplibregl-ctrl-attrib'; +} + +export function addDrawingControl(map) { + const draw = new MapboxDraw({ + displayControlsDefault: false, + controls: { + polygon: true, + trash: true, + }, + styles: drawStyles, + }); + + map.addControl(draw, 'top-left'); + return draw; +} diff --git a/ui/src/views/jobs/mutation/JobMutation.jsx b/ui/src/views/jobs/mutation/JobMutation.jsx index 002d2499..dac7202b 100644 --- a/ui/src/views/jobs/mutation/JobMutation.jsx +++ b/ui/src/views/jobs/mutation/JobMutation.jsx @@ -9,6 +9,7 @@ import NotificationAdapterMutator from './components/notificationAdapter/Notific import NotificationAdapterTable from '../../../components/table/NotificationAdapterTable'; import ProviderTable from '../../../components/table/ProviderTable'; import ProviderMutator from './components/provider/ProviderMutator'; +import AreaFilter from './components/areaFilter/AreaFilter'; import Headline from '../../../components/headline/Headline'; import { useActions, useSelector } from '../../../services/state/store'; import { xhrPost } from '../../../services/xhr'; @@ -55,6 +56,7 @@ export default function JobMutator() { const [notificationAdapterData, setNotificationAdapterData] = useState(defaultNotificationAdapter); const [shareWithUsers, setShareWithUsers] = useState(defaultShareWithUsers); const [enabled, setEnabled] = useState(defaultEnabled); + const [areaFilterData, setAreaFilterData] = useState(sourceJob?.areaFilter || null); const navigate = useNavigate(); const actions = useActions(); @@ -76,6 +78,7 @@ export default function JobMutator() { shareWithUsers, name, blacklist, + areaFilter: areaFilterData, enabled, jobId: jobToBeEdit?.id || null, }); @@ -206,6 +209,13 @@ export default function JobMutator() { /> + + setAreaFilterData(data)} /> + + { + map.current = mapInstance; + }; + + return ( +
+ +
+ ); +} diff --git a/ui/src/views/jobs/mutation/components/areaFilter/AreaFilter.less b/ui/src/views/jobs/mutation/components/areaFilter/AreaFilter.less new file mode 100644 index 00000000..df2defad --- /dev/null +++ b/ui/src/views/jobs/mutation/components/areaFilter/AreaFilter.less @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +.areaFilter-container { + height: 500px; + border-radius: 4px; + overflow: hidden; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); + + .map-container { + height: 100%; + } +} + diff --git a/ui/src/views/listings/Map.jsx b/ui/src/views/listings/Map.jsx index 40f32389..0bd16e32 100644 --- a/ui/src/views/listings/Map.jsx +++ b/ui/src/views/listings/Map.jsx @@ -20,54 +20,10 @@ import './Map.less'; import { xhrDelete } from '../../services/xhr.js'; import { Link, useNavigate } from 'react-router-dom'; import ListingDeletionModal from '../../components/ListingDeletionModal.jsx'; +import Map from '../../components/map/Map.jsx'; const { Text } = Typography; -const GERMANY_BOUNDS = [ - [5.866, 47.27], // Southwest coordinates - [15.042, 55.059], // Northeast coordinates -]; - -const STYLES = { - STANDARD: 'https://tiles.openfreemap.org/styles/bright', - SATELLITE: { - version: 8, - sources: { - 'satellite-tiles': { - type: 'raster', - tiles: ['https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}'], - tileSize: 256, - attribution: - 'Tiles © Esri — Source: Esri, i-cubed, USDA, USGS, AEX, GeoEye, Getmapping, Aerogrid, IGN, IGP, UPR-EGP, and the GIS User Community', - }, - 'satellite-labels': { - type: 'raster', - tiles: [ - 'https://server.arcgisonline.com/ArcGIS/rest/services/Reference/World_Boundaries_and_Places/MapServer/tile/{z}/{y}/{x}', - ], - tileSize: 256, - attribution: '© Esri', - }, - }, - layers: [ - { - id: 'satellite-tiles', - type: 'raster', - source: 'satellite-tiles', - minzoom: 0, - maxzoom: 19, - }, - { - id: 'satellite-labels', - type: 'raster', - source: 'satellite-labels', - minzoom: 0, - maxzoom: 19, - }, - ], - }, -}; - export default function MapView() { const mapContainer = useRef(null); const map = useRef(null); @@ -136,117 +92,24 @@ export default function MapView() { }; }, [navigate]); + // Get map instance reference after MapComponent renders useEffect(() => { - if (map.current) return; - - map.current = new maplibregl.Map({ - container: mapContainer.current, - style: STYLES[style], - center: [10.4515, 51.1657], // Center of Germany - zoom: 4, - maxBounds: GERMANY_BOUNDS, - antialias: true, - }); - - map.current.addControl( - new maplibregl.NavigationControl({ - showCompass: true, - visualizePitch: true, - visualizeRoll: true, - }), - 'top-right', - ); - - map.current.addControl( - new maplibregl.GeolocateControl({ - positionOptions: { - enableHighAccuracy: true, - }, - trackUserLocation: true, - }), - ); - - return () => { - map.current.remove(); - }; - }, []); - - useEffect(() => { - if (map.current) { - map.current.setStyle(STYLES[style]); - } - }, [style]); - - useEffect(() => { - if (show3dBuildings && style !== 'STANDARD') { - setStyle('STANDARD'); - } - }, [show3dBuildings, style]); - - useEffect(() => { - if (!map.current) return; - - map.current.setPitch(show3dBuildings ? 45 : 0); - }, [show3dBuildings]); - - useEffect(() => { - if (!map.current) return; - - const add3dLayer = () => { - if (!map.current || !map.current.isStyleLoaded()) return; - if (show3dBuildings) { - if (!map.current.getSource('openfreemap')) { - map.current.addSource('openfreemap', { - type: 'vector', - url: 'https://tiles.openfreemap.org/planet', - }); - } - if (!map.current.getLayer('3d-buildings')) { - const layers = map.current.getStyle().layers; - let labelLayerId; - for (let i = 0; i < layers.length; i++) { - if (layers[i].type === 'symbol' && layers[i].layout?.['text-field']) { - labelLayerId = layers[i].id; - break; - } - } - map.current.addLayer( - { - id: '3d-buildings', - source: 'openfreemap', - 'source-layer': 'building', - type: 'fill-extrusion', - minzoom: 15, - filter: ['!=', ['get', 'hide_3d'], true], - paint: { - 'fill-extrusion-color': [ - 'interpolate', - ['linear'], - ['get', 'render_height'], - 0, - 'lightgray', - 200, - 'royalblue', - 400, - 'lightblue', - ], - 'fill-extrusion-height': ['interpolate', ['linear'], ['zoom'], 15, 0, 16, ['get', 'render_height']], - 'fill-extrusion-base': ['case', ['>=', ['get', 'zoom'], 16], ['get', 'render_min_height'], 0], - 'fill-extrusion-opacity': 0.6, - }, - }, - labelLayerId, - ); - } - } else { - if (map.current.getLayer('3d-buildings')) { - map.current.removeLayer('3d-buildings'); + if (mapContainer.current && !map.current) { + // Wait for MapComponent to initialize the map + const checkMapReady = () => { + if (mapContainer.current?.map) { + map.current = mapContainer.current.map; + } else { + setTimeout(checkMapReady, 100); } - } - }; + }; + checkMapReady(); + } + }, []); - add3dLayer(); - }, [show3dBuildings, style]); + const handleMapReady = (mapInstance) => { + map.current = mapInstance; + }; const setMapStyle = (value) => { setStyle(value); @@ -573,7 +436,7 @@ export default function MapView() { description="Keep in mind, only listings with proper adresses are being shown on this map." /> -
+ Date: Sun, 22 Feb 2026 15:06:52 +0100 Subject: [PATCH 02/12] feat(): filter listings by area filter --- lib/FredyPipelineExecutioner.js | 40 +++++++++- lib/api/routes/jobRouter.js | 12 ++- lib/services/jobs/jobExecutionService.js | 1 + lib/services/storage/jobStorage.js | 16 +++- .../migrations/sql/11.add-spatial-filter.js | 22 ++++++ package.json | 1 + ui/src/components/map/Map.jsx | 74 +++++++++++++++---- ui/src/components/map/MapDrawingExtension.js | 26 +++++++ ui/src/views/jobs/mutation/JobMutation.jsx | 16 ++-- .../components/areaFilter/AreaFilter.jsx | 14 +--- .../components/areaFilter/AreaFilter.less | 3 - yarn.lock | 32 ++++++++ 12 files changed, 216 insertions(+), 41 deletions(-) create mode 100644 lib/services/storage/migrations/sql/11.add-spatial-filter.js diff --git a/lib/FredyPipelineExecutioner.js b/lib/FredyPipelineExecutioner.js index 85878e9f..a05c9847 100755 --- a/lib/FredyPipelineExecutioner.js +++ b/lib/FredyPipelineExecutioner.js @@ -14,6 +14,7 @@ import { geocodeAddress } from './services/geocoding/geoCodingService.js'; import { distanceMeters } from './services/listings/distanceCalculator.js'; import { getUserSettings } from './services/storage/settingsStorage.js'; import { updateListingDistance } from './services/storage/listingsStorage.js'; +import booleanPointInPolygon from '@turf/boolean-point-in-polygon'; /** * @typedef {Object} Listing @@ -58,16 +59,17 @@ class FredyPipelineExecutioner { * @param {(raw:any)=>Listing} providerConfig.normalize Function to convert raw scraped data into a Listing shape. * @param {(listing:Listing)=>boolean} providerConfig.filter Function to filter out unwanted listings. * @param {(url:string, waitForSelector?:string)=>Promise|Promise} [providerConfig.getListings] Optional override to fetch listings. - * * @param {Object} notificationConfig Notification configuration passed to notification adapters. + * @param {Object} spatialFilter Optional spatial filter configuration. * @param {string} providerId The ID of the provider currently in use. * @param {string} jobKey Key of the job that is currently running (from within the config). * @param {SimilarityCache} similarityCache Cache instance for checking similar entries. * @param browser */ - constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache, browser) { + constructor(providerConfig, notificationConfig, spatialFilter, providerId, jobKey, similarityCache, browser) { this._providerConfig = providerConfig; this._notificationConfig = notificationConfig; + this._spatialFilter = spatialFilter; this._providerId = providerId; this._jobKey = jobKey; this._similarityCache = similarityCache; @@ -87,6 +89,7 @@ class FredyPipelineExecutioner { .then(this._filter.bind(this)) .then(this._findNew.bind(this)) .then(this._geocode.bind(this)) + .then(this._filterByArea.bind(this)) .then(this._save.bind(this)) .then(this._calculateDistance.bind(this)) .then(this._filterBySimilarListings.bind(this)) @@ -113,6 +116,39 @@ class FredyPipelineExecutioner { return newListings; } + /** + * Filter listings by area using the provider's area filter if available. + * Only filters if areaFilter is set on the provider AND the listing has coordinates. + * + * @param {Listing[]} newListings New listings to filter by area. + * @returns {Promise} Resolves with listings that are within the area (or not filtered if no area is set). + */ + _filterByArea(newListings) { + const spatialFilter = this._spatialFilter; + + const polygonFeatures = spatialFilter?.features?.filter((f) => f.geometry?.type === 'Polygon'); + // If no area filter is set, return all listings + if (!polygonFeatures?.length) { + return newListings; + } + + // Filter listings by area - keep only those within the polygon + const filteredListings = newListings.filter((listing) => { + // If listing doesn't have coordinates, keep it (don't filter out) + if (listing.latitude == null || listing.longitude == null) { + return true; + } + + // Check if the point is inside the polygons + const point = [listing.longitude, listing.latitude]; // GeoJSON format: [lon, lat] + const isInPolygon = polygonFeatures.some((feature) => booleanPointInPolygon(point, feature)); + + return isInPolygon; + }); + + return filteredListings; + } + /** * Fetch listings from the provider, using the default Extractor flow unless * a provider-specific getListings override is supplied. diff --git a/lib/api/routes/jobRouter.js b/lib/api/routes/jobRouter.js index 91130788..1798cf77 100644 --- a/lib/api/routes/jobRouter.js +++ b/lib/api/routes/jobRouter.js @@ -163,7 +163,16 @@ jobRouter.post('/:jobId/run', async (req, res) => { }); jobRouter.post('/', async (req, res) => { - const { provider, notificationAdapter, name, blacklist = [], jobId, enabled, shareWithUsers = [] } = req.body; + const { + provider, + notificationAdapter, + name, + blacklist = [], + jobId, + enabled, + shareWithUsers = [], + spatialFilter = null, + } = req.body; const settings = await getSettings(); try { let jobFromDb = jobStorage.getJob(jobId); @@ -187,6 +196,7 @@ jobRouter.post('/', async (req, res) => { provider, notificationAdapter, shareWithUsers, + spatialFilter, }); } catch (error) { res.send(new Error(error)); diff --git a/lib/services/jobs/jobExecutionService.js b/lib/services/jobs/jobExecutionService.js index b52702d9..471279c7 100644 --- a/lib/services/jobs/jobExecutionService.js +++ b/lib/services/jobs/jobExecutionService.js @@ -181,6 +181,7 @@ export function initJobExecutionService({ providers, settings, intervalMs }) { await new FredyPipelineExecutioner( matchedProvider.config, job.notificationAdapter, + job.spatialFilter, prov.id, job.id, similarityCache, diff --git a/lib/services/storage/jobStorage.js b/lib/services/storage/jobStorage.js index e4fe2532..bf88003f 100644 --- a/lib/services/storage/jobStorage.js +++ b/lib/services/storage/jobStorage.js @@ -30,6 +30,7 @@ export const upsertJob = ({ notificationAdapter, userId, shareWithUsers = [], + spatialFilter = null, }) => { const id = jobId || nanoid(); const existing = SqliteConnection.query(`SELECT id, user_id FROM jobs WHERE id = @id LIMIT 1`, { id })[0]; @@ -42,7 +43,8 @@ export const upsertJob = ({ blacklist = @blacklist, provider = @provider, notification_adapter = @notification_adapter, - shared_with_user = @shareWithUsers + shared_with_user = @shareWithUsers, + spatial_filter = @spatialFilter WHERE id = @id`, { id, @@ -52,12 +54,13 @@ export const upsertJob = ({ shareWithUsers: toJson(shareWithUsers ?? []), provider: toJson(provider ?? []), notification_adapter: toJson(notificationAdapter ?? []), + spatialFilter: spatialFilter ? toJson(spatialFilter) : null, }, ); } else { SqliteConnection.execute( - `INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user) - VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers)`, + `INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user, spatial_filter) + VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers, @spatialFilter)`, { id, user_id: ownerId, @@ -67,6 +70,7 @@ export const upsertJob = ({ provider: toJson(provider ?? []), shareWithUsers: toJson(shareWithUsers ?? []), notification_adapter: toJson(notificationAdapter ?? []), + spatialFilter: spatialFilter ? toJson(spatialFilter) : null, }, ); } @@ -87,6 +91,7 @@ export const getJob = (jobId) => { j.provider, j.shared_with_user, j.notification_adapter AS notificationAdapter, + j.spatial_filter AS spatialFilter, (SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings FROM jobs j WHERE j.id = @id @@ -101,6 +106,7 @@ export const getJob = (jobId) => { provider: fromJson(row.provider, []), shared_with_user: fromJson(row.shared_with_user, []), notificationAdapter: fromJson(row.notificationAdapter, []), + spatialFilter: fromJson(row.spatialFilter, null), }; }; @@ -150,6 +156,7 @@ export const getJobs = () => { j.provider, j.shared_with_user, j.notification_adapter AS notificationAdapter, + j.spatial_filter AS spatialFilter, (SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings FROM jobs j WHERE j.enabled = 1 @@ -162,6 +169,7 @@ export const getJobs = () => { provider: fromJson(row.provider, []), shared_with_user: fromJson(row.shared_with_user, []), notificationAdapter: fromJson(row.notificationAdapter, []), + spatialFilter: fromJson(row.spatialFilter, null), })); }; @@ -251,6 +259,7 @@ export const queryJobs = ({ j.provider, j.shared_with_user, j.notification_adapter AS notificationAdapter, + j.spatial_filter AS spatialFilter, (SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings FROM jobs j ${whereSql} @@ -266,6 +275,7 @@ export const queryJobs = ({ provider: fromJson(row.provider, []), shared_with_user: fromJson(row.shared_with_user, []), notificationAdapter: fromJson(row.notificationAdapter, []), + spatialFilter: fromJson(row.spatialFilter, null), })); return { totalNumber, page: safePage, result }; diff --git a/lib/services/storage/migrations/sql/11.add-spatial-filter.js b/lib/services/storage/migrations/sql/11.add-spatial-filter.js new file mode 100644 index 00000000..9b563487 --- /dev/null +++ b/lib/services/storage/migrations/sql/11.add-spatial-filter.js @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +/* + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +// Migration: Add spatial_filter column to jobs table for storing GeoJSON-based spatial filters + +export function up(db) { + db.exec(` + ALTER TABLE jobs ADD COLUMN spatial_filter JSONB DEFAULT NULL; + `); +} + +export function down(db) { + db.exec(` + ALTER TABLE jobs DROP COLUMN spatial_filter; + `); +} diff --git a/package.json b/package.json index 72423e2d..2c635a33 100755 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "body-parser": "2.2.2", "chart.js": "^4.5.1", "cheerio": "^1.2.0", + "@turf/boolean-point-in-polygon": "^7.0.0", "cookie-session": "2.1.1", "handlebars": "4.7.8", "lodash": "4.17.23", diff --git a/ui/src/components/map/Map.jsx b/ui/src/components/map/Map.jsx index 3a26615a..5ad9fddd 100644 --- a/ui/src/components/map/Map.jsx +++ b/ui/src/components/map/Map.jsx @@ -3,11 +3,11 @@ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause */ -import { useEffect, useRef } from 'react'; +import { useEffect, useRef, useImperativeHandle, forwardRef } from 'react'; import maplibregl from 'maplibre-gl'; import 'maplibre-gl/dist/maplibre-gl.css'; import '@mapbox/mapbox-gl-draw/dist/mapbox-gl-draw.css'; -import { fixMapboxDrawCompatibility, addDrawingControl } from './MapDrawingExtension.js'; +import { fixMapboxDrawCompatibility, addDrawingControl, setupAreaFilterEventListeners } from './MapDrawingExtension.js'; import './Map.less'; export const GERMANY_BOUNDS = [ @@ -55,18 +55,39 @@ export const STYLES = { }, }; -export default function Map({ - mapContainerRef, - style = 'STANDARD', - show3dBuildings = false, - onMapReady = null, - enableDrawing = false, -}) { +export default forwardRef(function Map( + { + style = 'STANDARD', + show3dBuildings = false, + onMapReady = null, + enableDrawing = false, + initialSpatialFilter = null, + onDrawingChange = null, + }, + ref, +) { + const mapContainerRef = useRef(null); const mapRef = useRef(null); + const drawRef = useRef(null); - // Initialize map + // Expose methods to parent via ref + useImperativeHandle(ref, () => ({ + getDrawingData: () => { + if (drawRef.current) { + return drawRef.current.getAll(); + } + return null; + }, + setDrawingData: (data) => { + if (drawRef.current && data) { + drawRef.current.set(data); + } + }, + })); + + // Initialize map - ONLY when container changes, never reinitialize useEffect(() => { - if (mapRef.current) return; + if (mapRef.current) return; // Map already exists, don't reinitialize mapRef.current = new maplibregl.Map({ container: mapContainerRef.current, @@ -98,7 +119,7 @@ export default function Map({ // Initialize drawing extension only if enabled if (enableDrawing) { fixMapboxDrawCompatibility(); - addDrawingControl(mapRef.current); + drawRef.current = addDrawingControl(mapRef.current); } // Call onMapReady callback if provided @@ -107,10 +128,31 @@ export default function Map({ } return () => { - mapRef.current.remove(); - mapRef.current = null; + if (mapRef.current) { + mapRef.current.remove(); + mapRef.current = null; + } }; - }, [mapContainerRef, onMapReady, enableDrawing]); + }, [mapContainerRef]); // ONLY depend on mapContainerRef - nothing else! + + // Load spatial filter and setup area filter event listeners + useEffect(() => { + if (!mapRef.current || !drawRef.current || !enableDrawing) return; + + // Load initial spatial filter if provided + if (initialSpatialFilter) { + try { + drawRef.current.set(initialSpatialFilter); + } catch (error) { + console.error('Error loading spatial filter:', error); + } + } + + // Setup drawing event listeners + const cleanup = setupAreaFilterEventListeners(mapRef.current, drawRef.current, onDrawingChange); + + return cleanup; + }, [initialSpatialFilter, onDrawingChange, enableDrawing]); // Handle style changes useEffect(() => { @@ -186,4 +228,4 @@ export default function Map({ }, [show3dBuildings]); return
; -} +}); diff --git a/ui/src/components/map/MapDrawingExtension.js b/ui/src/components/map/MapDrawingExtension.js index 2f06941a..3643092f 100644 --- a/ui/src/components/map/MapDrawingExtension.js +++ b/ui/src/components/map/MapDrawingExtension.js @@ -149,3 +149,29 @@ export function addDrawingControl(map) { map.addControl(draw, 'top-left'); return draw; } + +export function setupAreaFilterEventListeners(map, draw, onDrawingChange) { + if (!map || !draw) return () => {}; + + const handleDrawChange = () => { + if (draw) { + const data = draw.getAll(); + if (onDrawingChange) { + onDrawingChange(data); + } + } + }; + + map.on('draw.create', handleDrawChange); + map.on('draw.update', handleDrawChange); + map.on('draw.delete', handleDrawChange); + + // Return cleanup function + return () => { + if (map) { + map.off('draw.create', handleDrawChange); + map.off('draw.update', handleDrawChange); + map.off('draw.delete', handleDrawChange); + } + }; +} diff --git a/ui/src/views/jobs/mutation/JobMutation.jsx b/ui/src/views/jobs/mutation/JobMutation.jsx index dac7202b..ff3cf550 100644 --- a/ui/src/views/jobs/mutation/JobMutation.jsx +++ b/ui/src/views/jobs/mutation/JobMutation.jsx @@ -3,7 +3,7 @@ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause */ -import { Fragment, useState } from 'react'; +import { Fragment, useState, useCallback } from 'react'; import NotificationAdapterMutator from './components/notificationAdapter/NotificationAdapterMutator'; import NotificationAdapterTable from '../../../components/table/NotificationAdapterTable'; @@ -45,6 +45,7 @@ export default function JobMutator() { const defaultNotificationAdapter = sourceJob?.notificationAdapter || []; const defaultEnabled = sourceJob?.enabled ?? true; const defaultShareWithUsers = sourceJob?.shared_with_user ?? []; + const defaultSpatialFilter = sourceJob?.spatialFilter || null; const [providerToEdit, setProviderToEdit] = useState(null); const [providerCreationVisible, setProviderCreationVisibility] = useState(false); @@ -56,10 +57,15 @@ export default function JobMutator() { const [notificationAdapterData, setNotificationAdapterData] = useState(defaultNotificationAdapter); const [shareWithUsers, setShareWithUsers] = useState(defaultShareWithUsers); const [enabled, setEnabled] = useState(defaultEnabled); - const [areaFilterData, setAreaFilterData] = useState(sourceJob?.areaFilter || null); + const [spatialFilter, setSpatialFilter] = useState(defaultSpatialFilter); const navigate = useNavigate(); const actions = useActions(); + // Memoize the spatial filter change handler to prevent map reinitializations + const handleSpatialFilterChange = useCallback((data) => { + setSpatialFilter(data); + }, []); + const isSavingEnabled = () => { return Boolean(notificationAdapterData.length && providerData.length && name); }; @@ -78,7 +84,7 @@ export default function JobMutator() { shareWithUsers, name, blacklist, - areaFilter: areaFilterData, + spatialFilter, enabled, jobId: jobToBeEdit?.id || null, }); @@ -211,9 +217,9 @@ export default function JobMutator() { - setAreaFilterData(data)} /> + { - map.current = mapInstance; - }; - +export default function AreaFilter({ spatialFilter = null, onChange = null }) { return (
); diff --git a/ui/src/views/jobs/mutation/components/areaFilter/AreaFilter.less b/ui/src/views/jobs/mutation/components/areaFilter/AreaFilter.less index df2defad..a7ae74c9 100644 --- a/ui/src/views/jobs/mutation/components/areaFilter/AreaFilter.less +++ b/ui/src/views/jobs/mutation/components/areaFilter/AreaFilter.less @@ -5,9 +5,6 @@ .areaFilter-container { height: 500px; - border-radius: 4px; - overflow: hidden; - box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); .map-container { height: 100%; diff --git a/yarn.lock b/yarn.lock index e34873e4..10204c1f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1919,6 +1919,17 @@ resolved "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz" integrity sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA== +"@turf/boolean-point-in-polygon@^7.0.0": + version "7.3.4" + resolved "https://registry.yarnpkg.com/@turf/boolean-point-in-polygon/-/boolean-point-in-polygon-7.3.4.tgz#654a940939fecddf1887ca4c95bd5a2f07a42de8" + integrity sha512-v/4hfyY90Vz9cDgs2GwjQf+Lft8o7mNCLJOTz/iv8SHAIgMMX0czEoIaNVOJr7tBqPqwin1CGwsncrkf5C9n8Q== + dependencies: + "@turf/helpers" "7.3.4" + "@turf/invariant" "7.3.4" + "@types/geojson" "^7946.0.10" + point-in-polygon-hao "^1.1.0" + tslib "^2.8.1" + "@turf/clone@7.3.4": version "7.3.4" resolved "https://registry.yarnpkg.com/@turf/clone/-/clone-7.3.4.tgz#ae2d9ccd77730181aaa76874308140515e55ddaa" @@ -1936,6 +1947,15 @@ "@types/geojson" "^7946.0.10" tslib "^2.8.1" +"@turf/invariant@7.3.4": + version "7.3.4" + resolved "https://registry.yarnpkg.com/@turf/invariant/-/invariant-7.3.4.tgz#d81f448aa4fdda36047337a688517581e91c12f0" + integrity sha512-88Eo4va4rce9sNZs6XiMJowWkikM3cS2TBhaCKlU+GFHdNf8PFEpiU42VDU8q5tOF6/fu21Rvlke5odgOGW4AQ== + dependencies: + "@turf/helpers" "7.3.4" + "@types/geojson" "^7946.0.10" + tslib "^2.8.1" + "@turf/meta@7.3.4": version "7.3.4" resolved "https://registry.yarnpkg.com/@turf/meta/-/meta-7.3.4.tgz#8e917d29de9da96a0f95f3f16119ba9abde7dee6" @@ -6017,6 +6037,13 @@ pify@^4.0.1: resolved "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz" integrity sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g== +point-in-polygon-hao@^1.1.0: + version "1.2.4" + resolved "https://registry.yarnpkg.com/point-in-polygon-hao/-/point-in-polygon-hao-1.2.4.tgz#8662abdcc84bcca230cc3ecbb0b0ab1a306f1bd6" + integrity sha512-x2pcvXeqhRHlNRdhLs/tgFapAbSSe86wa/eqmj1G6pWftbEs5aVRJhRGM6FYSUERKu0PjekJzMq0gsI2XyiclQ== + dependencies: + robust-predicates "^3.0.2" + possible-typed-array-names@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz" @@ -6762,6 +6789,11 @@ rimraf@^3.0.2: dependencies: glob "^7.1.3" +robust-predicates@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/robust-predicates/-/robust-predicates-3.0.2.tgz#d5b28528c4824d20fc48df1928d41d9efa1ad771" + integrity sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg== + rollup@^4.43.0: version "4.49.0" resolved "https://registry.npmjs.org/rollup/-/rollup-4.49.0.tgz" From 2739fab10d5bf19870effb65c50daabb3909482c Mon Sep 17 00:00:00 2001 From: strech345 Date: Sun, 22 Feb 2026 15:33:51 +0100 Subject: [PATCH 03/12] chore(): cleanup --- lib/FredyPipelineExecutioner.js | 3 +- .../migrations/sql/11.add-spatial-filter.js | 5 --- ui/src/components/map/Map.jsx | 38 +++++-------------- 3 files changed, 11 insertions(+), 35 deletions(-) diff --git a/lib/FredyPipelineExecutioner.js b/lib/FredyPipelineExecutioner.js index a05c9847..9be233e3 100755 --- a/lib/FredyPipelineExecutioner.js +++ b/lib/FredyPipelineExecutioner.js @@ -124,9 +124,8 @@ class FredyPipelineExecutioner { * @returns {Promise} Resolves with listings that are within the area (or not filtered if no area is set). */ _filterByArea(newListings) { - const spatialFilter = this._spatialFilter; + const polygonFeatures = this._spatialFilter?.features?.filter((f) => f.geometry?.type === 'Polygon'); - const polygonFeatures = spatialFilter?.features?.filter((f) => f.geometry?.type === 'Polygon'); // If no area filter is set, return all listings if (!polygonFeatures?.length) { return newListings; diff --git a/lib/services/storage/migrations/sql/11.add-spatial-filter.js b/lib/services/storage/migrations/sql/11.add-spatial-filter.js index 9b563487..b14301d2 100644 --- a/lib/services/storage/migrations/sql/11.add-spatial-filter.js +++ b/lib/services/storage/migrations/sql/11.add-spatial-filter.js @@ -3,12 +3,7 @@ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause */ -/* - * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause - */ - // Migration: Add spatial_filter column to jobs table for storing GeoJSON-based spatial filters - export function up(db) { db.exec(` ALTER TABLE jobs ADD COLUMN spatial_filter JSONB DEFAULT NULL; diff --git a/ui/src/components/map/Map.jsx b/ui/src/components/map/Map.jsx index 5ad9fddd..9a6cd8ae 100644 --- a/ui/src/components/map/Map.jsx +++ b/ui/src/components/map/Map.jsx @@ -3,7 +3,7 @@ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause */ -import { useEffect, useRef, useImperativeHandle, forwardRef } from 'react'; +import { useEffect, useRef } from 'react'; import maplibregl from 'maplibre-gl'; import 'maplibre-gl/dist/maplibre-gl.css'; import '@mapbox/mapbox-gl-draw/dist/mapbox-gl-draw.css'; @@ -55,36 +55,18 @@ export const STYLES = { }, }; -export default forwardRef(function Map( - { - style = 'STANDARD', - show3dBuildings = false, - onMapReady = null, - enableDrawing = false, - initialSpatialFilter = null, - onDrawingChange = null, - }, - ref, -) { +export default function Map({ + style = 'STANDARD', + show3dBuildings = false, + onMapReady = null, + enableDrawing = false, + initialSpatialFilter = null, + onDrawingChange = null, +}) { const mapContainerRef = useRef(null); const mapRef = useRef(null); const drawRef = useRef(null); - // Expose methods to parent via ref - useImperativeHandle(ref, () => ({ - getDrawingData: () => { - if (drawRef.current) { - return drawRef.current.getAll(); - } - return null; - }, - setDrawingData: (data) => { - if (drawRef.current && data) { - drawRef.current.set(data); - } - }, - })); - // Initialize map - ONLY when container changes, never reinitialize useEffect(() => { if (mapRef.current) return; // Map already exists, don't reinitialize @@ -228,4 +210,4 @@ export default forwardRef(function Map( }, [show3dBuildings]); return
; -}); +} From ae975d922f3274c31297cde1f4263a8f8e21c834 Mon Sep 17 00:00:00 2001 From: strech345 Date: Sun, 1 Mar 2026 16:50:16 +0100 Subject: [PATCH 04/12] feat(): solve feedback --- lib/FredyPipelineExecutioner.js | 2 +- .../storage/migrations/sql/11.add-spatial-filter.js | 6 ------ ui/src/components/map/Map.less | 4 ++++ ui/src/views/jobs/mutation/JobMutation.jsx | 2 +- .../jobs/mutation/components/areaFilter/AreaFilter.jsx | 2 +- .../jobs/mutation/components/areaFilter/AreaFilter.less | 9 ++------- 6 files changed, 9 insertions(+), 16 deletions(-) diff --git a/lib/FredyPipelineExecutioner.js b/lib/FredyPipelineExecutioner.js index 9be233e3..27b0db3f 100755 --- a/lib/FredyPipelineExecutioner.js +++ b/lib/FredyPipelineExecutioner.js @@ -89,10 +89,10 @@ class FredyPipelineExecutioner { .then(this._filter.bind(this)) .then(this._findNew.bind(this)) .then(this._geocode.bind(this)) - .then(this._filterByArea.bind(this)) .then(this._save.bind(this)) .then(this._calculateDistance.bind(this)) .then(this._filterBySimilarListings.bind(this)) + .then(this._filterByArea.bind(this)) .then(this._notify.bind(this)) .catch(this._handleError.bind(this)); } diff --git a/lib/services/storage/migrations/sql/11.add-spatial-filter.js b/lib/services/storage/migrations/sql/11.add-spatial-filter.js index b14301d2..fc869ba8 100644 --- a/lib/services/storage/migrations/sql/11.add-spatial-filter.js +++ b/lib/services/storage/migrations/sql/11.add-spatial-filter.js @@ -9,9 +9,3 @@ export function up(db) { ALTER TABLE jobs ADD COLUMN spatial_filter JSONB DEFAULT NULL; `); } - -export function down(db) { - db.exec(` - ALTER TABLE jobs DROP COLUMN spatial_filter; - `); -} diff --git a/ui/src/components/map/Map.less b/ui/src/components/map/Map.less index b2f7a49c..03c47659 100644 --- a/ui/src/components/map/Map.less +++ b/ui/src/components/map/Map.less @@ -3,6 +3,10 @@ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause */ +.map-container { + height: 100%; +} + /* Fix Mapbox Draw cursors for MapLibre GL compatibility */ .maplibregl-map.mouse-pointer .maplibregl-canvas-container.maplibregl-interactive { cursor: pointer; diff --git a/ui/src/views/jobs/mutation/JobMutation.jsx b/ui/src/views/jobs/mutation/JobMutation.jsx index ff3cf550..c00d3f6f 100644 --- a/ui/src/views/jobs/mutation/JobMutation.jsx +++ b/ui/src/views/jobs/mutation/JobMutation.jsx @@ -217,7 +217,7 @@ export default function JobMutator() { diff --git a/ui/src/views/jobs/mutation/components/areaFilter/AreaFilter.jsx b/ui/src/views/jobs/mutation/components/areaFilter/AreaFilter.jsx index c519856c..087895b6 100644 --- a/ui/src/views/jobs/mutation/components/areaFilter/AreaFilter.jsx +++ b/ui/src/views/jobs/mutation/components/areaFilter/AreaFilter.jsx @@ -8,7 +8,7 @@ import './AreaFilter.less'; export default function AreaFilter({ spatialFilter = null, onChange = null }) { return ( -
+
Date: Sun, 1 Mar 2026 17:00:37 +0100 Subject: [PATCH 05/12] feat(): solve most providers --- test/provider/einsAImmobilien.test.js | 9 ++++++++- test/provider/immoscout.test.js | 2 +- test/provider/immoswp.test.js | 2 +- test/provider/immowelt.test.js | 2 +- test/provider/kleinanzeigen.test.js | 9 ++++++++- test/provider/mcMakler.test.js | 2 +- test/provider/neubauKompass.test.js | 9 ++++++++- test/provider/ohneMakler.test.js | 2 +- test/provider/regionalimmobilien24.test.js | 1 + test/provider/wgGesucht.test.js | 2 +- test/provider/wohnungsboerse.test.js | 9 ++++++++- 11 files changed, 39 insertions(+), 10 deletions(-) diff --git a/test/provider/einsAImmobilien.test.js b/test/provider/einsAImmobilien.test.js index e8a310c5..e0ebb02b 100644 --- a/test/provider/einsAImmobilien.test.js +++ b/test/provider/einsAImmobilien.test.js @@ -14,7 +14,14 @@ describe('#einsAImmobilien testsuite()', () => { it('should test einsAImmobilien provider', async () => { const Fredy = await mockFredy(); return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'einsAImmobilien', similarityCache); + const fredy = new Fredy( + provider.config, + null, + null, + provider.metaInformation.id, + 'einsAImmobilien', + similarityCache, + ); fredy.execute().then((listings) => { expect(listings).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/immoscout.test.js b/test/provider/immoscout.test.js index 42bae651..29b12e8c 100644 --- a/test/provider/immoscout.test.js +++ b/test/provider/immoscout.test.js @@ -14,7 +14,7 @@ describe('#immoscout provider testsuite()', () => { it('should test immoscout provider', async () => { const Fredy = await mockFredy(); return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, '', similarityCache); + const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, '', similarityCache); fredy.execute().then((listings) => { expect(listings).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/immoswp.test.js b/test/provider/immoswp.test.js index 38d209d3..8f34d3b1 100644 --- a/test/provider/immoswp.test.js +++ b/test/provider/immoswp.test.js @@ -14,7 +14,7 @@ describe('#immoswp testsuite()', () => { it('should test immoswp provider', async () => { const Fredy = await mockFredy(); return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoswp', similarityCache); + const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'immoswp', similarityCache); fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/immowelt.test.js b/test/provider/immowelt.test.js index 9120b5f3..9e5a0349 100644 --- a/test/provider/immowelt.test.js +++ b/test/provider/immowelt.test.js @@ -14,7 +14,7 @@ describe('#immowelt testsuite()', () => { const Fredy = await mockFredy(); provider.init(providerConfig.immowelt, [], []); - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache); + const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'immowelt', similarityCache); const listing = await fredy.execute(); expect(listing).to.be.a('array'); diff --git a/test/provider/kleinanzeigen.test.js b/test/provider/kleinanzeigen.test.js index be134e7b..628cadff 100644 --- a/test/provider/kleinanzeigen.test.js +++ b/test/provider/kleinanzeigen.test.js @@ -14,7 +14,14 @@ describe('#kleinanzeigen testsuite()', () => { const Fredy = await mockFredy(); provider.init(providerConfig.kleinanzeigen, [], []); return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'kleinanzeigen', similarityCache); + const fredy = new Fredy( + provider.config, + null, + null, + provider.metaInformation.id, + 'kleinanzeigen', + similarityCache, + ); fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/mcMakler.test.js b/test/provider/mcMakler.test.js index dc94fd43..d35414f9 100644 --- a/test/provider/mcMakler.test.js +++ b/test/provider/mcMakler.test.js @@ -14,7 +14,7 @@ describe('#mcMakler testsuite()', () => { const Fredy = await mockFredy(); provider.init(providerConfig.mcMakler, []); - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'mcMakler', similarityCache); + const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'mcMakler', similarityCache); const listing = await fredy.execute(); expect(listing).to.be.a('array'); diff --git a/test/provider/neubauKompass.test.js b/test/provider/neubauKompass.test.js index 7db3f5f1..1182bb29 100644 --- a/test/provider/neubauKompass.test.js +++ b/test/provider/neubauKompass.test.js @@ -14,7 +14,14 @@ describe('#neubauKompass testsuite()', () => { it('should test neubauKompass provider', async () => { const Fredy = await mockFredy(); return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache); + const fredy = new Fredy( + provider.config, + null, + null, + provider.metaInformation.id, + 'neubauKompass', + similarityCache, + ); fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/ohneMakler.test.js b/test/provider/ohneMakler.test.js index e21f7966..eec70f02 100644 --- a/test/provider/ohneMakler.test.js +++ b/test/provider/ohneMakler.test.js @@ -14,7 +14,7 @@ describe('#ohneMakler testsuite()', () => { const Fredy = await mockFredy(); provider.init(providerConfig.ohneMakler, []); - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'ohneMakler', similarityCache); + const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'ohneMakler', similarityCache); const listing = await fredy.execute(); expect(listing).to.be.a('array'); diff --git a/test/provider/regionalimmobilien24.test.js b/test/provider/regionalimmobilien24.test.js index a2db6568..2a4dfd97 100644 --- a/test/provider/regionalimmobilien24.test.js +++ b/test/provider/regionalimmobilien24.test.js @@ -17,6 +17,7 @@ describe('#regionalimmobilien24 testsuite()', () => { const fredy = new Fredy( provider.config, null, + null, provider.metaInformation.id, 'regionalimmobilien24', similarityCache, diff --git a/test/provider/wgGesucht.test.js b/test/provider/wgGesucht.test.js index d5c14ddf..ad1ca9c0 100644 --- a/test/provider/wgGesucht.test.js +++ b/test/provider/wgGesucht.test.js @@ -14,7 +14,7 @@ describe('#wgGesucht testsuite()', () => { it('should test wgGesucht provider', async () => { const Fredy = await mockFredy(); return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'wgGesucht', similarityCache); + const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'wgGesucht', similarityCache); fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/wohnungsboerse.test.js b/test/provider/wohnungsboerse.test.js index 4c4bc2b8..2f270720 100644 --- a/test/provider/wohnungsboerse.test.js +++ b/test/provider/wohnungsboerse.test.js @@ -14,7 +14,14 @@ describe('#wohnungsboerse testsuite()', () => { it('should test wohnungsboerse provider', async () => { const Fredy = await mockFredy(); return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'wohnungsboerse', similarityCache); + const fredy = new Fredy( + provider.config, + null, + null, + provider.metaInformation.id, + 'wohnungsboerse', + similarityCache, + ); fredy.execute().then((listings) => { expect(listings).to.be.a('array'); const notificationObj = get(); From a134da396b5ea0a8aab80f8b6d919d363a486952 Mon Sep 17 00:00:00 2001 From: strech345 Date: Sun, 1 Mar 2026 17:18:27 +0100 Subject: [PATCH 06/12] feat(): solve maybe other providers --- test/provider/immobilienDe.test.js | 2 +- test/provider/sparkasse.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/provider/immobilienDe.test.js b/test/provider/immobilienDe.test.js index 08029b9b..a17485c4 100644 --- a/test/provider/immobilienDe.test.js +++ b/test/provider/immobilienDe.test.js @@ -14,7 +14,7 @@ describe('#immobilien.de testsuite()', () => { it('should test immobilien.de provider', async () => { const Fredy = await mockFredy(); return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache); + const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'test1', similarityCache); fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/sparkasse.test.js b/test/provider/sparkasse.test.js index 3e76fef1..875f8af4 100644 --- a/test/provider/sparkasse.test.js +++ b/test/provider/sparkasse.test.js @@ -14,7 +14,7 @@ describe('#sparkasse testsuite()', () => { const Fredy = await mockFredy(); provider.init(providerConfig.sparkasse, []); - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'sparkasse', similarityCache); + const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'sparkasse', similarityCache); const listing = await fredy.execute(); expect(listing).to.be.a('array'); From 85643fc6a869a2ae2967e053f3285335f0913bb4 Mon Sep 17 00:00:00 2001 From: strech345 Date: Sat, 7 Mar 2026 17:17:44 +0100 Subject: [PATCH 07/12] feat(): add specFilter config, also add rooms to listing --- lib/FredyPipelineExecutioner.js | 69 ++++++++++++++++--- lib/api/routes/jobRouter.js | 2 + lib/services/jobs/jobExecutionService.js | 10 +-- lib/services/storage/jobStorage.js | 16 ++++- lib/services/storage/listingsStorage.js | 22 ++---- .../migrations/sql/12.add-listing-specs.js | 10 +++ .../sql/13.add-rooms-to-listings.js | 10 +++ lib/utils/extract-number.js | 17 +++++ ui/src/views/jobs/mutation/JobMutation.jsx | 40 ++++++++++- ui/src/views/jobs/mutation/JobMutation.less | 18 +++++ 10 files changed, 172 insertions(+), 42 deletions(-) create mode 100644 lib/services/storage/migrations/sql/12.add-listing-specs.js create mode 100644 lib/services/storage/migrations/sql/13.add-rooms-to-listings.js create mode 100644 lib/utils/extract-number.js diff --git a/lib/FredyPipelineExecutioner.js b/lib/FredyPipelineExecutioner.js index 27b0db3f..123c517e 100755 --- a/lib/FredyPipelineExecutioner.js +++ b/lib/FredyPipelineExecutioner.js @@ -15,6 +15,7 @@ import { distanceMeters } from './services/listings/distanceCalculator.js'; import { getUserSettings } from './services/storage/settingsStorage.js'; import { updateListingDistance } from './services/storage/listingsStorage.js'; import booleanPointInPolygon from '@turf/boolean-point-in-polygon'; +import { extractNumber } from './utils/extract-number.js'; /** * @typedef {Object} Listing @@ -22,8 +23,13 @@ import booleanPointInPolygon from '@turf/boolean-point-in-polygon'; * @property {string} title Title or headline of the listing. * @property {string} [address] Optional address/location text. * @property {string} [price] Optional price text/value. + * @property {string} [size] Optional size text/value. + * @property {string} [rooms] Optional number of rooms text/value. * @property {string} [url] Link to the listing detail page. * @property {any} [meta] Provider-specific additional metadata. + * @property {number | null} [roomsInt] Optional number of rooms. + * @property {number | null} [sizeInt] Optional size of the listing. + * @property {number | null} [priceInt] Optional price of the listing. */ /** @@ -44,7 +50,9 @@ import booleanPointInPolygon from '@turf/boolean-point-in-polygon'; * 5) Identify new listings (vs. previously stored hashes) * 6) Persist new listings * 7) Filter out entries similar to already seen ones - * 8) Dispatch notifications + * 8) Filter out entries that do not match the job's specFilter + * 9) Filter out entries that do not match the job's spatialFilter + * 10) Dispatch notifications */ class FredyPipelineExecutioner { /** @@ -58,20 +66,25 @@ class FredyPipelineExecutioner { * @param {string} providerConfig.crawlContainer CSS selector for the container holding listing items. * @param {(raw:any)=>Listing} providerConfig.normalize Function to convert raw scraped data into a Listing shape. * @param {(listing:Listing)=>boolean} providerConfig.filter Function to filter out unwanted listings. + * + * @param {Object} job Job configuration. + * @param {string} job.id Job ID. + * @param {Object} job.notificationAdapter Notification configuration passed to notification adapters. + * @param {Object | null} job.spatialFilter Optional spatial filter configuration. + * @param {Object | null} job.specFilter Optional listing specifications (minRooms, minSize, maxPrice). + * * @param {(url:string, waitForSelector?:string)=>Promise|Promise} [providerConfig.getListings] Optional override to fetch listings. - * @param {Object} notificationConfig Notification configuration passed to notification adapters. - * @param {Object} spatialFilter Optional spatial filter configuration. * @param {string} providerId The ID of the provider currently in use. - * @param {string} jobKey Key of the job that is currently running (from within the config). * @param {SimilarityCache} similarityCache Cache instance for checking similar entries. * @param browser */ - constructor(providerConfig, notificationConfig, spatialFilter, providerId, jobKey, similarityCache, browser) { + constructor(providerConfig, job, providerId, similarityCache, browser) { this._providerConfig = providerConfig; - this._notificationConfig = notificationConfig; - this._spatialFilter = spatialFilter; + this._jobNotificationConfig = job.notificationAdapter; + this._jobKey = job.id; + this._jobSpecFilter = job.specFilter; + this._jobSpatialFilter = job.spatialFilter; this._providerId = providerId; - this._jobKey = jobKey; this._similarityCache = similarityCache; this._browser = browser; } @@ -92,6 +105,7 @@ class FredyPipelineExecutioner { .then(this._save.bind(this)) .then(this._calculateDistance.bind(this)) .then(this._filterBySimilarListings.bind(this)) + .then(this._filterBySpecs.bind(this)) .then(this._filterByArea.bind(this)) .then(this._notify.bind(this)) .catch(this._handleError.bind(this)); @@ -124,7 +138,7 @@ class FredyPipelineExecutioner { * @returns {Promise} Resolves with listings that are within the area (or not filtered if no area is set). */ _filterByArea(newListings) { - const polygonFeatures = this._spatialFilter?.features?.filter((f) => f.geometry?.type === 'Polygon'); + const polygonFeatures = this._jobSpatialFilter?.features?.filter((f) => f.geometry?.type === 'Polygon'); // If no area filter is set, return all listings if (!polygonFeatures?.length) { @@ -148,6 +162,30 @@ class FredyPipelineExecutioner { return filteredListings; } + /** + * Filter listings based on its specifications (minRooms, minSize, maxPrice). + * + * @param {Listing[]} newListings New listings to filter. + * @returns {Promise} Resolves with listings that pass the specification filters. + */ + _filterBySpecs(newListings) { + const { minRooms, minSize, maxPrice } = this._jobSpecFilter || {}; + + // If no specs are set, return all listings + if (!minRooms && !minSize && !maxPrice) { + return newListings; + } + + const filtered = newListings.filter((listing) => { + if (minRooms && listing.roomsInt && listing.roomsInt < minRooms) return false; + if (minSize && listing.sizeInt && listing.sizeInt < minSize) return false; + if (maxPrice && listing.priceInt && listing.priceInt > maxPrice) return false; + return true; + }); + + return filtered; + } + /** * Fetch listings from the provider, using the default Extractor flow unless * a provider-specific getListings override is supplied. @@ -182,7 +220,16 @@ class FredyPipelineExecutioner { * @returns {Listing[]} Normalized listings. */ _normalize(listings) { - return listings.map(this._providerConfig.normalize); + return listings.map((listing) => { + const normalized = this._providerConfig.normalize(listing); + // TODO: every provider should return price, size and rooms in numbers. Move this logic into the provider-specific normalize function. + return { + ...normalized, + priceInt: extractNumber(normalized.price), + sizeInt: extractNumber(normalized.size), + roomsInt: extractNumber(normalized.rooms), + }; + }); } /** @@ -227,7 +274,7 @@ class FredyPipelineExecutioner { if (newListings.length === 0) { throw new NoNewListingsWarning(); } - const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey); + const sendNotifications = notify.send(this._providerId, newListings, this._jobNotificationConfig, this._jobKey); return Promise.all(sendNotifications).then(() => newListings); } diff --git a/lib/api/routes/jobRouter.js b/lib/api/routes/jobRouter.js index 1798cf77..f059bc80 100644 --- a/lib/api/routes/jobRouter.js +++ b/lib/api/routes/jobRouter.js @@ -172,6 +172,7 @@ jobRouter.post('/', async (req, res) => { enabled, shareWithUsers = [], spatialFilter = null, + specFilter = null, } = req.body; const settings = await getSettings(); try { @@ -197,6 +198,7 @@ jobRouter.post('/', async (req, res) => { notificationAdapter, shareWithUsers, spatialFilter, + specFilter, }); } catch (error) { res.send(new Error(error)); diff --git a/lib/services/jobs/jobExecutionService.js b/lib/services/jobs/jobExecutionService.js index 471279c7..2e421e81 100644 --- a/lib/services/jobs/jobExecutionService.js +++ b/lib/services/jobs/jobExecutionService.js @@ -178,15 +178,7 @@ export function initJobExecutionService({ providers, settings, intervalMs }) { browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, {}); } - await new FredyPipelineExecutioner( - matchedProvider.config, - job.notificationAdapter, - job.spatialFilter, - prov.id, - job.id, - similarityCache, - browser, - ).execute(); + await new FredyPipelineExecutioner(matchedProvider.config, job, prov.id, similarityCache, browser).execute(); } catch (err) { logger.error(err); } diff --git a/lib/services/storage/jobStorage.js b/lib/services/storage/jobStorage.js index bf88003f..a734c5c2 100644 --- a/lib/services/storage/jobStorage.js +++ b/lib/services/storage/jobStorage.js @@ -31,6 +31,7 @@ export const upsertJob = ({ userId, shareWithUsers = [], spatialFilter = null, + specFilter = null, }) => { const id = jobId || nanoid(); const existing = SqliteConnection.query(`SELECT id, user_id FROM jobs WHERE id = @id LIMIT 1`, { id })[0]; @@ -44,7 +45,8 @@ export const upsertJob = ({ provider = @provider, notification_adapter = @notification_adapter, shared_with_user = @shareWithUsers, - spatial_filter = @spatialFilter + spatial_filter = @spatialFilter, + spec_filter = @specFilter WHERE id = @id`, { id, @@ -55,12 +57,13 @@ export const upsertJob = ({ provider: toJson(provider ?? []), notification_adapter: toJson(notificationAdapter ?? []), spatialFilter: spatialFilter ? toJson(spatialFilter) : null, + specFilter: specFilter ? toJson(specFilter) : null, }, ); } else { SqliteConnection.execute( - `INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user, spatial_filter) - VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers, @spatialFilter)`, + `INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user, spatial_filter, spec_filter) + VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers, @spatialFilter, @specFilter)`, { id, user_id: ownerId, @@ -71,6 +74,7 @@ export const upsertJob = ({ shareWithUsers: toJson(shareWithUsers ?? []), notification_adapter: toJson(notificationAdapter ?? []), spatialFilter: spatialFilter ? toJson(spatialFilter) : null, + specFilter: specFilter ? toJson(specFilter) : null, }, ); } @@ -92,6 +96,7 @@ export const getJob = (jobId) => { j.shared_with_user, j.notification_adapter AS notificationAdapter, j.spatial_filter AS spatialFilter, + j.spec_filter AS specFilter, (SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings FROM jobs j WHERE j.id = @id @@ -107,6 +112,7 @@ export const getJob = (jobId) => { shared_with_user: fromJson(row.shared_with_user, []), notificationAdapter: fromJson(row.notificationAdapter, []), spatialFilter: fromJson(row.spatialFilter, null), + specFilter: fromJson(row.specFilter, null), }; }; @@ -157,6 +163,7 @@ export const getJobs = () => { j.shared_with_user, j.notification_adapter AS notificationAdapter, j.spatial_filter AS spatialFilter, + j.spec_filter AS specFilter, (SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings FROM jobs j WHERE j.enabled = 1 @@ -170,6 +177,7 @@ export const getJobs = () => { shared_with_user: fromJson(row.shared_with_user, []), notificationAdapter: fromJson(row.notificationAdapter, []), spatialFilter: fromJson(row.spatialFilter, null), + specFilter: fromJson(row.specFilter, null), })); }; @@ -260,6 +268,7 @@ export const queryJobs = ({ j.shared_with_user, j.notification_adapter AS notificationAdapter, j.spatial_filter AS spatialFilter, + j.spec_filter AS specFilter, (SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings FROM jobs j ${whereSql} @@ -276,6 +285,7 @@ export const queryJobs = ({ shared_with_user: fromJson(row.shared_with_user, []), notificationAdapter: fromJson(row.notificationAdapter, []), spatialFilter: fromJson(row.spatialFilter, null), + specFilter: fromJson(row.specFilter, null), })); return { totalNumber, page: safePage, result }; diff --git a/lib/services/storage/listingsStorage.js b/lib/services/storage/listingsStorage.js index d8a54d02..819c6667 100755 --- a/lib/services/storage/listingsStorage.js +++ b/lib/services/storage/listingsStorage.js @@ -174,9 +174,9 @@ export const storeListings = (jobId, providerId, listings) => { SqliteConnection.withTransaction((db) => { const stmt = db.prepare( - `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address, + `INSERT INTO listings (id, hash, provider, job_id, price, size, rooms, title, image_url, description, address, link, created_at, is_active, latitude, longitude) - VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link, + VALUES (@id, @hash, @provider, @job_id, @price, @size, @rooms, @title, @image_url, @description, @address, @link, @created_at, 1, @latitude, @longitude) ON CONFLICT(job_id, hash) DO NOTHING`, ); @@ -187,8 +187,9 @@ export const storeListings = (jobId, providerId, listings) => { hash: item.id, provider: providerId, job_id: jobId, - price: extractNumber(item.price), - size: extractNumber(item.size), + price: item.priceInt, + size: item.sizeInt, + rooms: item.roomsInt, title: item.title, image_url: item.image, description: item.description, @@ -202,19 +203,6 @@ export const storeListings = (jobId, providerId, listings) => { } }); - /** - * Extract the first number from a string like "1.234 €" or "70 m²". - * Removes dots/commas before parsing. Returns null on invalid input. - * @param {string|undefined|null} str - * @returns {number|null} - */ - function extractNumber(str) { - if (!str) return null; - const cleaned = str.replace(/\./g, '').replace(',', '.'); - const num = parseFloat(cleaned); - return isNaN(num) ? null : num; - } - /** * Remove any parentheses segments (including surrounding whitespace) from a string. * Returns null for empty input. diff --git a/lib/services/storage/migrations/sql/12.add-listing-specs.js b/lib/services/storage/migrations/sql/12.add-listing-specs.js new file mode 100644 index 00000000..c7b7ec79 --- /dev/null +++ b/lib/services/storage/migrations/sql/12.add-listing-specs.js @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +export function up(db) { + db.exec(` + ALTER TABLE jobs ADD COLUMN spec_filter JSONB DEFAULT NULL; + `); +} diff --git a/lib/services/storage/migrations/sql/13.add-rooms-to-listings.js b/lib/services/storage/migrations/sql/13.add-rooms-to-listings.js new file mode 100644 index 00000000..870a8b74 --- /dev/null +++ b/lib/services/storage/migrations/sql/13.add-rooms-to-listings.js @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +export function up(db) { + db.exec(` + ALTER TABLE listings ADD COLUMN rooms INTEGER; + `); +} diff --git a/lib/utils/extract-number.js b/lib/utils/extract-number.js new file mode 100644 index 00000000..ec74545a --- /dev/null +++ b/lib/utils/extract-number.js @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +/** + * Extract the first number from a string like "1.234 €" or "70 m²". + * Removes dots/commas before parsing. Returns null on invalid input. + * @param {string|undefined|null} str + * @returns {number|null} + */ +export const extractNumber = (str) => { + if (!str) return null; + const cleaned = str.replace(/\./g, '').replace(',', '.'); + const num = parseFloat(cleaned); + return isNaN(num) ? null : num; +}; diff --git a/ui/src/views/jobs/mutation/JobMutation.jsx b/ui/src/views/jobs/mutation/JobMutation.jsx index c00d3f6f..623f0f33 100644 --- a/ui/src/views/jobs/mutation/JobMutation.jsx +++ b/ui/src/views/jobs/mutation/JobMutation.jsx @@ -24,9 +24,15 @@ import { IconPlayCircle, IconPlusCircle, IconUser, - IconClear, + IconFilter, } from '@douyinfe/semi-icons'; +const SPEC_FILTERS = [ + { key: 'maxPrice', translation: 'Max Price' }, + { key: 'minSize', translation: 'Min Size (m²)' }, + { key: 'minRooms', translation: 'Min Rooms' }, +]; + export default function JobMutator() { const jobs = useSelector((state) => state.jobsData.jobs); const shareableUserList = useSelector((state) => state.jobsData.shareableUserList); @@ -46,6 +52,7 @@ export default function JobMutator() { const defaultEnabled = sourceJob?.enabled ?? true; const defaultShareWithUsers = sourceJob?.shared_with_user ?? []; const defaultSpatialFilter = sourceJob?.spatialFilter || null; + const defaultSpecFilter = sourceJob?.specFilter || null; const [providerToEdit, setProviderToEdit] = useState(null); const [providerCreationVisible, setProviderCreationVisibility] = useState(false); @@ -58,6 +65,7 @@ export default function JobMutator() { const [shareWithUsers, setShareWithUsers] = useState(defaultShareWithUsers); const [enabled, setEnabled] = useState(defaultEnabled); const [spatialFilter, setSpatialFilter] = useState(defaultSpatialFilter); + const [specFilter, setSpecFilter] = useState(defaultSpecFilter); const navigate = useNavigate(); const actions = useActions(); @@ -66,6 +74,12 @@ export default function JobMutator() { setSpatialFilter(data); }, []); + const handleSpecFilterChange = (key, value) => { + if (!SPEC_FILTERS.map(({ key }) => key).includes(key)) return; + + setSpecFilter({ ...specFilter, [key]: value ? parseFloat(value) : null }); + }; + const isSavingEnabled = () => { return Boolean(notificationAdapterData.length && providerData.length && name); }; @@ -85,6 +99,7 @@ export default function JobMutator() { name, blacklist, spatialFilter, + specFilter, enabled, jobId: jobToBeEdit?.id || null, }); @@ -204,7 +219,7 @@ export default function JobMutator() { @@ -216,6 +231,27 @@ export default function JobMutator() { +
+ {SPEC_FILTERS.map((filter) => ( +
+
{filter.translation}
+ handleSpecFilterChange(filter.key, value)} + /> +
+ ))} +
+ + + diff --git a/ui/src/views/jobs/mutation/JobMutation.less b/ui/src/views/jobs/mutation/JobMutation.less index 2f14cb07..ac6c98b2 100644 --- a/ui/src/views/jobs/mutation/JobMutation.less +++ b/ui/src/views/jobs/mutation/JobMutation.less @@ -3,6 +3,24 @@ float: right; margin-bottom: 1rem; } + + &__specFilter { + display: flex; + gap: 1.5rem; + flex-wrap: wrap; + } + + &__specFilterItem { + display: flex; + flex-direction: column; + gap: 0.5rem; + flex: 1; + min-width: 150px; + } + + &__specFilterLabel { + font-weight: 500; + } } .semi-select-option-list-wrapper { From 067682b99e60f7eaf7fd0a153b5191306e779275 Mon Sep 17 00:00:00 2001 From: strech345 Date: Sat, 7 Mar 2026 17:18:12 +0100 Subject: [PATCH 08/12] feat(): change tests --- test/provider/einsAImmobilien.test.js | 17 +++++++++-------- test/provider/immobilienDe.test.js | 10 +++++++++- test/provider/immoscout.test.js | 9 ++++++++- test/provider/immoswp.test.js | 10 +++++++++- test/provider/immowelt.test.js | 9 ++++++++- test/provider/kleinanzeigen.test.js | 16 ++++++++-------- test/provider/mcMakler.test.js | 9 ++++++++- test/provider/neubauKompass.test.js | 17 +++++++++-------- test/provider/ohneMakler.test.js | 9 ++++++++- test/provider/regionalimmobilien24.test.js | 16 ++++++++-------- test/provider/sparkasse.test.js | 9 ++++++++- test/provider/wgGesucht.test.js | 10 +++++++++- test/provider/wohnungsboerse.test.js | 17 +++++++++-------- 13 files changed, 110 insertions(+), 48 deletions(-) diff --git a/test/provider/einsAImmobilien.test.js b/test/provider/einsAImmobilien.test.js index e0ebb02b..642231da 100644 --- a/test/provider/einsAImmobilien.test.js +++ b/test/provider/einsAImmobilien.test.js @@ -13,15 +13,16 @@ describe('#einsAImmobilien testsuite()', () => { provider.init(providerConfig.einsAImmobilien, [], []); it('should test einsAImmobilien provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: '', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy( - provider.config, - null, - null, - provider.metaInformation.id, - 'einsAImmobilien', - similarityCache, - ); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listings) => { expect(listings).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/immobilienDe.test.js b/test/provider/immobilienDe.test.js index a17485c4..655c9a43 100644 --- a/test/provider/immobilienDe.test.js +++ b/test/provider/immobilienDe.test.js @@ -13,8 +13,16 @@ describe('#immobilien.de testsuite()', () => { provider.init(providerConfig.immobilienDe, [], []); it('should test immobilien.de provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'test1', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'test1', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/immoscout.test.js b/test/provider/immoscout.test.js index 29b12e8c..3720323b 100644 --- a/test/provider/immoscout.test.js +++ b/test/provider/immoscout.test.js @@ -13,8 +13,15 @@ describe('#immoscout provider testsuite()', () => { provider.init(providerConfig.immoscout, [], []); it('should test immoscout provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: '', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, '', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); fredy.execute().then((listings) => { expect(listings).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/immoswp.test.js b/test/provider/immoswp.test.js index 8f34d3b1..2545f8c3 100644 --- a/test/provider/immoswp.test.js +++ b/test/provider/immoswp.test.js @@ -13,8 +13,16 @@ describe('#immoswp testsuite()', () => { provider.init(providerConfig.immoswp, [], []); it('should test immoswp provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'immoswp', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'immoswp', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/immowelt.test.js b/test/provider/immowelt.test.js index 9e5a0349..ebc6d07c 100644 --- a/test/provider/immowelt.test.js +++ b/test/provider/immowelt.test.js @@ -12,9 +12,16 @@ import * as provider from '../../lib/provider/immowelt.js'; describe('#immowelt testsuite()', () => { it('should test immowelt provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'immowelt', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; provider.init(providerConfig.immowelt, [], []); - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'immowelt', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + const listing = await fredy.execute(); expect(listing).to.be.a('array'); diff --git a/test/provider/kleinanzeigen.test.js b/test/provider/kleinanzeigen.test.js index 628cadff..42ab4928 100644 --- a/test/provider/kleinanzeigen.test.js +++ b/test/provider/kleinanzeigen.test.js @@ -12,16 +12,16 @@ import * as provider from '../../lib/provider/kleinanzeigen.js'; describe('#kleinanzeigen testsuite()', () => { it('should test kleinanzeigen provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'kleinanzeigen', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; provider.init(providerConfig.kleinanzeigen, [], []); return await new Promise((resolve) => { - const fredy = new Fredy( - provider.config, - null, - null, - provider.metaInformation.id, - 'kleinanzeigen', - similarityCache, - ); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/mcMakler.test.js b/test/provider/mcMakler.test.js index d35414f9..e556b70b 100644 --- a/test/provider/mcMakler.test.js +++ b/test/provider/mcMakler.test.js @@ -12,9 +12,16 @@ import * as provider from '../../lib/provider/mcMakler.js'; describe('#mcMakler testsuite()', () => { it('should test mcMakler provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'mcMakler', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; provider.init(providerConfig.mcMakler, []); - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'mcMakler', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + const listing = await fredy.execute(); expect(listing).to.be.a('array'); diff --git a/test/provider/neubauKompass.test.js b/test/provider/neubauKompass.test.js index 1182bb29..1606bd52 100644 --- a/test/provider/neubauKompass.test.js +++ b/test/provider/neubauKompass.test.js @@ -13,15 +13,16 @@ describe('#neubauKompass testsuite()', () => { provider.init(providerConfig.neubauKompass, [], []); it('should test neubauKompass provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'neubauKompass', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy( - provider.config, - null, - null, - provider.metaInformation.id, - 'neubauKompass', - similarityCache, - ); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/ohneMakler.test.js b/test/provider/ohneMakler.test.js index eec70f02..1d5d9200 100644 --- a/test/provider/ohneMakler.test.js +++ b/test/provider/ohneMakler.test.js @@ -12,9 +12,16 @@ import * as provider from '../../lib/provider/ohneMakler.js'; describe('#ohneMakler testsuite()', () => { it('should test ohneMakler provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'ohneMakler', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; provider.init(providerConfig.ohneMakler, []); - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'ohneMakler', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + const listing = await fredy.execute(); expect(listing).to.be.a('array'); diff --git a/test/provider/regionalimmobilien24.test.js b/test/provider/regionalimmobilien24.test.js index 2a4dfd97..cb7ed38b 100644 --- a/test/provider/regionalimmobilien24.test.js +++ b/test/provider/regionalimmobilien24.test.js @@ -12,16 +12,16 @@ import * as provider from '../../lib/provider/regionalimmobilien24.js'; describe('#regionalimmobilien24 testsuite()', () => { it('should test regionalimmobilien24 provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'regionalimmobilien24', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; provider.init(providerConfig.regionalimmobilien24, []); - const fredy = new Fredy( - provider.config, - null, - null, - provider.metaInformation.id, - 'regionalimmobilien24', - similarityCache, - ); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + const listing = await fredy.execute(); expect(listing).to.be.a('array'); diff --git a/test/provider/sparkasse.test.js b/test/provider/sparkasse.test.js index 875f8af4..cd01461a 100644 --- a/test/provider/sparkasse.test.js +++ b/test/provider/sparkasse.test.js @@ -12,9 +12,16 @@ import * as provider from '../../lib/provider/sparkasse.js'; describe('#sparkasse testsuite()', () => { it('should test sparkasse provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'sparkasse', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; provider.init(providerConfig.sparkasse, []); - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'sparkasse', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + const listing = await fredy.execute(); expect(listing).to.be.a('array'); diff --git a/test/provider/wgGesucht.test.js b/test/provider/wgGesucht.test.js index ad1ca9c0..d051c480 100644 --- a/test/provider/wgGesucht.test.js +++ b/test/provider/wgGesucht.test.js @@ -13,8 +13,16 @@ describe('#wgGesucht testsuite()', () => { provider.init(providerConfig.wgGesucht, [], []); it('should test wgGesucht provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'wgGesucht', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'wgGesucht', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/wohnungsboerse.test.js b/test/provider/wohnungsboerse.test.js index 2f270720..9510e5ac 100644 --- a/test/provider/wohnungsboerse.test.js +++ b/test/provider/wohnungsboerse.test.js @@ -13,15 +13,16 @@ describe('#wohnungsboerse testsuite()', () => { provider.init(providerConfig.wohnungsboerse, [], []); it('should test wohnungsboerse provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'wohnungsboerse', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy( - provider.config, - null, - null, - provider.metaInformation.id, - 'wohnungsboerse', - similarityCache, - ); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listings) => { expect(listings).to.be.a('array'); const notificationObj = get(); From a7c727398178c91203511c1b9edc1a818707650f Mon Sep 17 00:00:00 2001 From: strech345 Date: Sat, 7 Mar 2026 17:19:29 +0100 Subject: [PATCH 09/12] feat(): fix kleinanzeigen parser --- lib/provider/kleinanzeigen.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/provider/kleinanzeigen.js b/lib/provider/kleinanzeigen.js index d72c474c..09a25a91 100755 --- a/lib/provider/kleinanzeigen.js +++ b/lib/provider/kleinanzeigen.js @@ -10,10 +10,14 @@ let appliedBlackList = []; let appliedBlacklistedDistricts = []; function normalize(o) { - const size = o.size || '--- m²'; + const parts = (o.tags || '').split('·').map((p) => p.trim()); + const size = parts.find((p) => p.includes('m²')) || '--- m²'; + const rooms = parts.find((p) => p.includes('Zi.')) || '--- Zi.'; const id = buildHash(o.id, o.price); const link = `https://www.kleinanzeigen.de${o.link}`; - return Object.assign(o, { id, size, link }); + + delete o.tags; + return Object.assign(o, { id, size, rooms, link }); } function applyBlacklist(o) { @@ -33,7 +37,7 @@ const config = { crawlFields: { id: '.aditem@data-adid | int', price: '.aditem-main--middle--price-shipping--price | removeNewline | trim', - size: '.aditem-main .text-module-end | removeNewline | trim', + tags: '.aditem-main--middle--tags | removeNewline | trim', title: '.aditem-main .text-module-begin a | removeNewline | trim', link: '.aditem-main .text-module-begin a@href | removeNewline | trim', description: '.aditem-main .aditem-main--middle--description | removeNewline | trim', From 82d08d7a92754227dc1ded7703b664a97d7bb0d9 Mon Sep 17 00:00:00 2001 From: strech345 Date: Sun, 8 Mar 2026 19:37:18 +0100 Subject: [PATCH 10/12] feat(): add spec filter switch for listing overviiews --- lib/FredyPipelineExecutioner.js | 10 +++++-- lib/api/routes/listingsRouter.js | 3 ++ lib/services/storage/listingsStorage.js | 10 +++++++ .../components/grid/listings/ListingsGrid.jsx | 28 +++++++++++++++++-- 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/lib/FredyPipelineExecutioner.js b/lib/FredyPipelineExecutioner.js index 123c517e..a4a02392 100755 --- a/lib/FredyPipelineExecutioner.js +++ b/lib/FredyPipelineExecutioner.js @@ -222,7 +222,7 @@ class FredyPipelineExecutioner { _normalize(listings) { return listings.map((listing) => { const normalized = this._providerConfig.normalize(listing); - // TODO: every provider should return price, size and rooms in numbers. Move this logic into the provider-specific normalize function. + // TODO: every provider should return price, size and rooms in type number. This makes it more strong and strict of the provider output. String formats like "m², Zi,..." should not be part and can be added on fe or massages. Move this logic into the provider-specific normalize function. return { ...normalized, priceInt: extractNumber(normalized.price), @@ -240,7 +240,13 @@ class FredyPipelineExecutioner { * @returns {Listing[]} Filtered listings that pass validation and provider filter. */ _filter(listings) { - const keys = Object.keys(this._providerConfig.crawlFields); + // i removed it because crawlFields might be different than fields which are required. + // like for kleinanzeigen we have tags (includes multiple fields) but will be than extract at normalize, and deleted because its only internal used. + // I would suggest that we define a standard list like (id, price, rooms, size, title, link, description, address, image, url) + // it might be that some of this props value is null, wich is ok without id, link, title + // Also this might be not needed when using typings with typescript. I would suggest to move the whole project to typescript to have save typings. + //const keys = Object.keys(this._providerConfig.crawlFields); + const keys = ['id', 'link', 'title']; const filteredListings = listings.filter((item) => keys.every((key) => key in item)); return filteredListings.filter(this._providerConfig.filter); } diff --git a/lib/api/routes/listingsRouter.js b/lib/api/routes/listingsRouter.js index cf953333..179c7938 100644 --- a/lib/api/routes/listingsRouter.js +++ b/lib/api/routes/listingsRouter.js @@ -27,6 +27,7 @@ listingsRouter.get('/table', async (req, res) => { sortfield = null, sortdir = 'asc', freeTextFilter, + filterByJobSettings, } = req.query || {}; // normalize booleans (accept true, 'true', 1, '1' for true; false, 'false', 0, '0' for false) @@ -37,6 +38,7 @@ listingsRouter.get('/table', async (req, res) => { }; const normalizedActivity = toBool(activityFilter); const normalizedWatch = toBool(watchListFilter); + const normalizedFilterByJobSettings = toBool(filterByJobSettings) ?? true; let jobFilter = null; let jobIdFilter = null; @@ -56,6 +58,7 @@ listingsRouter.get('/table', async (req, res) => { jobIdFilter: jobIdFilter, providerFilter, watchListFilter: normalizedWatch, + filterByJobSettings: normalizedFilterByJobSettings, sortField: sortfield || null, sortDir: sortdir === 'desc' ? 'desc' : 'asc', userId: req.session.currentUser, diff --git a/lib/services/storage/listingsStorage.js b/lib/services/storage/listingsStorage.js index 819c6667..31f9395b 100755 --- a/lib/services/storage/listingsStorage.js +++ b/lib/services/storage/listingsStorage.js @@ -243,6 +243,7 @@ export const queryListings = ({ providerFilter, watchListFilter, freeTextFilter, + filterByJobSettings, sortField = null, sortDir = 'asc', userId = null, @@ -296,6 +297,15 @@ export const queryListings = ({ whereParts.push('(wl.id IS NULL)'); } + // filterByJobSettings: when true, filter listings by spec_filter in job settings + if (filterByJobSettings === true) { + whereParts.push(`( + (json_extract(j.spec_filter, '$.minRooms') IS NULL OR l.rooms IS NULL OR l.rooms >= json_extract(j.spec_filter, '$.minRooms')) AND + (json_extract(j.spec_filter, '$.minSize') IS NULL OR l.size IS NULL OR l.size >= json_extract(j.spec_filter, '$.minSize')) AND + (json_extract(j.spec_filter, '$.maxPrice') IS NULL OR l.price IS NULL OR l.price <= json_extract(j.spec_filter, '$.maxPrice')) + )`); + } + // Build whereSql (filtering by manually_deleted = 0) whereParts.push('(l.manually_deleted = 0)'); diff --git a/ui/src/components/grid/listings/ListingsGrid.jsx b/ui/src/components/grid/listings/ListingsGrid.jsx index d555db21..714479b4 100644 --- a/ui/src/components/grid/listings/ListingsGrid.jsx +++ b/ui/src/components/grid/listings/ListingsGrid.jsx @@ -19,6 +19,7 @@ import { Select, Popover, Empty, + Switch, } from '@douyinfe/semi-ui-19'; import { IconBriefcase, @@ -64,6 +65,7 @@ const ListingsGrid = () => { const [jobNameFilter, setJobNameFilter] = useState(null); const [activityFilter, setActivityFilter] = useState(null); const [providerFilter, setProviderFilter] = useState(null); + const [filterByJobSettings, setFilterByJobSettings] = useState(true); const [showFilterBar, setShowFilterBar] = useState(false); const [deleteModalVisible, setDeleteModalVisible] = useState(false); @@ -76,13 +78,23 @@ const ListingsGrid = () => { sortfield: sortField, sortdir: sortDir, freeTextFilter, - filter: { watchListFilter, jobNameFilter, activityFilter, providerFilter }, + filter: { watchListFilter, jobNameFilter, activityFilter, providerFilter, filterByJobSettings }, }); }; useEffect(() => { loadData(); - }, [page, sortField, sortDir, freeTextFilter, providerFilter, activityFilter, jobNameFilter, watchListFilter]); + }, [ + page, + sortField, + sortDir, + freeTextFilter, + providerFilter, + activityFilter, + jobNameFilter, + watchListFilter, + filterByJobSettings, + ]); const handleFilterChange = useMemo(() => debounce((value) => setFreeTextFilter(value), 500), []); @@ -227,6 +239,18 @@ const ListingsGrid = () => {
+ +
+
+ Options: +
+
+ setFilterByJobSettings(val)} size="small" /> + + Filter by Job Settings + +
+
)} From 8b368a10bee1e3b0ee62a50511a5e0c57c02a414 Mon Sep 17 00:00:00 2001 From: strech345 Date: Sun, 8 Mar 2026 20:05:09 +0100 Subject: [PATCH 11/12] feat(): add rooms and size to the overview and detail of a listing --- .../components/grid/listings/ListingsGrid.jsx | 53 +++++++++++-------- .../grid/listings/ListingsGrid.less | 13 +++++ ui/src/views/listings/ListingDetail.jsx | 20 ++++--- 3 files changed, 58 insertions(+), 28 deletions(-) diff --git a/ui/src/components/grid/listings/ListingsGrid.jsx b/ui/src/components/grid/listings/ListingsGrid.jsx index 714479b4..20d1d9cf 100644 --- a/ui/src/components/grid/listings/ListingsGrid.jsx +++ b/ui/src/components/grid/listings/ListingsGrid.jsx @@ -34,6 +34,8 @@ import { IconFilter, IconActivity, IconEyeOpened, + IconGridView, + IconExpand, } from '@douyinfe/semi-icons'; import { useNavigate } from 'react-router-dom'; import ListingDeletionModal from '../../ListingDeletionModal.jsx'; @@ -207,6 +209,13 @@ const ListingsGrid = () => { ))} + +
+ setFilterByJobSettings(val)} size="small" /> + + Job Settings + +
@@ -239,18 +248,6 @@ const ListingsGrid = () => {
- -
-
- Options: -
-
- setFilterByJobSettings(val)} size="small" /> - - Filter by Job Settings - -
-
)} @@ -305,9 +302,21 @@ const ListingsGrid = () => { {cap(item.title)} - } size="small"> - {item.price} € - + + } size="small"> + {item.price} € + + {item.size && ( + } size="small"> + {item.size} m² + + )} + {item.rooms && ( + } size="small"> + {item.rooms} Rooms + + )} + } @@ -317,12 +326,14 @@ const ListingsGrid = () => { > {item.address || 'No address provided'} - }> - {timeService.format(item.created_at, false)} - - }> - {item.provider.charAt(0).toUpperCase() + item.provider.slice(1)} - + + }> + {item.provider.charAt(0).toUpperCase() + item.provider.slice(1)} + + }> + {timeService.format(item.created_at, false)} + + {item.distance_to_destination ? ( }> {item.distance_to_destination} m to chosen address diff --git a/ui/src/components/grid/listings/ListingsGrid.less b/ui/src/components/grid/listings/ListingsGrid.less index d9a2473a..9bf6212a 100644 --- a/ui/src/components/grid/listings/ListingsGrid.less +++ b/ui/src/components/grid/listings/ListingsGrid.less @@ -45,6 +45,7 @@ } &--inactive { + .listingsGrid__imageContainer, .listingsGrid__content { opacity: 0.6; @@ -135,4 +136,16 @@ background: var(--semi-color-primary-hover); } } + + // Ensure icons and text are vertically aligned + .semi-typography { + display: inline-flex; + align-items: center; + + .semi-typography-icon { + display: flex; + align-items: center; + margin-top: 1px; // Minor nudge if needed, but flex should handle most + } + } } diff --git a/ui/src/views/listings/ListingDetail.jsx b/ui/src/views/listings/ListingDetail.jsx index 24bd81f6..39467619 100644 --- a/ui/src/views/listings/ListingDetail.jsx +++ b/ui/src/views/listings/ListingDetail.jsx @@ -31,7 +31,8 @@ import { IconLink, IconStar, IconStarStroked, - IconRealSize, + IconExpand, + IconGridView, } from '@douyinfe/semi-icons'; import maplibregl from 'maplibre-gl'; import 'maplibre-gl/dist/maplibre-gl.css'; @@ -259,6 +260,17 @@ export default function ListingDetail() { if (!listing) return null; const data = [ + { key: 'Price', value: `${listing.price} €`, Icon: }, + { + key: 'Size', + value: listing.size ? `${listing.size} m²` : 'N/A', + Icon: , + }, + { + key: 'Rooms', + value: listing.rooms ? `${listing.rooms} Rooms` : 'N/A', + Icon: , + }, { key: 'Job', value: listing.job_name, @@ -269,12 +281,6 @@ export default function ListingDetail() { value: listing.provider.charAt(0).toUpperCase() + listing.provider.slice(1), Icon: , }, - { key: 'Price', value: `${listing.price} €`, Icon: }, - { - key: 'Size', - value: listing.size ? `${listing.size} m²` : 'N/A', - Icon: , - }, { key: 'Added', value: timeService.format(listing.created_at), From 18cbe241374e74b24e94b2ac4d4367b370913a3e Mon Sep 17 00:00:00 2001 From: strech345 Date: Sun, 8 Mar 2026 20:07:18 +0100 Subject: [PATCH 12/12] feat(): rem label --- ui/src/components/grid/listings/ListingsGrid.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/components/grid/listings/ListingsGrid.jsx b/ui/src/components/grid/listings/ListingsGrid.jsx index 20d1d9cf..272d76d5 100644 --- a/ui/src/components/grid/listings/ListingsGrid.jsx +++ b/ui/src/components/grid/listings/ListingsGrid.jsx @@ -213,7 +213,7 @@ const ListingsGrid = () => {
setFilterByJobSettings(val)} size="small" /> - Job Settings + Job Filters