diff --git a/Dockerfile b/Dockerfile index ded10c3..07a41eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,6 @@ RUN wget https://hslstoragekarttatuotanto.blob.core.windows.net/openmaptiles/til EXPOSE 8080 CMD \ - yarn run data-fetcher && \ (Xorg -dpi 96 -nolisten tcp -noreset +extension GLX +extension RANDR +extension RENDER -logfile ./10.log -config ./xorg.conf :10 & \ DISPLAY=":10" yarn forever start --spinSleepTime 60000 --minUptime 30000 -c "node ${NODE_OPTS}" \ node_modules/tessera/bin/tessera.js --port 8080 --config config.js --multiprocess \ diff --git a/config.js b/config.js index 59545dd..1253c38 100644 --- a/config.js +++ b/config.js @@ -7,19 +7,6 @@ const sourcesUrl = "http://localhost:8080/"; const rasterHeaders = { "headers": { "Cache-Control": "public,max-age=604800" } }; const vectorHeaders = { "headers": { "Cache-Control": "public,max-age=43200" } }; -// Common defaults for vector data layers. -const geojsonSourceProps = { - "protocol": "geojson:", - "query": {}, - "minzoom": 5, - "maxzoom": 20, - "bounds": [18, 58, 32, 71], -}; - -// Center data to give hint for clients where to zoom. -const hslCenter = [24.9, 60.1, 14]; -const finlandCenter = [26, 63, 8]; - module.exports = { // v3 endpoints @@ -244,357 +231,5 @@ module.exports = { }) }, ...rasterHeaders, - }, - - - // V2 endpoints - // Will be deprecated soon. - - // Vector endpoints - - // New v2 endpoint. Uses OpenMapTiles schema. - "/map/v2/hsl-vector-map": { - "source": `mbtiles://${DATA_DIR}/finland.mbtiles`, - ...vectorHeaders, - }, - - // Citybike maps - "/map/v2/hsl-citybike-map": { - "source": { - ...geojsonSourceProps, - "center": hslCenter, - "name": "HSL Citybikes", - "sources": [{ - "id": "stations", - "file": `${DATA_DIR}/hsl-citybikes.geojson`, - }] - }, - ...vectorHeaders, - }, - "/map/v2/waltti-citybike-map": { - "source": { - ...geojsonSourceProps, - "center": finlandCenter, - "name": "Waltti Citybikes", - "maxzoom": 20, - "sources": [{ - "id": "stations", - "file": `${DATA_DIR}/waltti-citybikes.geojson`, - }] - }, - ...vectorHeaders, - }, - "/map/v2/finland-citybike-map": { - "source": { - ...geojsonSourceProps, - "center": finlandCenter, - "name": "Finland Citybikes", - "sources": [{ - "id": "stations", - "file": `${DATA_DIR}/finland-citybikes.geojson`, - }] - }, - ...vectorHeaders, - }, - - // Stop maps - "/map/v2/hsl-stop-map": { - "source": { - ...geojsonSourceProps, - "center": hslCenter, - "name": "Stops", - "sources": [{ - "id": "stops", - "file": `${DATA_DIR}/hsl-stops.geojson`, - }, { - "id": "stations", - "file": `${DATA_DIR}/hsl-stations.geojson`, - }] - }, - ...vectorHeaders, - - }, - "/map/v2/finland-stop-map": { - "source": { - ...geojsonSourceProps, - "center": finlandCenter, - "name": "Stops", - "sources": [{ - "id": "stops", - "file": `${DATA_DIR}/finland-stops.geojson`, - }, { - "id": "stations", - "file": `${DATA_DIR}/finland-stations.geojson`, - }] - }, - ...vectorHeaders, - - }, - "/map/v2/waltti-stop-map": { - "source": { - ...geojsonSourceProps, - "center": finlandCenter, - "name": "Stops", - "sources": [{ - "id": "stops", - "file": `${DATA_DIR}/waltti-stops.geojson`, - }, { - "id": "stations", - "file": `${DATA_DIR}/waltti-stations.geojson`, - }] - }, - ...vectorHeaders, - - }, - - // Park and ride map (Liityntäpysäköinti) - "/map/v2/hsl-parkandride-map": { - "source": { - ...geojsonSourceProps, - "center": hslCenter, - "name": "HSL Park & Ride", - "sources": [{ - "id": "facilities", - "file": `${DATA_DIR}/hsl-parkandride-facilities.geojson`, - }, { - "id": "hubs", - "file": `${DATA_DIR}/hsl-parkandride-hubs.geojson`, - }, { - "id": "facility-points", - "file": `${DATA_DIR}/hsl-parkandride-facility-points.geojson`, - }] - }, - ...vectorHeaders, - }, - - // Ticket sales point map - "/map/v2/hsl-ticket-sales-map": { - "source": { - ...geojsonSourceProps, - "center": hslCenter, - "name": "Ticket sales", - "sources": [{ - "id": "ticket-sales", - "file": `${DATA_DIR}/hsl-ticket-sales.geojson`, - }] - }, - ...vectorHeaders, - }, - - // Raster endpoints - // Default layer tile is 512x512 pixels, but 256x256 layers are defined separately. - - // The main map. Reittiopas style. - "/map/v2/hsl-map": { - "source": { - "protocol": "gl:", - "query": { bufferWidth: 8 }, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - simplified: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, - "/map/v2/hsl-map-256": { - "source": { - "protocol": "gl:", - "query": { layerTileSize: 256, bufferWidth: 8 }, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - simplified: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, - - // Swedish map style - "/map/v2/hsl-map-sv": { - "source": { - "protocol": "gl:", - "query": { bufferWidth: 8 }, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - text_sv: { enabled: true }, - simplified: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, - "/map/v2/hsl-map-sv-256": { - "source": { - "protocol": "gl:", - "query": { layerTileSize: 256, bufferWidth: 8 }, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - text_sv: { enabled: true }, - simplified: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, - - // English map style - "/map/v2/hsl-map-en": { - "source": { - "protocol": "gl:", - "query": { bufferWidth: 8 }, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - text_en: { enabled: true }, - simplified: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, - "/map/v2/hsl-map-en-256": { - "source": { - "protocol": "gl:", - "query": { layerTileSize: 256, bufferWidth: 8 }, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - text_en: { enabled: true }, - simplified: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, - - // Bilingual map style - "/map/v2/hsl-map-fi-sv": { - "source": { - "protocol": "gl:", - "query": { bufferWidth: 8 }, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - text_fisv: { enabled: true }, - simplified: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, - "/map/v2/hsl-map-fi-sv-256": { - "source": { - "protocol": "gl:", - "query": { layerTileSize: 256, bufferWidth: 8 }, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - text_fisv: { enabled: true }, - simplified: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, - - // Map with no texts - "/map/v2/hsl-map-no-text": { - "source": { - "protocol": "gl:", - "query": {}, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - text: { enabled: false }, - simplified: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, - "/map/v2/hsl-map-no-text-256": { - "source": { - "protocol": "gl:", - "query": { layerTileSize: 256 }, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - text: { enabled: false }, - simplified: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, - - // Greyscale map - "/map/v2/hsl-map-greyscale": { - "source": { - "protocol": "gl:", - "query": { bufferWidth: 8 }, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - greyscale: { enabled: true }, - simplified: { enabled: true }, - municipal_borders: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, - "/map/v2/hsl-map-greyscale-256": { - "source": { - "protocol": "gl:", - "query": { layerTileSize: 256, bufferWidth: 8 }, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - greyscale: { enabled: true }, - simplified: { enabled: true }, - municipal_borders: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, - - // Greyscale map without texts - "/map/v2/hsl-map-greyscale-no-text": { - "source": { - "protocol": "gl:", - "query": {}, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - text: { enabled: false }, - greyscale: { enabled: true }, - simplified: { enabled: true }, - municipal_borders: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, - "/map/v2/hsl-map-greyscale-256-no-text": { - "source": { - "protocol": "gl:", - "query": { layerTileSize: 256 }, - "style": hslMapStyle.generateStyle({ - sourcesUrl, - components: { - text: { enabled: false }, - greyscale: { enabled: true }, - simplified: { enabled: true }, - municipal_borders: { enabled: true } - } - }) - }, - ...rasterHeaders, - }, + } }; diff --git a/data-fetcher/configurations.js b/data-fetcher/configurations.js deleted file mode 100644 index 6146d4f..0000000 --- a/data-fetcher/configurations.js +++ /dev/null @@ -1,128 +0,0 @@ -const { - HSL_OTP_URL, - FINLAND_OTP_URL, - WALTTI_OTP_URL, - PARKANDRIDE_URL, - TICKET_SALES_URL -} = require("../constants"); -const { queries, wranglers } = require("./data"); - -const layers = [ - { - name: "hsl-stops", - sources: [ - { - url: HSL_OTP_URL, - gqlQuery: queries.stopQuery, - wrangler: wranglers.stopWrangler, - file: "hsl-stops.geojson", - }, - { - url: HSL_OTP_URL, - gqlQuery: queries.stationQuery, - wrangler: wranglers.stationWrangler, - file: "hsl-stations.geojson", - } - ] - }, - { - name: "finland-stops", - sources: [ - { - url: FINLAND_OTP_URL, - gqlQuery: queries.stopQuery, - wrangler: wranglers.stopWrangler, - file: "finland-stops.geojson", - }, - { - url: FINLAND_OTP_URL, - gqlQuery: queries.stationQuery, - wrangler: wranglers.stationWrangler, - file: "finland-stations.geojson", - } - ] - }, - { - name: "waltti-stops", - sources: [ - { - url: WALTTI_OTP_URL, - gqlQuery: queries.stopQuery, - wrangler: wranglers.stopWrangler, - file: "waltti-stops.geojson", - }, - { - url: WALTTI_OTP_URL, - gqlQuery: queries.stationQuery, - wrangler: wranglers.stationWrangler, - file: "waltti-stations.geojson", - } - ] - }, - { - name: "hsl-citybikes", - sources: [ - { - url: HSL_OTP_URL, - gqlQuery: queries.citybikeQuery, - wrangler: wranglers.citybikeWrangler, - file: "hsl-citybikes.geojson", - } - ] - }, - { - name: "finland-citybikes", - sources: [ - { - url: FINLAND_OTP_URL, - gqlQuery: queries.citybikeQuery, - wrangler: wranglers.citybikeWrangler, - file: "finland-citybikes.geojson", - } - ] - }, - { - name: "waltti-citybikes", - sources: [ - { - url: WALTTI_OTP_URL, - gqlQuery: queries.citybikeQuery, - wrangler: wranglers.citybikeWrangler, - file: "waltti-citybikes.geojson", - } - ] - }, - { - name: "hsl-parkandride", - sources: [ - { - url: `${PARKANDRIDE_URL}facilities.geojson`, - wrangler: wranglers.dummyGeojsonWrangler, - file: "hsl-parkandride-facilities.geojson", - }, - { - url: `${PARKANDRIDE_URL}hubs.geojson`, - wrangler: wranglers.dummyGeojsonWrangler, - file: "hsl-parkandride-hubs.geojson", - }, - // Same data as facilities, but converted to points - { - url: `${PARKANDRIDE_URL}facilities.geojson`, - wrangler: wranglers.geojsonPolygonToPointWrangler, - file: "hsl-parkandride-facility-points.geojson", - } - ] - }, - { - name: "hsl-ticket-sales", - sources: [ - { - url: TICKET_SALES_URL, - wrangler: wranglers.dummyGeojsonWrangler, - file: "hsl-ticket-sales.geojson", - } - ] - } -]; - -module.exports = layers; diff --git a/data-fetcher/data.js b/data-fetcher/data.js deleted file mode 100644 index 55f80e2..0000000 --- a/data-fetcher/data.js +++ /dev/null @@ -1,186 +0,0 @@ -const _ = require("lodash"); -const centroid = require("@turf/centroid").default; - -// Wrapper to log errors -const errorLogger = (wrangler) => ( - (body) => { - try { - return wrangler(body); - } catch (err) { - console.error("Response body for the following error was:\n", body); - throw err; - } - } -); - -// This just modifies response body to json object -const dummyGeojsonWrangler = (body) => { - const geojsonData = JSON.parse(body); - return geojsonData; -}; - -// Converts geojson polygons to points. -const geojsonPolygonToPointWrangler = (body) => { - const originalGeojsonData = JSON.parse(body); - const centeredFeatures = originalGeojsonData.features.map((f) => ({ - ...f, // Keep the original properties - geometry: centroid(f).geometry, - })); - - return { - type: "FeatureCollection", - features: centeredFeatures, - }; -}; - -const stopQuery = ` - query stops { - stops { - gtfsId - name - code - platformCode - lat - lon - locationType - desc - parentStation { - gtfsId - } - patterns { - headsign - route { - mode - shortName - gtfsType: type - } - } - } - } -`; - -const stopWrangler = (body) => { - const { data } = JSON.parse(body); - - return { - type: "FeatureCollection", - features: data.stops.map((stop) => ({ - type: "Feature", - geometry: { - type: "Point", - coordinates: [stop.lon, stop.lat] - }, - properties: { - gtfsId: stop.gtfsId, - name: stop.name, - code: stop.code, - platform: stop.platformCode == null ? "null" : stop.platformCode, // TODO: 'null' -string should be changed to null after the map style of HSL app has been updated. - desc: stop.desc, - parentStation: stop.parentStation == null ? "null" : stop.parentStation.gtfsId, // TODO: 'null' -string should be changed to null after the map style of HSL app has been updated. - type: stop.patterns == null ? null : _.uniq(stop.patterns.map((pattern) => pattern.route.mode)).join(","), - patterns: stop.patterns == null ? null : JSON.stringify(stop.patterns.map((pattern) => ({ - headsign: pattern.headsign, - type: pattern.route.mode, - shortName: pattern.route.shortName, - gtfsType: pattern.route.gtfsType, - }))) - } - })) - }; -}; - -const stationQuery = ` - query stations{ - stations { - gtfsId - name - lat - lon - locationType - stops { - gtfsId - patterns { - route { - mode - shortName - gtfsType: type - } - } - } - } - } -`; - -const stationWrangler = (body) => { - const { data } = JSON.parse(body); - - return { - type: "FeatureCollection", - features: data.stations.map((station) => ({ - type: "Feature", - geometry: { - type: "Point", - coordinates: [station.lon, station.lat] - }, - properties: { - gtfsId: station.gtfsId, - name: station.name, - type: _.uniq(_.flatten(station.stops.map((stop) => ( - stop.patterns.map((pattern) => ( - pattern.route.mode - )) - )))).join(","), - stops: JSON.stringify(station.stops.map((stop) => stop.gtfsId)), - routes: JSON.stringify(_.uniqWith(_.flatten(station.stops.map((stop) => ( - stop.patterns.map((pattern) => pattern.route) - ))), _.isEqual)), - } - })) - }; -}; - -const citybikeQuery = ` - query bikerentals { - bikeRentalStations { - stationId - name - networks - lon - lat - } - } -`; - -const citybikeWrangler = (body) => { - const { data } = JSON.parse(body); - return { - type: "FeatureCollection", - features: data.bikeRentalStations.map((station) => ({ - type: "Feature", - geometry: { - type: "Point", - coordinates: [station.lon, station.lat] - }, - properties: { - id: station.stationId, - name: station.name, - networks: station.networks.join() - } - })) - }; -}; - -module.exports = { - queries: { - stopQuery, - stationQuery, - citybikeQuery, - }, - wranglers: { - dummyGeojsonWrangler: errorLogger(dummyGeojsonWrangler), - geojsonPolygonToPointWrangler: errorLogger(geojsonPolygonToPointWrangler), - stopWrangler: errorLogger(stopWrangler), - stationWrangler: errorLogger(stationWrangler), - citybikeWrangler: errorLogger(citybikeWrangler), - }, -}; diff --git a/data-fetcher/fetcher.js b/data-fetcher/fetcher.js deleted file mode 100644 index 5ef5cf0..0000000 --- a/data-fetcher/fetcher.js +++ /dev/null @@ -1,65 +0,0 @@ -const fs = require("fs"); -const request = require("requestretry"); - -const { DATA_DIR } = require("../constants"); - -const dummyData = { "type": "Feature", "properties": {}, "geometry": null }; // Empty geometry to be used if url doesn't respond correctly - -const saveJson = (data, filename) => { - const content = JSON.stringify(data); // All data should be GeoJSON at this point. - fs.writeFileSync(`${DATA_DIR}/${filename}`, content); -}; - -const fetchAndSaveData = (dataUrl, wrangler, filename, gqlQuery) => { - const uri = dataUrl.startsWith("http") ? dataUrl : `http://${dataUrl}`; - - const commonRequestParams = { - uri, - maxAttempts: Number(process.env.DOWNLOAD_RETRY_COUNT) || 3, - retryDelay: 15000, - followAllRedirects: true, - fullResponse: false, - retryStrategy: (err, response) => { - const retryNetworkErr = request.RetryStrategies.NetworkError(err, response); - const retryHttpErr = request.RetryStrategies.HTTPError(err, response); - - /* eslint-disable no-console */ - if (retryNetworkErr) console.log(`A network error on ${uri}. The error code was ${err.code}. Retrying...`); - if (retryHttpErr) console.log(`A http error on ${uri}. The response status code was ${response.statusCode}. Retrying...`); - /* eslint-enable no-console */ - - return retryNetworkErr || retryHttpErr; - }, - }; - - const gqlRequestParams = { - body: gqlQuery, - method: "POST", - headers: { - "Content-Type": "application/graphql", - // These are some OTP specific headers. Restructure them to specific queries - // if some other gql api is used, and these headers broke the query. - "OTPTimeout": "120000", - "OTPMaxResolves": "100000000", - } - }; - - return ( - // Currently there are only two types of sources. - request(gqlQuery ? { ...commonRequestParams, ...gqlRequestParams } : commonRequestParams) - // Wrangler converts response to geojson format - .then( - wrangler, - (err) => { throw err; } // Handle error later - ) - .then( - (data) => { saveJson(data, filename); }, - // Save dummy data if error happened - (err) => { saveJson(dummyData, filename); return err; } - ) - ); -}; - -module.exports = { - fetchAndSaveData -}; diff --git a/data-fetcher/index.js b/data-fetcher/index.js deleted file mode 100644 index 91f28de..0000000 --- a/data-fetcher/index.js +++ /dev/null @@ -1,23 +0,0 @@ -// Script to download and save all data sources from configuration file. - -const _ = require("lodash"); -const layers = require("./configurations"); -const { fetchAndSaveData } = require("./fetcher"); - -const dataProcesses = layers.map((layer) => ( - Promise.all(layer.sources.map((source) => fetchAndSaveData( - source.url, - source.wrangler, - source.file, - source.gqlQuery, - ).then((err) => { - if (err) console.error(`Error fetching ${source.file}\n`, err); - return _.pickBy({ - file: source.file, - status: !err ? "ok" : "error", - error: err && err.toString(), - }, (d) => d !== undefined); - }))))); - -Promise.all(dataProcesses) - .then((status) => console.log("Download process done:\n", status)); diff --git a/package.json b/package.json index b102c2e..c1424d0 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,6 @@ "description": "HSL map server", "main": "config.js", "scripts": { - "data-fetcher": "node data-fetcher/index.js", "test": "echo \"Error: no test specified\" && exit 1" }, "repository": {