Skip to content

Commit f8c173e

Browse files
Rub21Rub21
authored
Update TM-api and planet dump configurations (#370)
* Use alembic for migration - TM * Update values for tm api * Add EXTRA_CORS_ORIGINS in TM config * Delete EXTRA_CORS_ORIGINS from env vars * Add PLANET_DUMP_NG_METADATA_URL as env var to add metadata * Install curl to download metadata for add in pbf files * Update docker base image for planet generators
1 parent c5f14f9 commit f8c173e

File tree

10 files changed

+55
-20
lines changed

10 files changed

+55
-20
lines changed

‎images/full-history/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.964.h5e9b11b
1+
FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.968.hb111c99
22

33
VOLUME /mnt/data
44
COPY ./start.sh /

‎images/full-history/start.sh‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -101,13 +101,22 @@ upload_planet_file() {
101101
# ===============================
102102

103103
if [ "$PLANET_EXPORT_METHOD" == "planet-dump-ng" ]; then
104-
download_dump_file
105-
echo "Generating history planet file with planet-dump-ng..."
106-
export PLANET_EPOCH_DATE="$PLANET_EPOCH_DATE"
107-
planet-dump-ng \
108-
--dump-file "$dumpFile" \
109-
--history-pbf "$local_planetHistoryPBFFile"
110-
104+
download_dump_file
105+
echo "Generating history planet file with planet-dump-ng..."
106+
export PLANET_EPOCH_DATE="$PLANET_EPOCH_DATE"
107+
108+
if [ -n "$PLANET_DUMP_NG_METADATA_URL" ]; then
109+
echo "Downloading metadata file..."
110+
curl "$PLANET_DUMP_NG_METADATA_URL" -o metadata.yml
111+
planet-dump-ng \
112+
--dump-file "$dumpFile" \
113+
--history-pbf "$local_planetHistoryPBFFile" \
114+
-M metadata.yml
115+
else
116+
planet-dump-ng \
117+
--dump-file "$dumpFile" \
118+
--history-pbf "$local_planetHistoryPBFFile"
119+
fi
111120
elif [ "$PLANET_EXPORT_METHOD" == "osmosis" ]; then
112121
echo "Generating history planet file with osmosis..."
113122
# Creating full history

‎images/osm-processor/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ RUN set -ex \
4545
tmux \
4646
zsh \
4747
git \
48+
curl \
4849
libxml2 \
4950
libboost-filesystem1.74.0 \
5051
libboost-program-options1.74.0 \
@@ -56,4 +57,4 @@ RUN set -ex \
5657
libosmpbf1 \
5758
&& rm -rf /var/lib/apt/lists/*
5859

59-
COPY --from=builder /opt/planet-dump-ng/planet-dump-ng /usr/local/bin/planet-dump-ng
60+
COPY --from=builder /opt/planet-dump-ng/planet-dump-ng /usr/local/bin/planet-dump-ng

‎images/planet-dump/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.964.h5e9b11b
1+
FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.968.hb111c99
22

33
VOLUME /mnt/data
44
COPY ./start.sh /

‎images/planet-dump/start.sh‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -100,11 +100,20 @@ upload_planet_file() {
100100
# ===============================
101101

102102
if [ "$PLANET_EXPORT_METHOD" == "planet-dump-ng" ]; then
103-
download_dump_file
104-
echo "Generating planet file with planet-dump-ng..."
105-
planet-dump-ng \
106-
--dump-file "$dumpFile" \
107-
--pbf "$local_planetPBFFile"
103+
download_dump_file
104+
echo "Generating planet file with planet-dump-ng..."
105+
106+
if [ -n "$PLANET_DUMP_NG_METADATA_URL" ]; then
107+
curl "$PLANET_DUMP_NG_METADATA_URL" -o metadata.yml
108+
planet-dump-ng \
109+
--dump-file "$dumpFile" \
110+
--pbf "$local_planetPBFFile" \
111+
-M metadata.yml
112+
else
113+
planet-dump-ng \
114+
--dump-file "$dumpFile" \
115+
--pbf "$local_planetPBFFile"
116+
fi
108117
elif [ "$PLANET_EXPORT_METHOD" == "osmosis" ]; then
109118
echo "Generating planet file with osmosis..."
110119
if [ -z "$MEMORY_JAVACMD_OPTIONS" ]; then

‎osm-seed/templates/jobs/full-history-job.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ spec:
5151
value: {{ .Values.fullHistory.env.DUMP_CLOUD_URL | quote}}
5252
- name: PLANET_EPOCH_DATE
5353
value: {{ .Values.fullHistory.env.PLANET_EPOCH_DATE | quote}}
54+
- name: PLANET_DUMP_NG_METADATA_URL
55+
value: {{ .Values.fullHistory.env.PLANET_DUMP_NG_METADATA_URL | quote}}
5456
# In case cloudProvider=aws
5557
{{- if eq .Values.cloudProvider "aws" }}
5658
- name: AWS_S3_BUCKET

‎osm-seed/templates/jobs/planet-dump-job.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ spec:
5555
value: {{ .Values.planetDump.env.DUMP_CLOUD_URL | quote}}
5656
- name: PLANET_EPOCH_DATE
5757
value: {{ .Values.planetDump.env.PLANET_EPOCH_DATE | quote}}
58+
- name: PLANET_DUMP_NG_METADATA_URL
59+
value: {{ .Values.planetDump.env.PLANET_DUMP_NG_METADATA_URL | quote}}
5860
# In case cloudProvider=aws
5961
{{- if eq .Values.cloudProvider "aws" }}
6062
- name: AWS_S3_BUCKET

‎osm-seed/templates/tasking-manager-api/tm-configMap.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ data:
99
POSTGRES_PASSWORD: {{ quote .Values.tmDb.env.POSTGRES_PASSWORD | default "" }}
1010
POSTGRES_USER: {{ quote .Values.tmDb.env.POSTGRES_USER | default "" }}
1111
POSTGRES_PORT: "5432"
12-
TM_DB: {{ quote (printf "postgresql://%s:%s@%s-tasking-manager-db/%s" .Values.tmDb.env.POSTGRES_USER .Values.tmDb.env.POSTGRES_PASSWORD .Release.Name .Values.tmDb.env.POSTGRES_DB) }}
12+
TM_DB: {{ quote (printf "postgresql+asyncpg://%s:%s@%s-tasking-manager-db/%s" .Values.tmDb.env.POSTGRES_USER .Values.tmDb.env.POSTGRES_PASSWORD .Release.Name .Values.tmDb.env.POSTGRES_DB) }}
1313
TM_ORG_NAME: {{ quote .Values.tmApi.env.TM_ORG_NAME | default "" }}
1414
TM_ORG_CODE: {{ quote .Values.tmApi.env.TM_ORG_CODE | default "" }}
1515
TM_ORG_URL: {{ quote .Values.tmApi.env.TM_ORG_URL | default "" }}
@@ -44,4 +44,6 @@ data:
4444
TM_REDIRECT_URI: {{ quote .Values.tmApi.env.TM_REDIRECT_URI | default "" }}
4545
TM_SCOPE: {{ quote .Values.tmApi.env.TM_SCOPE | default "" }}
4646
TM_APP_API_VERSION: {{ quote .Values.tmApi.env.TM_APP_API_VERSION | default "v2" }}
47-
{{- end }}
47+
OHSOME_STATS_TOKEN: {{ quote .Values.tmApi.env.OHSOME_STATS_TOKEN }}
48+
POSTGRES_TEST_DB: {{ quote .Values.tmApi.env.POSTGRES_TEST_DB }}
49+
{{- end }}

‎osm-seed/templates/tasking-manager-api/tm-deployment.yaml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,12 @@ spec:
2626
initContainers:
2727
- name: db-upgrade
2828
image: "{{ .Values.tmApi.image.name }}:{{ .Values.tmApi.image.tag }}"
29-
command: ["flask", "db", "upgrade"]
29+
command:
30+
- alembic
31+
- -c
32+
- /usr/src/app/migrations/alembic.ini
33+
- upgrade
34+
- head
3035
envFrom:
3136
- configMapRef:
3237
name: {{ .Release.Name }}-tm-common-env

‎osm-seed/values.yaml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -262,9 +262,10 @@ fullHistory:
262262
schedule: "* * * * *"
263263
env:
264264
OVERWRITE_FHISTORY_FILE: false
265-
PLANET_EXPORT_METHOD: osmosis
265+
PLANET_EXPORT_METHOD: planet-dump-ng
266266
DUMP_CLOUD_URL: s3://osm-seed/db.dump
267267
PLANET_EPOCH_DATE: '1970-01-01'
268+
PLANET_DUMP_NG_METADATA_URL: https://osmseed.org/metadata/planet-meta.yaml
268269
resources:
269270
enabled: false
270271
requests:
@@ -348,7 +349,9 @@ planetDump:
348349
env:
349350
OVERWRITE_PLANET_FILE: false
350351
DUMP_CLOUD_URL : s3://osm-seed/db.dump
351-
PLANET_EXPORT_METHOD: osmosis
352+
PLANET_EXPORT_METHOD: planet-dump-ng
353+
PLANET_EPOCH_DATE: '1970-01-01'
354+
PLANET_DUMP_NG_METADATA_URL: https://osmseed.org/metadata/planet-meta.yaml
352355
resources:
353356
enabled: false
354357
requests:
@@ -726,6 +729,8 @@ tmApi:
726729
TM_DEFAULT_LOCALE: "en"
727730
TM_APP_API_URL: ""
728731
TM_APP_BASE_URL: ""
732+
OHSOME_STATS_TOKEN: "dummy-token-for-migrations"
733+
POSTGRES_TEST_DB: "postgresql://user:pass@db:5432/test_db"
729734
resources:
730735
enabled: false
731736
requests:

0 commit comments

Comments
 (0)