From 05aa20d4952977387c979920fd3cc3dc3b3ec3b5 Mon Sep 17 00:00:00 2001 From: Sebastian Neuser Date: Mon, 5 May 2025 16:41:28 +0200 Subject: [PATCH 1/5] clean: Remove obsolete docker-compose version spec --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d20d556..f30a805 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3.5' services: parsedmarc: build: ./parsedmarc/ From 78932c8263f226646485f629d260e95be7dc1b2f Mon Sep 17 00:00:00 2001 From: Sebastian Neuser Date: Mon, 5 May 2025 16:44:46 +0200 Subject: [PATCH 2/5] impr: Update elasticsearch version; add healthcheck --- docker-compose.yml | 11 +++++++++-- grafana/grafana-provisioning/datasources/all.yml | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f30a805..d7507a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,15 +6,22 @@ services: - ./output_files:/output command: parsedmarc -c /parsedmarc.ini /input/* --debug depends_on: - - elasticsearch + elasticsearch: + condition: service_healthy restart: on-failure elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.5 + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.28 environment: - discovery.type=single-node volumes: - ./elastic_data:/usr/share/elasticsearch/data + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:9200 || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + restart: on-failure grafana: build: ./grafana/ diff --git a/grafana/grafana-provisioning/datasources/all.yml b/grafana/grafana-provisioning/datasources/all.yml index ae59743..5cc91dd 100644 --- a/grafana/grafana-provisioning/datasources/all.yml +++ b/grafana/grafana-provisioning/datasources/all.yml @@ -9,7 +9,7 @@ datasources: database: '[dmarc_aggregate-]YYYY-MM-DD' isDefault: true jsonData: - esVersion: 7.17.5 + esVersion: 7.17.28 timeField: 'date_range' interval: 'Daily' version: 1 @@ -22,7 +22,7 @@ datasources: database: '[dmarc_forensic-]YYYY-MM-DD' isDefault: false jsonData: - esVersion: 7.17.5 + esVersion: 7.17.28 timeField: 'arrival_date' interval: 'Daily' version: 1 From 0207493e5711d382fbd9c522a8561b8a2b69ac8b Mon Sep 17 00:00:00 2001 From: Sebastian Neuser Date: Mon, 5 May 2025 16:50:06 +0200 Subject: [PATCH 3/5] impr: Update parsedmarc Python version; mount config --- .gitignore | 2 -- docker-compose.yml | 5 +++-- parsedmarc/Dockerfile | 5 +---- parsedmarc/{parsedmarc.sample.ini => parsedmarc.ini} | 3 ++- 4 files changed, 6 insertions(+), 9 deletions(-) rename parsedmarc/{parsedmarc.sample.ini => parsedmarc.ini} (89%) diff --git a/.gitignore b/.gitignore index f345aae..9aa7d22 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ /files/ /output_files/ -/parsedmarc/GeoLite2-Country* /elastic_data/ -parsedmarc.ini diff --git a/docker-compose.yml b/docker-compose.yml index d7507a5..8f21a4a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,11 @@ services: parsedmarc: - build: ./parsedmarc/ + build: ./parsedmarc volumes: - ./files:/input:ro - ./output_files:/output - command: parsedmarc -c /parsedmarc.ini /input/* --debug + - ./parsedmarc/parsedmarc.ini:/parsedmarc.ini + command: parsedmarc -c /parsedmarc.ini /input/* depends_on: elasticsearch: condition: service_healthy diff --git a/parsedmarc/Dockerfile b/parsedmarc/Dockerfile index c2ea0cd..416564a 100644 --- a/parsedmarc/Dockerfile +++ b/parsedmarc/Dockerfile @@ -1,9 +1,6 @@ -FROM python:3.9-alpine3.16 +FROM python:3.13-alpine RUN apk add --update --no-cache libxml2-dev libxslt-dev RUN apk add --update --no-cache --virtual .build_deps build-base libffi-dev \ && pip install parsedmarc \ && apk del .build_deps - -COPY parsedmarc.ini / -#COPY GeoLite2-Country.mmdb /usr/share/GeoIP/GeoLite2-Country.mmdb diff --git a/parsedmarc/parsedmarc.sample.ini b/parsedmarc/parsedmarc.ini similarity index 89% rename from parsedmarc/parsedmarc.sample.ini rename to parsedmarc/parsedmarc.ini index 454a6c6..809931d 100644 --- a/parsedmarc/parsedmarc.sample.ini +++ b/parsedmarc/parsedmarc.ini @@ -1,7 +1,8 @@ [general] +offline = True +output = /output/ save_aggregate = True save_forensic = True -output = /output/ [elasticsearch] hosts = elasticsearch:9200 From 105347ed723dec9186553977d55c6a8bf0220f8f Mon Sep 17 00:00:00 2001 From: Sebastian Neuser Date: Mon, 5 May 2025 17:09:01 +0200 Subject: [PATCH 4/5] qol: Add output directories to repo to ensure sane folder permissions --- .gitignore | 9 ++++++--- docker-compose.yml | 2 +- elastic_data/.gitkeep | 0 input_files/.gitkeep | 0 output_files/.gitkeep | 0 5 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 elastic_data/.gitkeep create mode 100644 input_files/.gitkeep create mode 100644 output_files/.gitkeep diff --git a/.gitignore b/.gitignore index 9aa7d22..1e70852 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ -/files/ -/output_files/ -/elastic_data/ +/elastic_data/* +!/elastic_data/.gitkeep +/input_files/* +!/input_files/.gitkeep +/output_files/* +!/output_files/.gitkeep diff --git a/docker-compose.yml b/docker-compose.yml index 8f21a4a..eadb7c3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ services: parsedmarc: build: ./parsedmarc volumes: - - ./files:/input:ro + - ./input_files:/input:ro - ./output_files:/output - ./parsedmarc/parsedmarc.ini:/parsedmarc.ini command: parsedmarc -c /parsedmarc.ini /input/* diff --git a/elastic_data/.gitkeep b/elastic_data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/input_files/.gitkeep b/input_files/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/output_files/.gitkeep b/output_files/.gitkeep new file mode 100644 index 0000000..e69de29 From 907bbbfbc2f0834f867b143c4b8f03cde3481ea2 Mon Sep 17 00:00:00 2001 From: Sebastian Neuser Date: Mon, 5 May 2025 19:39:53 +0200 Subject: [PATCH 5/5] impr: Use latest Grafana; move plugin install to Dockerfile --- docker-compose.yml | 6 +----- grafana/Dockerfile | 18 ++++++++++++++---- .../dashboards/dashboards.yml} | 0 .../datasources/all.yml | 0 4 files changed, 15 insertions(+), 9 deletions(-) rename grafana/{grafana-provisioning/dashboards/all.yml => provisioning/dashboards/dashboards.yml} (100%) rename grafana/{grafana-provisioning => provisioning}/datasources/all.yml (100%) diff --git a/docker-compose.yml b/docker-compose.yml index eadb7c3..b160a2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,10 +25,6 @@ services: restart: on-failure grafana: - build: ./grafana/ + build: ./grafana ports: - 3000:3000 - user: root - environment: - GF_INSTALL_PLUGINS: grafana-piechart-panel,grafana-worldmap-panel - GF_AUTH_ANONYMOUS_ENABLED: 'true' diff --git a/grafana/Dockerfile b/grafana/Dockerfile index 6d7d2fd..ec820ea 100644 --- a/grafana/Dockerfile +++ b/grafana/Dockerfile @@ -1,6 +1,16 @@ -FROM grafana/grafana:8.5.4 +FROM grafana/grafana-oss -ADD --chown=grafana:root https://raw.githubusercontent.com/domainaware/parsedmarc/master/grafana/Grafana-DMARC_Reports.json /var/lib/grafana/dashboards/ -RUN chmod 644 /etc/grafana/provisioning +# Install dashboard and dependencies +ARG DMARC_DASHBOARD=https://raw.githubusercontent.com/domainaware/parsedmarc/master/grafana/Grafana-DMARC_Reports.json +ADD --chown=grafana:root ${DMARC_DASHBOARD} /var/lib/grafana/dashboards/ +RUN grafana-cli --pluginsDir "${GF_PATHS_PLUGINS}" plugins install grafana-piechart-panel +RUN grafana-cli --pluginsDir "${GF_PATHS_PLUGINS}" plugins install grafana-worldmap-panel -COPY grafana-provisioning/ /etc/grafana/provisioning/ +# Copy provisioning files and fix permissions +COPY --chown=grafana:root provisioning/ /etc/grafana/provisioning/ +USER root +RUN chmod -R +X /etc/grafana/provisioning +USER grafana + +# Configure Grafana +ENV GF_AUTH_ANONYMOUS_ENABLED=true diff --git a/grafana/grafana-provisioning/dashboards/all.yml b/grafana/provisioning/dashboards/dashboards.yml similarity index 100% rename from grafana/grafana-provisioning/dashboards/all.yml rename to grafana/provisioning/dashboards/dashboards.yml diff --git a/grafana/grafana-provisioning/datasources/all.yml b/grafana/provisioning/datasources/all.yml similarity index 100% rename from grafana/grafana-provisioning/datasources/all.yml rename to grafana/provisioning/datasources/all.yml