From 01a51efb17190e644cb8e28f064d8f8f1ae3b76c Mon Sep 17 00:00:00 2001 From: NUZAT-TABASSUM Date: Mon, 4 Aug 2025 12:51:40 +0200 Subject: [PATCH 1/4] adding database backup feature --- .yamllint | 31 ++++++++++-- defaults/main.yml | 11 +++++ molecule/default/converge.yml | 3 +- molecule/default/verify.yml | 56 ++++++++++++++++++++++ tasks/backup.yml | 71 ++++++++++++++++++++++++++++ tasks/main.yml | 7 +++ templates/database-backup.service.j2 | 14 ++++++ templates/database-backup.sh.j2 | 22 +++++++++ templates/database-backup.timer.j2 | 9 ++++ 9 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 tasks/backup.yml create mode 100644 templates/database-backup.service.j2 create mode 100644 templates/database-backup.sh.j2 create mode 100644 templates/database-backup.timer.j2 diff --git a/.yamllint b/.yamllint index d5530ae..fc6c772 100644 --- a/.yamllint +++ b/.yamllint @@ -1,11 +1,36 @@ --- +# Yamllint configuration should be compatible with Ansible, +# see https://ansible.readthedocs.io/projects/lint/rules/yaml/#yamllint-configuration extends: default rules: - line-length: disable + comments: + # https://github.com/prettier/prettier/issues/6780 + min-spaces-from-content: 1 + # https://github.com/adrienverge/yamllint/issues/384 + comments-indentation: false + document-start: disable + # 160 chars was the default used by old E204 rule, but + # you can easily change it or disable in your .yamllint file. + line-length: + max: 200 + # We are adding an extra space inside braces as that's how prettier does it + # and we are trying not to fight other linters. + braces: + min-spaces-inside: 0 # yamllint defaults to 0 + max-spaces-inside: 1 # yamllint defaults to 0 + # key-duplicates: + # forbid-duplicated-merge-keys: true # not enabled by default + octal-values: + forbid-implicit-octal: true # yamllint defaults to false + forbid-explicit-octal: true # yamllint defaults to false + # quoted-strings: + # quote-type: double + # required: only-when-needed + ignore: | - venv/ - .roles/ .cache/ + .github/ + venv/ diff --git a/defaults/main.yml b/defaults/main.yml index 0b9bbfd..5ff2f55 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -10,3 +10,14 @@ database_root_user: root opencast_mariadb_max_connections: 512 # set buffer pool size to ~80% of total memory if db runs on its own host opencast_mariadb_innodb_buffer_pool_size: "{{ ((ansible_memtotal_mb / 1024) * 0.8) | int }}G" + + +# === Database backup feature (enabled) === +database_backup_enabled: false +database_backup_output_path: None +database_backup_schedule: "*-*-* 05:00:00" # Systemd OnCalendar format +database_backup_keep: 7 +database_backup_dbs: [] # list of databases +database_backup_owner: backup +database_backup_group: backup +database_backup_user: backup diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 50f0e8f..7714f2e 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -4,7 +4,8 @@ vars: database_password: "1234" database_root_password: "5678" + database_backup_user_password: "5432" tasks: - name: Include opencast_mariadb ansible.builtin.include_role: - name: opencast_mariadb + name: elan.opencast_mariadb diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index 950871c..ce03420 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -3,6 +3,8 @@ hosts: all vars: database_password: "1234" + vars_files: + - ../../defaults/main.yml tasks: - name: Check MariaDB socket exists ansible.builtin.wait_for: @@ -42,3 +44,57 @@ ansible.builtin.fail: msg: Opencast database does not exist when: "query_databases.rowcount.0 != 1" + + # ─────────────────────────────────────────────────────────── + # Backup configuration (only when enabled) + # ─────────────────────────────────────────────────────────── + + - name: Verify backup configuration when enabled + when: database_backup_enabled | default(false) + block: + - name: Assert backups are enabled + ansible.builtin.assert: + that: + - database_backup_enabled | default(false) + fail_msg: "Backups are disabled; skipping backup verification." + + - name: Ensure backup directory exists + ansible.builtin.stat: + path: "{{ database_backup_output_path }}" + register: backup_dir_stat + + - name: Assert backup directory is present and writable + ansible.builtin.assert: + that: + - backup_dir_stat.stat.exists + - backup_dir_stat.stat.isdir + fail_msg: > + Backup directory {{ database_backup_output_path }} + is missing or not a directory. + - name: Check database-backup.service is installed and enabled + ansible.builtin.systemd: + name: database-backup.service + enabled: true + state: started + + - name: Check database-backup.timer is installed and enabled + ansible.builtin.systemd: + name: database-backup.timer + enabled: true + state: started + + - name: Slurp timer unit file for inspection + ansible.builtin.slurp: + path: /etc/systemd/system/database-backup.timer + register: timer_unit + + - name: Assert OnCalendar line in timer unit matches schedule + ansible.builtin.assert: + that: + - "'OnCalendar={{ database_backup_schedule }}' in (timer_unit.content | b64decode)" + fail_msg: > + database-backup.timer does not contain + OnCalendar={{ database_backup_schedule }}. + success_msg: > + Timer unit file correctly contains + OnCalendar={{ database_backup_schedule }}. diff --git a/tasks/backup.yml b/tasks/backup.yml new file mode 100644 index 0000000..2f76374 --- /dev/null +++ b/tasks/backup.yml @@ -0,0 +1,71 @@ +--- +- name: Fail if backup enabled but no output path given + ansible.builtin.fail: + msg: "database_backup_output_path must be set when database_backup_enabled = true" + when: + - database_backup_enabled + - database_backup_output_path | length == 0 + +- name: Ensure backup OS user exists + ansible.builtin.user: + name: "{{ database_backup_owner }}" + state: present + shell: /usr/sbin/nologin + system: true + +- name: Ensure MariaDB backup user exists for each database + community.mysql.mysql_user: + name: "{{ database_backup_user }}" + password: "{{ database_backup_user_password }}" + host: "localhost" + priv: "{{ item }}.*:SELECT,LOCK TABLES,SHOW VIEW,EVENT,TRIGGER" + state: present + login_user: "{{ database_root_user }}" + login_password: "{{ database_root_password }}" + loop: "{{ database_backup_dbs }}" + when: database_backup_enabled + no_log: true + +- name: Ensure backup output directory exists + ansible.builtin.file: + path: "{{ database_backup_output_path }}" + state: directory + owner: "{{ database_backup_owner }}" + group: "{{ database_backup_group }}" + mode: "0750" + when: database_backup_enabled + +- name: Install backup script + ansible.builtin.template: + src: database-backup.sh.j2 + dest: "{{ database_backup_output_path }}/database-backup.sh" + owner: "{{ database_backup_owner }}" + group: "{{ database_backup_group }}" + mode: "0750" + when: database_backup_enabled + +- name: Install systemd service unit + ansible.builtin.template: + src: database-backup.service.j2 + dest: /etc/systemd/system/database-backup.service + mode: "0644" + when: database_backup_enabled + +- name: Install systemd timer unit + ansible.builtin.template: + src: database-backup.timer.j2 + dest: /etc/systemd/system/database-backup.timer + mode: "0644" + when: database_backup_enabled + +- name: Reload systemd daemon (if timers changed) + ansible.builtin.systemd: + daemon_reload: true + when: database_backup_enabled + +- name: Ensure backup timer is enabled and running + ansible.builtin.systemd: + name: database-backup.timer + enabled: true + state: started + when: database_backup_enabled diff --git a/tasks/main.yml b/tasks/main.yml index e1e246b..1a45ae2 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -132,3 +132,10 @@ - "localhost" - "{{ ansible_fqdn }}" no_log: true + +############################################################################### +# database backup +############################################################################### +- name: Include backup setup tasks + ansible.builtin.include_tasks: backup.yml + when: database_backup_enabled diff --git a/templates/database-backup.service.j2 b/templates/database-backup.service.j2 new file mode 100644 index 0000000..62e8c72 --- /dev/null +++ b/templates/database-backup.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Opencast Database Backup +After=network.target +After=local-fs.target +After=remote-fs.target + +[Service] +Type=oneshot +User={{ database_backup_owner }} +Group={{ database_backup_group }} +ExecStart={{ database_backup_output_path }}/database-backup.sh + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/templates/database-backup.sh.j2 b/templates/database-backup.sh.j2 new file mode 100644 index 0000000..ca51701 --- /dev/null +++ b/templates/database-backup.sh.j2 @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +DBUSER="{{ database_backup_user }}" +DBPASS="{{ database_backup_user_password }}" +OUTDIR="{{ database_backup_output_path }}" +KEEP={{ database_backup_keep }} +DBS=({% for db in database_backup_dbs %}{{ db }} {% endfor %}) +TS=$(date +%Y%m%d-%H%M%S) + +# Loop through each database name +for DB in "${DBS[@]}"; do + echo "Backing up $DB → $OUTDIR/db-backup-${DB}-${TS}.dump.gz" + + # Run pg_dump and compress into a .gz file + mysqldump -F c "$DBUSER" -p"$DBPASS" \ + | gzip > "${OUTDIR}/db-backup-${DB}-${TS}.dump.gz" + + # Remove older dumps, keep only the newest $KEEP + ls -1t "${OUTDIR}/db-backup-${DB}-"*.dump.gz \ + | tail -n +$((KEEP + 1)) \ + | xargs -r rm -- +done \ No newline at end of file diff --git a/templates/database-backup.timer.j2 b/templates/database-backup.timer.j2 new file mode 100644 index 0000000..05ac1f9 --- /dev/null +++ b/templates/database-backup.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Run database backup daily + +[Timer] +OnCalendar={{ database_backup_schedule }} +Persistent=true + +[Install] +WantedBy=timers.target \ No newline at end of file From b4f97436c622d97d2679590503b4422d94316c74 Mon Sep 17 00:00:00 2001 From: NUZAT-TABASSUM Date: Wed, 26 Nov 2025 13:13:46 +0100 Subject: [PATCH 2/4] improving backup script --- .ansible/roles/elan.opencast_mariadb | 1 + README.md | 43 ++++++++++++++++++++++++++++ defaults/main.yml | 13 +++++---- molecule/default/converge.yml | 7 ++--- molecule/default/verify.yml | 1 + tasks/backup.yml | 22 ++++++++++++-- templates/database-backup.sh.j2 | 10 +++---- templates/database-restore.sh.j2 | 39 +++++++++++++++++++++++++ 8 files changed, 119 insertions(+), 17 deletions(-) create mode 120000 .ansible/roles/elan.opencast_mariadb create mode 100644 templates/database-restore.sh.j2 diff --git a/.ansible/roles/elan.opencast_mariadb b/.ansible/roles/elan.opencast_mariadb new file mode 120000 index 0000000..7244198 --- /dev/null +++ b/.ansible/roles/elan.opencast_mariadb @@ -0,0 +1 @@ +/home/tabassum/my_work/maria_db/opencast_mariadb \ No newline at end of file diff --git a/README.md b/README.md index 336cf2f..1a70bc7 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,49 @@ If you want to inspect a running test instance use `molecule login --host -.sql.gz` + +### Backup variables + +- `database_backup_enabled` (default: `false`) + - Enable/disable the backup feature. + +- `database_backup_output_path` (default: empty) + - **Required when** `database_backup_enabled: true`. + - Directory where the scripts and backup files are stored (e.g. `/var/backups/mariadb`). + +- `database_backup_schedule` (default: `*-*-* 05:00:00`) + - systemd `OnCalendar` schedule for the timer. + +- `database_backup_keep` (default: `7`) + - Number of newest backups to keep **per database**. Older dumps are removed. + +- `database_backup_dbs` + - List of database names to back up (e.g. `["opencast"]`). + +- `database_backup_owner` / `database_backup_group` + - OS user/group that owns `database_backup_output_path` and the scripts. + +- `database_backup_user` / `database_backup_user_password` + - MariaDB user/password used by the backup script (`mysqldump`). + - Should have read-only style privileges required for backups (e.g. `SELECT, LOCK TABLES, SHOW VIEW, EVENT, TRIGGER`). + +### Restore script + +A restore script is installed into the same output directory (next to the backup script). +It is meant to be used manually when you really want to restore a dump. + +Usage: +```bash +/database-restore.sh + ## License [BSD-3-Clause](LICENSE) diff --git a/defaults/main.yml b/defaults/main.yml index 5ff2f55..801b18a 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -13,11 +13,14 @@ opencast_mariadb_innodb_buffer_pool_size: "{{ ((ansible_memtotal_mb / 1024) * 0. # === Database backup feature (enabled) === -database_backup_enabled: false -database_backup_output_path: None +database_backup_enabled: true +database_backup_output_path: "/bckup/" database_backup_schedule: "*-*-* 05:00:00" # Systemd OnCalendar format database_backup_keep: 7 -database_backup_dbs: [] # list of databases -database_backup_owner: backup -database_backup_group: backup +# list of databases +database_backup_dbs: + # - "{{ opencast }}" + - "opencast" +database_backup_owner: mariadbbackup +database_backup_group: mariadbbackup database_backup_user: backup diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 7714f2e..1f678ea 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -1,11 +1,10 @@ --- - name: Converge hosts: all + # become: true vars: database_password: "1234" database_root_password: "5678" database_backup_user_password: "5432" - tasks: - - name: Include opencast_mariadb - ansible.builtin.include_role: - name: elan.opencast_mariadb + roles: + - elan.opencast_mariadb diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index ce03420..0efcf2c 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -71,6 +71,7 @@ fail_msg: > Backup directory {{ database_backup_output_path }} is missing or not a directory. + - name: Check database-backup.service is installed and enabled ansible.builtin.systemd: name: database-backup.service diff --git a/tasks/backup.yml b/tasks/backup.yml index 2f76374..332db3e 100644 --- a/tasks/backup.yml +++ b/tasks/backup.yml @@ -10,15 +10,17 @@ ansible.builtin.user: name: "{{ database_backup_owner }}" state: present - shell: /usr/sbin/nologin system: true + when: + - database_backup_enabled + - database_backup_user != "root" - name: Ensure MariaDB backup user exists for each database community.mysql.mysql_user: name: "{{ database_backup_user }}" password: "{{ database_backup_user_password }}" host: "localhost" - priv: "{{ item }}.*:SELECT,LOCK TABLES,SHOW VIEW,EVENT,TRIGGER" + priv: "{{ item }}.*:SELECT,LOCK TABLES,SHOW VIEW,TRIGGER" state: present login_user: "{{ database_root_user }}" login_password: "{{ database_root_password }}" @@ -50,6 +52,7 @@ dest: /etc/systemd/system/database-backup.service mode: "0644" when: database_backup_enabled + register: database_backup_service_unit - name: Install systemd timer unit ansible.builtin.template: @@ -57,11 +60,14 @@ dest: /etc/systemd/system/database-backup.timer mode: "0644" when: database_backup_enabled + register: database_backup_timer_unit - name: Reload systemd daemon (if timers changed) ansible.builtin.systemd: daemon_reload: true - when: database_backup_enabled + when: + - database_backup_enabled + - database_backup_service_unit.changed or database_backup_timer_unit.changed - name: Ensure backup timer is enabled and running ansible.builtin.systemd: @@ -69,3 +75,13 @@ enabled: true state: started when: database_backup_enabled + + # Restore database +- name: Install restore script + ansible.builtin.template: + src: database-restore.sh.j2 + dest: "{{ database_backup_output_path }}/database-restore.sh" + owner: "{{ database_backup_owner }}" + group: "{{ database_backup_group }}" + mode: "0750" + when: database_backup_enabled diff --git a/templates/database-backup.sh.j2 b/templates/database-backup.sh.j2 index ca51701..9fa5d0e 100644 --- a/templates/database-backup.sh.j2 +++ b/templates/database-backup.sh.j2 @@ -4,19 +4,19 @@ DBUSER="{{ database_backup_user }}" DBPASS="{{ database_backup_user_password }}" OUTDIR="{{ database_backup_output_path }}" KEEP={{ database_backup_keep }} -DBS=({% for db in database_backup_dbs %}{{ db }} {% endfor %}) +DBS=({{ database_backup_dbs | join(' ') }}) TS=$(date +%Y%m%d-%H%M%S) # Loop through each database name for DB in "${DBS[@]}"; do - echo "Backing up $DB → $OUTDIR/db-backup-${DB}-${TS}.dump.gz" + echo "Backing up $DB database to $OUTDIR/db-backup-${DB}-${TS}.sql.gz" # Run pg_dump and compress into a .gz file - mysqldump -F c "$DBUSER" -p"$DBPASS" \ - | gzip > "${OUTDIR}/db-backup-${DB}-${TS}.dump.gz" + mysqldump -u "$DBUSER" -p"$DBPASS" "$DB" \ + | gzip > "${OUTDIR}/db-backup-${DB}-${TS}.sql.gz" # Remove older dumps, keep only the newest $KEEP - ls -1t "${OUTDIR}/db-backup-${DB}-"*.dump.gz \ + ls -1t "${OUTDIR}/db-backup-${DB}-"*.sql.gz \ | tail -n +$((KEEP + 1)) \ | xargs -r rm -- done \ No newline at end of file diff --git a/templates/database-restore.sh.j2 b/templates/database-restore.sh.j2 new file mode 100644 index 0000000..53c6697 --- /dev/null +++ b/templates/database-restore.sh.j2 @@ -0,0 +1,39 @@ +#!/bin/bash + +# Usage: ./database-restore.sh + +DB_NAME=$1 +BACKUP_FILE=$2 + +if [ -z "$DB_NAME" ] || [ -z "$BACKUP_FILE" ]; then + echo "Usage: $0 " + exit 1 +fi + +if [ ! -f "$BACKUP_FILE" ]; then + echo "Error: Backup file $BACKUP_FILE does not exist." + exit 2 +fi + +DB_USER="{{ database_root_user }}" +DB_PASS="{{ database_root_password }}" + +echo "Restoring database $DB_NAME from $BACKUP_FILE..." + +# Drop and recreate the database +mysql -u"$DB_USER" -p"$DB_PASS" -e "DROP DATABASE IF EXISTS \`$DB_NAME\`; CREATE DATABASE \`$DB_NAME\`;" + +# Import the backup file +if [[ "$BACKUP_FILE" == *.gz ]]; then + gunzip -c "$BACKUP_FILE" | mysql -u"$DB_USER" "-p$DB_PASS" "$DB_NAME" +else + mysql -u"$DB_USER" "-p$DB_PASS" "$DB_NAME" < "$BACKUP_FILE" +fi + + +if [ $? -eq 0 ]; then + echo "Restore completed successfully." +else + echo "Restore failed." + exit 3 +fi \ No newline at end of file From 5b7f6df213a83e28adb6f65216c4cba9d2abd178 Mon Sep 17 00:00:00 2001 From: NUZAT-TABASSUM Date: Wed, 26 Nov 2025 13:43:26 +0100 Subject: [PATCH 3/4] fix issues --- README.md | 28 +++++++--------------------- defaults/main.yml | 8 ++++---- molecule/default/converge.yml | 1 - 3 files changed, 11 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 1a70bc7..d173fdf 100644 --- a/README.md +++ b/README.md @@ -67,28 +67,14 @@ When enabled, backups are written as compressed dumps: ### Backup variables -- `database_backup_enabled` (default: `false`) - - Enable/disable the backup feature. - -- `database_backup_output_path` (default: empty) - - **Required when** `database_backup_enabled: true`. - - Directory where the scripts and backup files are stored (e.g. `/var/backups/mariadb`). - -- `database_backup_schedule` (default: `*-*-* 05:00:00`) - - systemd `OnCalendar` schedule for the timer. - +- `database_backup_enabled` (default: `false`) - Enable/disable the backup feature. +- `database_backup_output_path` (default: empty) - **Required when** `database_backup_enabled: true`.Directory where the scripts and backup files are stored (e.g. `/var/backups/mariadb`). +- `database_backup_schedule` (default: `*-*-* 05:00:00`) - systemd `OnCalendar` schedule for the timer. - `database_backup_keep` (default: `7`) - Number of newest backups to keep **per database**. Older dumps are removed. - -- `database_backup_dbs` - - List of database names to back up (e.g. `["opencast"]`). - -- `database_backup_owner` / `database_backup_group` - - OS user/group that owns `database_backup_output_path` and the scripts. - -- `database_backup_user` / `database_backup_user_password` - - MariaDB user/password used by the backup script (`mysqldump`). - - Should have read-only style privileges required for backups (e.g. `SELECT, LOCK TABLES, SHOW VIEW, EVENT, TRIGGER`). +- `database_backup_dbs`- List of database names to back up (e.g. `["opencast"]`). +- `database_backup_owner` / `database_backup_group`- OS user/group that owns `database_backup_output_path` and the scripts. +- `database_backup_user` / `database_backup_user_password`- MariaDB user/password used by the backup script. ### Restore script @@ -98,7 +84,7 @@ It is meant to be used manually when you really want to restore a dump. Usage: ```bash /database-restore.sh - +``` ## License [BSD-3-Clause](LICENSE) diff --git a/defaults/main.yml b/defaults/main.yml index 801b18a..1665f52 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -13,14 +13,14 @@ opencast_mariadb_innodb_buffer_pool_size: "{{ ((ansible_memtotal_mb / 1024) * 0. # === Database backup feature (enabled) === -database_backup_enabled: true -database_backup_output_path: "/bckup/" +database_backup_enabled: false +database_backup_output_path: "" database_backup_schedule: "*-*-* 05:00:00" # Systemd OnCalendar format database_backup_keep: 7 # list of databases database_backup_dbs: - # - "{{ opencast }}" - - "opencast" + - "{{ database_name }}" + database_backup_owner: mariadbbackup database_backup_group: mariadbbackup database_backup_user: backup diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 1f678ea..fbbd6f9 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -1,7 +1,6 @@ --- - name: Converge hosts: all - # become: true vars: database_password: "1234" database_root_password: "5678" From 1c7b3107b688bcbfd80029870f535fddc7054a75 Mon Sep 17 00:00:00 2001 From: NUZAT-TABASSUM Date: Wed, 26 Nov 2025 13:49:53 +0100 Subject: [PATCH 4/4] fix lint issues --- defaults/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 1665f52..51a1fc1 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -20,7 +20,6 @@ database_backup_keep: 7 # list of databases database_backup_dbs: - "{{ database_name }}" - database_backup_owner: mariadbbackup database_backup_group: mariadbbackup database_backup_user: backup