diff --git a/.gitignore b/.gitignore index baf53f3..29f0eab 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ huly_v7.conf .cr.secret .rp.secret .idea +backup-offsite.env +backups/ diff --git a/README.md b/README.md index 5aae414..2cd9fe0 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,20 @@ To update an existing self-hosted deployment to a new Huly version: docker compose up -d ``` +## Backups + +Use the `backup.sh` and `restore.sh` helpers to back up and restore a Docker Compose +deployment (database, file store, and config). See +[`guides/backup-restore.md`](guides/backup-restore.md) for details. + +> [!IMPORTANT] +> Take a backup **before** any version upgrade. + +```bash +./backup.sh # snapshot to ./backups/huly-backup-/ +./restore.sh backups/huly-backup- +``` + ## Disable default content in new workspaces By default, Huly can initialize new workspaces with predefined content. To disable that behavior, set `INIT_REPO_DIR` in the `workspace` service to a non-existing path: diff --git a/backup-offsite.env.example b/backup-offsite.env.example new file mode 100644 index 0000000..d99bd30 --- /dev/null +++ b/backup-offsite.env.example @@ -0,0 +1,21 @@ +# backup-offsite.env.example +# +# Offsite target for `./backup.sh --offsite`. Copy this to `backup-offsite.env` +# (which is gitignored) and fill in your S3-compatible storage details. Works with +# Backblaze B2, DigitalOcean Spaces, AWS S3, Wasabi, MinIO, etc. +# +# A backup that lives only on the same host as the deployment is not disaster +# recovery. `--offsite` pushes each completed backup to this bucket via rclone. + +BACKUP_S3_ENDPOINT=https://s3.us-west-000.backblazeb2.com +BACKUP_S3_REGION=us-west-000 +BACKUP_S3_BUCKET=my-huly-backups +BACKUP_S3_PATH_PREFIX=huly +BACKUP_S3_ACCESS_KEY=your-access-key-id +BACKUP_S3_SECRET_KEY=your-secret-key + +# Optional: rclone S3 provider (Other works for most; AWS, Minio, Wasabi, etc. also valid). +# BACKUP_S3_PROVIDER=Other + +# Retention on the remote is best handled by a bucket lifecycle policy (expire objects +# after N days), rather than by this script. diff --git a/backup.sh b/backup.sh new file mode 100755 index 0000000..e48d8ed --- /dev/null +++ b/backup.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# +# backup.sh - Offline backup of a Docker Compose deployment. +# +# Creates a timestamped, self-contained backup of everything needed to restore the +# deployment: the CockroachDB data, the MinIO file store, and the local config and +# secret files. The stack is stopped only while the volumes are archived so the copy +# is crash-consistent, then restarted immediately. This is the recommended backup to +# take before a version upgrade (see MIGRATION.md). +# +# The search index (elastic) and event log (redpanda) are skipped by default: they +# are rebuilt automatically and are not required to restore. Use --full to include +# them. +# +# Usage: ./backup.sh [--output=DIR] [--keep=N] [--full] [--offsite] [--help] + +set -euo pipefail + +cd "$(dirname "$0")" + +OUTPUT_DIR="./backups" +KEEP=0 +FULL=false +OFFSITE=false + +for arg in "$@"; do + case $arg in + --output=*) OUTPUT_DIR="${arg#*=}" ;; + --keep=*) KEEP="${arg#*=}" ;; + --full) FULL=true ;; + --offsite) OFFSITE=true ;; + --help) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --output=DIR Directory to write backups into (default: ./backups)" + echo " --keep=N Keep only the N most recent backups (default: keep all)" + echo " --full Also back up the search index (elastic) and event log (redpanda)" + echo " --offsite After the local backup, upload it to S3-compatible storage (see backup-offsite.env.example)" + echo " --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $arg" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +if docker compose version >/dev/null 2>&1; then + COMPOSE="docker compose" +elif command -v docker-compose >/dev/null 2>&1; then + COMPOSE="docker-compose" +else + echo "Error: docker compose is not available." >&2 + exit 1 +fi + +# Always bring the stack back up, even if archiving fails partway through, so a +# failed backup can never leave the deployment stopped. +STOPPED=false +restart_stack() { + if [ "$STOPPED" = true ]; then + echo "Ensuring the stack is running again..." + $COMPOSE start || echo -e "\033[31mWARNING: could not restart the stack - check 'docker compose ps'.\033[0m" + STOPPED=false + fi +} +trap restart_stack EXIT + +# Print the host source (named volume or bind path) backing the given mount +# destination on a specific container, or nothing. +source_on() { + # $1 = container id, $2 = mount destination + docker inspect "$1" --format \ + "{{range .Mounts}}{{if eq .Destination \"$2\"}}{{if .Name}}{{.Name}}{{else}}{{.Source}}{{end}}{{end}}{{end}}" 2>/dev/null || true +} + +# Resolve the volume backing a mount destination. Try the expected service name +# first; if that service does not exist (deployments differ, e.g. "cockroach" vs +# "cockroachdb"), scan every container in the project for the destination. +# Always exits 0 (prints nothing when not found) so it is safe under `set -e`. +mount_source() { + # $1 = expected service name, $2 = mount destination + local cid src + cid=$($COMPOSE ps -aq "$1" 2>/dev/null | head -1 || true) + if [ -n "$cid" ]; then + src=$(source_on "$cid" "$2") + [ -n "$src" ] && { echo "$src"; return 0; } + fi + for cid in $($COMPOSE ps -aq 2>/dev/null || true); do + src=$(source_on "$cid" "$2") + [ -n "$src" ] && { echo "$src"; return 0; } + done + return 0 +} + +# Archive a volume or bind path into the backup directory. +archive() { + # $1 = source (volume name or host path), $2 = output tar name, $3 = label + if [ -z "$1" ]; then + echo -e " \033[33mskipping $3 (not found)\033[0m" + return 0 + fi + echo " - $3 -> $2" + docker run --rm -v "$1":/data:ro -v "$DEST_ABS":/backup alpine \ + tar czf "/backup/$2" -C /data . +} + +STAMP=$(date +%Y%m%d-%H%M%S) +DEST="$OUTPUT_DIR/huly-backup-$STAMP" +mkdir -p "$DEST/config" +DEST_ABS=$(cd "$DEST" && pwd) + +echo -e "\033[1;34mResolving data volumes...\033[0m" +CR_SRC=$(mount_source cockroach /cockroach/cockroach-data) +FILES_SRC=$(mount_source minio /data) +MONGO_SRC=$(mount_source mongodb /data/db) +ELASTIC_SRC=$(mount_source elastic /usr/share/elasticsearch/data) +REDPANDA_SRC=$(mount_source redpanda /var/lib/redpanda/data) + +echo "Stopping stack for a consistent snapshot..." +$COMPOSE stop +STOPPED=true + +echo "Archiving data volumes..." +archive "$CR_SRC" cockroach.tar.gz "CockroachDB" +archive "$FILES_SRC" files.tar.gz "MinIO files" +[ -n "$MONGO_SRC" ] && archive "$MONGO_SRC" mongodb.tar.gz "MongoDB (legacy)" +if [ "$FULL" = true ]; then + archive "$ELASTIC_SRC" elastic.tar.gz "Elasticsearch index" + archive "$REDPANDA_SRC" redpanda.tar.gz "Redpanda log" +fi + +# Bring the stack back up as soon as the volumes are archived; the rest of the work +# (copying config, manifest, pruning) does not need the stack stopped. +restart_stack + +echo "Copying config and secret files..." +for f in .env huly.conf huly_v7.conf nginx.conf .huly.secret .cr.secret .rp.secret; do + [ -f "$f" ] && cp -p "$f" "$DEST/config/" +done +[ -d traefik ] && cp -rp traefik "$DEST/config/" + +{ + echo "created: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "full: $FULL" + grep -hE '^HULY_VERSION=' .env huly.conf huly_v7.conf 2>/dev/null | tail -1 || true + echo "archives:" + for a in "$DEST"/*.tar.gz; do + [ -f "$a" ] && echo " - $(basename "$a")" + done +} > "$DEST/manifest.txt" + +if [ "$KEEP" -gt 0 ]; then + echo "Pruning old backups, keeping $KEEP..." + # Timestamped names sort chronologically, and bash expands globs lexically, + # so this array is oldest-first. + shopt -s nullglob + existing=("$OUTPUT_DIR"/huly-backup-*/) + shopt -u nullglob + remove=$((${#existing[@]} - KEEP)) + if [ "$remove" -gt 0 ]; then + for ((i = 0; i < remove; i++)); do + echo " removing ${existing[i]}" + rm -rf "${existing[i]}" + done + fi +fi + +# Optional: push the completed backup to S3-compatible offsite storage via rclone. +# A backup that lives only on the same host is not disaster recovery. Configure via +# environment or a gitignored backup-offsite.env (see backup-offsite.env.example). +if [ "$OFFSITE" = true ]; then + # shellcheck source=/dev/null + if [ -f ./backup-offsite.env ]; then . ./backup-offsite.env; fi + : "${BACKUP_S3_BUCKET:?--offsite needs BACKUP_S3_BUCKET (set env or create backup-offsite.env)}" + : "${BACKUP_S3_ACCESS_KEY:?--offsite needs BACKUP_S3_ACCESS_KEY}" + : "${BACKUP_S3_SECRET_KEY:?--offsite needs BACKUP_S3_SECRET_KEY}" + : "${BACKUP_S3_ENDPOINT:?--offsite needs BACKUP_S3_ENDPOINT}" + PREFIX="${BACKUP_S3_PATH_PREFIX:-huly}" + DEST_NAME=$(basename "$DEST") + echo "Uploading backup offsite to s3://${BACKUP_S3_BUCKET}/${PREFIX}/${DEST_NAME}/ ..." + if docker run --rm -v "$DEST_ABS":/data:ro \ + -e RCLONE_S3_PROVIDER="${BACKUP_S3_PROVIDER:-Other}" \ + -e RCLONE_S3_ENV_AUTH=false \ + -e RCLONE_S3_ACCESS_KEY_ID="$BACKUP_S3_ACCESS_KEY" \ + -e RCLONE_S3_SECRET_ACCESS_KEY="$BACKUP_S3_SECRET_KEY" \ + -e RCLONE_S3_ENDPOINT="$BACKUP_S3_ENDPOINT" \ + -e RCLONE_S3_REGION="${BACKUP_S3_REGION:-us-east-1}" \ + rclone/rclone copy /data ":s3:${BACKUP_S3_BUCKET}/${PREFIX}/${DEST_NAME}/" -v; then + echo -e "\033[1;32mOffsite upload complete.\033[0m Set a bucket lifecycle policy for offsite retention." + else + echo -e "\033[31mWARNING: offsite upload failed - the local backup at $DEST is intact.\033[0m" + fi +fi + +echo -e "\033[1;32mBackup complete: $DEST\033[0m" +du -sh "$DEST" | awk '{print "Total size: " $1}' diff --git a/guides/backup-restore.md b/guides/backup-restore.md new file mode 100644 index 0000000..9b68a31 --- /dev/null +++ b/guides/backup-restore.md @@ -0,0 +1,101 @@ +# Backup and Restore (Docker Compose) + +This guide covers backing up and restoring a Docker Compose deployment. For +Kubernetes, the Helm chart ships scheduled backups under +`helm/huly/templates/backup/` instead. + +> [!IMPORTANT] +> Always take a backup **before upgrading** to a new version. See `MIGRATION.md`. + +## What gets backed up + +| Data | Source | Included by default | +|---|---|---| +| Database | CockroachDB volume (`cr_data`) | Yes | +| Files / attachments | MinIO volume (`files`) | Yes | +| Config and secrets | `.env`, `huly*.conf`, `nginx.conf`, `.huly.secret`, `.cr.secret`, `.rp.secret`, `traefik/` | Yes | +| Legacy database | MongoDB volume (`mongodb`), if present | Yes, when detected | +| Search index | Elasticsearch volume (`elastic`) | No - rebuilt automatically | +| Event log | Redpanda volume (`redpanda`) | No - transient | + +The search index and event log are intentionally skipped: they are regenerated from +the database and file store, so excluding them keeps backups small and restores fast. +Use `--full` if you want them included anyway. + +## Backup + +`backup.sh` takes a **cold** snapshot: it stops the stack so the copy is +crash-consistent, archives the data volumes, copies your config, and restarts the +stack. Expect a short period of downtime for the duration of the snapshot. + +```bash +./backup.sh # back up to ./backups/huly-backup-/ +./backup.sh --output=/mnt/backups +./backup.sh --keep=7 # keep only the 7 most recent backups +./backup.sh --full # also include the search index and event log +``` + +Each backup is a self-contained directory: + +``` +backups/huly-backup-20260630-141500/ + cockroach.tar.gz + files.tar.gz + config/ + .env + huly_v7.conf + nginx.conf + ... + manifest.txt +``` + +Copy that directory off the server (to object storage or another host) for real +disaster recovery - a backup that lives only on the same disk as the deployment is +not a backup. + +## Offsite copies (disaster recovery) + +A backup that lives only on the same host as the deployment is not disaster recovery: if +the server is lost, so are the backups. Pass `--offsite` to also upload each backup to +S3-compatible storage (Backblaze B2, DigitalOcean Spaces, AWS S3, Wasabi, MinIO, etc.): + +```bash +cp backup-offsite.env.example backup-offsite.env # then fill in your bucket + keys +./backup.sh --offsite +``` + +`backup-offsite.env` is gitignored (it holds your keys). The upload runs `rclone` in a +throwaway container, so there is nothing extra to install. For retention on the remote, set +a bucket lifecycle policy (expire objects after N days) rather than pruning by hand. + +## Restore + +> [!WARNING] +> Restoring **replaces** the current data volumes and overwrites local config files. +> Test a restore on a clean or non-production environment before you need it for +> real. + +```bash +./restore.sh backups/huly-backup-20260630-141500 +./restore.sh backups/huly-backup-20260630-141500 --yes # skip the confirmation prompt +``` + +`restore.sh` creates the stack's volumes if they do not exist, writes the archived +data back into them, restores the config files, and starts the stack. The search +index rebuilds automatically over the first few minutes after start. + +## Verifying a backup + +A backup you have never restored is a guess, not a backup. Periodically: + +1. Spin up a throwaway host (or a separate project directory). +2. Run `restore.sh` against a recent backup there. +3. Confirm you can log in and see your workspaces. + +## Notes + +- Run these scripts from the repository root, next to `compose.yml`. +- They use only `docker`, `docker compose`, and a temporary `alpine` container, so + there is nothing extra to install. +- For zero-downtime logical backups, a future enhancement could mirror the hot + CockroachDB dump and `rclone` file sync already used by the Helm CronJobs. diff --git a/restore.sh b/restore.sh new file mode 100755 index 0000000..3a3849a --- /dev/null +++ b/restore.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# +# restore.sh - Restore a backup created by backup.sh into a Docker Compose deployment. +# +# WARNING: this REPLACES the current data volumes and overwrites local config files. +# Stop and think before running it against a live deployment. Restore into a clean or +# test environment first when you can. +# +# Usage: ./restore.sh BACKUP_DIR [--yes] [--help] + +set -euo pipefail + +cd "$(dirname "$0")" + +RESTORE_DIR="" +ASSUME_YES=false + +for arg in "$@"; do + case $arg in + --yes) ASSUME_YES=true ;; + --help) + echo "Usage: $0 BACKUP_DIR [OPTIONS]" + echo "Arguments:" + echo " BACKUP_DIR A backup directory created by backup.sh" + echo "Options:" + echo " --yes Do not prompt for confirmation" + echo " --help Show this help message" + exit 0 + ;; + -*) + echo "Unknown option: $arg" + echo "Use --help for usage information" + exit 1 + ;; + *) RESTORE_DIR="$arg" ;; + esac +done + +if [ -z "$RESTORE_DIR" ] || [ ! -d "$RESTORE_DIR" ]; then + echo "Error: provide a valid backup directory. See --help." >&2 + exit 1 +fi +if [ ! -f "$RESTORE_DIR/manifest.txt" ]; then + echo "Error: $RESTORE_DIR does not look like a backup (no manifest.txt)." >&2 + exit 1 +fi + +if docker compose version >/dev/null 2>&1; then + COMPOSE="docker compose" +elif command -v docker-compose >/dev/null 2>&1; then + COMPOSE="docker-compose" +else + echo "Error: docker compose is not available." >&2 + exit 1 +fi + +SRC_ABS=$(cd "$RESTORE_DIR" && pwd) + +echo -e "\033[1;33mThis will overwrite current data and config from:\033[0m $SRC_ABS" +cat "$RESTORE_DIR/manifest.txt" +if [ "$ASSUME_YES" != true ]; then + read -r -p "Continue and overwrite the current deployment? (y/N): " ANSWER + case "${ANSWER:-N}" in + [Yy]*) ;; + *) echo "Aborted."; exit 0 ;; + esac +fi + +# Ensure containers and volumes exist, then stop everything so the volumes are safe +# to overwrite (never write to a volume while its container is running). +echo "Preparing stack (creating containers, stopping the stack)..." +$COMPOSE up --no-start +$COMPOSE stop + +source_on() { + # $1 = container id, $2 = mount destination + docker inspect "$1" --format \ + "{{range .Mounts}}{{if eq .Destination \"$2\"}}{{if .Name}}{{.Name}}{{else}}{{.Source}}{{end}}{{end}}{{end}}" 2>/dev/null || true +} + +# Resolve the volume backing a mount destination: try the expected service name +# first, then scan every container in the project (so deployments that name the +# service differently, e.g. "cockroach" vs "cockroachdb", still resolve). +# Always exits 0 (prints nothing when not found) so it is safe under `set -e`. +mount_source() { + # $1 = expected service name, $2 = mount destination + local cid src + cid=$($COMPOSE ps -aq "$1" 2>/dev/null | head -1 || true) + if [ -n "$cid" ]; then + src=$(source_on "$cid" "$2") + [ -n "$src" ] && { echo "$src"; return 0; } + fi + for cid in $($COMPOSE ps -aq 2>/dev/null || true); do + src=$(source_on "$cid" "$2") + [ -n "$src" ] && { echo "$src"; return 0; } + done + return 0 +} + +restore_vol() { + # $1 = target source (volume name or host path), $2 = tar file in the backup + [ -f "$SRC_ABS/$2" ] || return 0 + if [ -z "$1" ]; then + echo -e " \033[33mcannot resolve target for $2, skipping\033[0m" + return 0 + fi + echo " - $2 -> $1" + docker run --rm -v "$1":/data -v "$SRC_ABS":/backup:ro alpine \ + sh -c 'rm -rf /data/* /data/.[!.]* /data/..?* 2>/dev/null; tar xzf "/backup/'"$2"'" -C /data' +} + +echo "Restoring data volumes..." +restore_vol "$(mount_source cockroach /cockroach/cockroach-data)" cockroach.tar.gz +restore_vol "$(mount_source minio /data)" files.tar.gz +restore_vol "$(mount_source mongodb /data/db)" mongodb.tar.gz +restore_vol "$(mount_source elastic /usr/share/elasticsearch/data)" elastic.tar.gz +restore_vol "$(mount_source redpanda /var/lib/redpanda/data)" redpanda.tar.gz + +echo "Restoring config and secret files..." +if [ -d "$RESTORE_DIR/config" ]; then + cp -rp "$RESTORE_DIR"/config/. . +fi + +echo "Starting stack..." +$COMPOSE up -d + +echo -e "\033[1;32mRestore complete.\033[0m The search index may take a few minutes to rebuild."