feat(backup): route all backup traffic through rclone-gateway

core/agent/htask.go

@@ -464,7 +464,7 @@ func obscureTaskInput(jsonStr string) string {
 }
 
 func isSensitive(target string) bool {
-	sensitiveList := []string{"password", "secret", "token"}
+	sensitiveList := []string{"password", "secret", "token", "key", "pass"}
 	ltarget := strings.ToLower(target)
 	for _, sensitive := range sensitiveList {
 		if strings.HasSuffix(ltarget, sensitive) {

core/api-server/configuration/configuration.go

@@ -93,6 +93,6 @@ func Init() {
 	if os.Getenv("SENSITIVE_LIST") != "" {
 		Config.SensitiveList = strings.Split(os.Getenv("SENSITIVE_LIST"), ",")
 	} else {
-		Config.SensitiveList = []string{"password", "secret", "token", "jwt", "admpass", "adminpass"}
+		Config.SensitiveList = []string{"password", "secret", "token", "jwt", "admpass", "adminpass", "key", "pass"}
 	}
 }

core/build-image.sh

@@ -76,14 +76,15 @@ printf "CORE_IMAGE=${repobase}/core:%s\n" "${IMAGETAG:-latest}" >> "${core_env_f
 printf "REDIS_IMAGE=${repobase}/redis:%s\n" "${IMAGETAG:-latest}" >> "${core_env_file}"
 printf "RSYNC_IMAGE=${repobase}/rsync:%s\n" "${IMAGETAG:-latest}" >> "${core_env_file}"
 printf "RESTIC_IMAGE=${repobase}/restic:%s\n" "${IMAGETAG:-latest}" >> "${core_env_file}"
+printf "RCLONE_IMAGE=${repobase}/rclone:%s\n" "${IMAGETAG:-latest}" >> "${core_env_file}"
 printf "SUPPORT_IMAGE=${repobase}/support:%s\n" "${IMAGETAG:-latest}" >> "${core_env_file}"
 printf "PROMTAIL_IMAGE=docker.io/grafana/alloy:v1.11.3\n" >> "${core_env_file}"
 printf "NODE_EXPORTER_IMAGE=quay.io/prometheus/node-exporter:v1.10.2\n" >> "${core_env_file}"
 chmod -c 644 "${core_env_file}"
 source "${core_env_file}"
 buildah add "${container}" ${core_env_file} /etc/nethserver/core.env
 buildah config \
-    --label="org.nethserver.images=${REDIS_IMAGE} ${RSYNC_IMAGE} ${RESTIC_IMAGE} ${PROMTAIL_IMAGE} ${SUPPORT_IMAGE} ${NODE_EXPORTER_IMAGE}" \
+    --label="org.nethserver.images=${REDIS_IMAGE} ${RSYNC_IMAGE} ${RESTIC_IMAGE} ${RCLONE_IMAGE} ${PROMTAIL_IMAGE} ${SUPPORT_IMAGE} ${NODE_EXPORTER_IMAGE}" \
     --label="org.nethserver.flags=core_module" \
     --entrypoint=/ "${container}"
 buildah commit "${container}" "${repobase}/${reponame}"
@@ -124,29 +125,44 @@ buildah commit "${container}" "${repobase}/${reponame}"
 buildah rm "${container}"
 images+=("${repobase}/${reponame}")
 
-echo "Building the restic/rclone image..."
-container=$(buildah from docker.io/library/alpine:3.22.4)
+echo "Building the restic image..."
+container=$(buildah from docker.io/library/alpine:3.23.4)
 reponame="restic"
-buildah add "${container}" restic/ /
 buildah run ${container} sh <<'EOF'
-apk add --no-cache restic rclone
-addgroup -S restic
-adduser -S -D -H -h /dev/null -s /sbin/nologin -G restic restic
-mkdir -v -p -m 0750 /srv/repo
-chown -c restic:restic /srv/repo
+apk add --no-cache restic
 EOF
 buildah config \
     --cmd='[]' \
     --entrypoint='["/usr/bin/restic"]' \
-    --env='RCLONE_CONFIG=/dev/null' \
-    --volume=/srv/repo \
+    ${container}
+buildah commit "${container}" "${repobase}/${reponame}"
+buildah rm "${container}"
+images+=("${repobase}/${reponame}")
+
+echo "Building the rclone image..."
+container=$(buildah from docker.io/library/alpine:3.23.4)
+reponame="rclone"
+buildah add "${container}" rclone/ /
+buildah run ${container} sh <<'EOF'
+addgroup -S rclone -g 101
+adduser -u 100 -S -D -h /var/lib/rclone -s /sbin/nologin -G rclone rclone
+apk add --no-cache rclone haproxy python3
+EOF
+buildah config \
+    --user=rclone:rclone \
+    --cmd='[]' \
+    --entrypoint='["/usr/bin/rclone"]' \
+    --env=RCLONE_CACHE_DIR=/var/cache/rclone \
+    --env=RCLONE_CONFIG=/etc/rclone/rclone.conf \
+    --env=RCLONE_UNIX_SOCKET=/var/lib/rclone/rclone.sock \
+    --env=RCLONE_LOG_SYSTEMD=1 \
     ${container}
 buildah commit "${container}" "${repobase}/${reponame}"
 buildah rm "${container}"
 images+=("${repobase}/${reponame}")
 
 echo "Building the rsync image..."
-container=$(buildah from docker.io/library/alpine:3.22.4)
+container=$(buildah from docker.io/library/alpine:3.23.4)
 reponame="rsync"
 buildah run ${container} -- apk add --no-cache rsync
 buildah add "${container}" rsync/entrypoint.sh /entrypoint.sh
@@ -159,7 +175,7 @@ buildah rm "${container}"
 images+=("${repobase}/${reponame}")
 
 echo "Building the support image..."
-container=$(buildah from docker.io/library/alpine:3.22.4)
+container=$(buildah from docker.io/library/alpine:3.23.4)
 reponame="support"
 buildah run ${container} -- sh <<'EOF'
 apk add --no-cache openvpn gettext-envsubst

core/imageroot/etc/systemd/system/backup-timers.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Backup timers (from Redis state)
+After=redis.service
+Requires=redis.service
+ConditionPathExists=/var/lib/nethserver/node/state/rclone
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=runagent -m node schedule-backup start-timers
+ExecStop=runagent -m node schedule-backup stop-timers

core/imageroot/etc/systemd/system/rclone-gateway.service

@@ -0,0 +1,50 @@
+[Unit]
+Description=Rclone Gateway server
+After=redis.service
+Wants=redis.service backup-timers.service
+StartLimitIntervalSec=10s
+StartLimitBurst=3
+ConditionPathExists=/etc/wireguard/wg0.conf
+
+[Service]
+Type=forking
+WorkingDirectory=/var/lib/nethserver/node/state
+PIDFile=%t/%N.pid
+Environment=PODMAN_SYSTEMD_UNIT=%n
+Environment=BACKUP_VOLUME=rclone-webdav
+EnvironmentFile=/etc/nethserver/core.env
+EnvironmentFile=-/var/lib/nethserver/node/state/rclone-webdav.env
+Restart=always
+TimeoutStopSec=120
+TimeoutStartSec=120
+SuccessExitStatus=143
+ExecStartPre=/bin/rm -f %t/%N.pid %t/%N.cid
+ExecStartPre=mkdir -vp rclone haproxy
+ExecStartPre=-runagent -m node rclonegwctl write-configuration --rclonedir=rclone --haproxydir=haproxy
+ExecStart=/usr/bin/podman run \
+    --conmon-pidfile=%t/%N.pid \
+    --cidfile=%t/%N.cid \
+    --cgroups=no-conmon \
+    --detach \
+    --init \
+    --log-opt=tag=%N \
+    --replace --name=%N \
+    --network=host \
+    --volume=./rclone:/etc/rclone:ro,Z \
+    --volume=./haproxy:/etc/haproxy:ro,Z \
+    --volume=${BACKUP_VOLUME}:/srv/repo:z \
+    --mount=type=tmpfs,tmpfs-size=10M,destination=/var/lib/rclone,chown=true \
+    --volume=/dev/log:/dev/log \
+    --volume=rclone-cache:/var/cache/rclone:Z \
+    --entrypoint=rclone-gateway-entrypoint.sh \
+    --env-file=rclone-webdav.env \
+    ${RCLONE_IMAGE}
+ExecStartPost=bash -c '{ while ! exec 3<>/dev/tcp/127.0.0.1/4694; do sleep 5 ; done } &>/dev/null'
+ExecReload=runagent -m node rclonegwctl write-configuration --rclonedir=rclone --haproxydir=haproxy
+ExecReload=runagent -m node podman exec %N reload-config
+ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N.cid -t 115
+ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N.cid
+
+[Install]
+WantedBy=default.target
+Alias=rclone-webdav.service