initial version with client side encryption of the PostgreSQL database and upload to S3

Author: willemvd

Dockerfile

@@ -0,0 +1,18 @@
+FROM alpine:3.4
+MAINTAINER Willem willemvd@github
+
+COPY backup.sh /tmp/backup.sh
+
+RUN apk --no-cache add \
+      curl \
+      less \
+      groff \
+      jq \
+      python \
+      py-pip \
+      postgresql \
+      openssl && \
+      pip install --upgrade pip awscli && \
+      chmod -R 777 /tmp
+
+CMD /tmp/backup.sh

README.md

@@ -0,0 +1,32 @@
+# PostgreSQL, client-side encrypted, Amazon S3 backupper
+
+Docker image that is used to backup a PostgreSQL database, encrypt the dump file locally and upload to a S3 bucket.
+Your own settings define how secure you want it to be :)
+
+Import difference with other PostgreSQL to S3 backup tools is **client-side encryption**.
+This means that it is not necessary to send any unencrypted data towards Amazon 
+
+The following (required) environment variables need to be set in order to run everything smoothly:
+
+```properties
+# Used in the backup file name 
+# and to place the file inside a subfolder of the S3 bucket
+BACKUP_TYPE=<type of backup, should also be a folder/key in the S3 bucket>
+
+# PostgreSQL settings
+PGHOST=<database hostname>
+PGPORT=<port of the database server, default 5432>
+PGDATABASE=<database name>
+PGUSER=<database user>
+PGPASSWORD=<database password>
+
+# OpenSSL settings
+ENCRYPTION_PASS_PHRASE=<strong password>
+OPENSSL_CIPHER_TYPE=<strong cipher like aes-256-cbc>
+
+# Amazon settings
+AWS_ACCESS_KEY_ID=<access key id got from Amazon>
+AWS_SECRET_ACCESS_KEY=<secret access key got from Amazon>
+AWS_DEFAULT_REGION=<Amazon region name e.g. eu-central-1>
+S3_BUCKET_NAME=<name of the bucket in S3>
+```

backup.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+
+#exit if one of the commands fail
+set -e
+
+DATE=$(date +%Y-%m-%d-%H-%M-%S)
+
+# backup type is used to seperate types like hourly or daily (folders in s3 bucket should be named like that)
+#BACKUP_TYPE=
+
+BACKUP_FILE_NAME=$PGDATABASE-backup-$BACKUP_TYPE-$DATE.sql
+
+cd /tmp
+
+echo "Start dumping of database '$PGDATABASE' from '$PGHOST:$PGPORT/$PGDATABASE' with user '$PGUSER' to file '$BACKUP_FILE_NAME'"
+START_DUMP=$(date +%s)
+
+# all PostgreSQL settings are set with environment variables
+#PGDATABASE=
+#PGUSER=
+#PGPASSWORD=
+#PGHOST=
+#PGPORT=
+pg_dump --format=custom > $BACKUP_FILE_NAME
+
+DONE_DUMP_IN=$(($(date +%s)-$START_DUMP))
+echo "Done dumping ($DONE_DUMP_IN sec) database"
+
+echo ""
+
+# OpenSSL settings
+#OPENSSL_CIPHER_TYPE=
+#ENCRYPTION_PASS_PHRASE=
+
+ENCRYPTION_FILE_NAME_SUFFIX=.enc
+
+echo "Start encrypting database dump file '$BACKUP_FILE_NAME' with cipher type '$OPENSSL_CIPHER_TYPE' and password as defined in environment variable ENCRYPTION_PASS_PHRASE"
+START_ENC=$(date +%s)
+openssl enc -in $BACKUP_FILE_NAME -out $BACKUP_FILE_NAME$ENCRYPTION_FILE_NAME_SUFFIX -e -$OPENSSL_CIPHER_TYPE -pass pass:$ENCRYPTION_PASS_PHRASE
+
+DONE_ENC_IN=$(($(date +%s)-$START_ENC))
+echo "Done encrypting ($DONE_ENC_IN sec) database dump file, now remove unencrypted file"
+
+rm -f $BACKUP_FILE_NAME
+
+echo ""
+
+echo "Start uploading encrypted '$BACKUP_FILE_NAME' to 's3://$S3_BUCKET_NAME/$BACKUP_TYPE/$BACKUP_FILE_NAME'"
+START_UPLOAD=$(date +%s)
+
+# AWS connection settings
+#AWS_ACCESS_KEY_ID=
+#AWS_SECRET_ACCESS_KEY=
+#AWS_DEFAULT_REGION=
+
+# bucket name to upload to
+#S3_BUCKET_NAME=
+aws s3 cp ./$BACKUP_FILE_NAME$ENCRYPTION_FILE_NAME_SUFFIX s3://$S3_BUCKET_NAME/$BACKUP_TYPE/$BACKUP_FILE_NAME
+
+DONE_UPLOAD_IN=$(($(date +%s)-$START_UPLOAD))
+echo "Done uploading ($DONE_UPLOAD_IN sec) 's3://$S3_BUCKET_NAME/$BACKUP_TYPE/$BACKUP_FILE_NAME'"

openshift-postgresql-s3-backup-scheduledJob.yaml

@@ -0,0 +1,87 @@
+apiVersion: v1
+kind: List
+items:
+- apiVersion: batch/v2alpha1
+  kind: ScheduledJob
+  metadata:
+    name: postgresql-s3-hourly-backup
+  spec:
+    # every start of the hour
+    schedule: "0 * * * *"
+    jobTemplate:
+      spec:
+        template:
+          spec:
+            # 30 minutes timeout
+            activeDeadlineSeconds: 1800
+            containers:
+            - name: postgresql-s3-hourly-backup
+              image: willemvd/postgresql-client-side-encrypted-s3-backup
+              env:
+              - name: PGDATABASE
+                value:
+              - name: PGHOST
+                value:
+              - name: PGPORT
+                value: "5432"
+              - name: PGUSER
+                value:
+              - name: PGPASSWORD
+                value:
+              - name: ENCRYPTION_PASS_PHRASE
+                value:
+              - name: OPENSSL_CIPHER_TYPE
+                value: aes-256-cbc
+              - name: AWS_ACCESS_KEY_ID
+                value:
+              - name: AWS_SECRET_ACCESS_KEY
+                value:
+              - name: AWS_DEFAULT_REGION
+                value:
+              - name: S3_BUCKET_NAME
+                value:
+              - name: BACKUP_TYPE
+                value: hourly
+            restartPolicy: OnFailure
+- apiVersion: batch/v2alpha1
+  kind: ScheduledJob
+  metadata:
+    name: postgresql-s3-daily-backup
+  spec:
+    # daily at 02:30 (do not run at the whole hour to prevent potential issues with hourly run)
+    schedule: "30 2 * * *"
+    jobTemplate:
+      spec:
+        template:
+          spec:
+            # 30 minutes timeout
+            activeDeadlineSeconds: 1800
+            containers:
+            - name: postgresql-s3-daily-backup
+              image: willemvd/postgresql-client-side-encrypted-s3-backup
+              env:
+              - name: PGDATABASE
+                value:
+              - name: PGHOST
+                value:
+              - name: PGPORT
+                value: "5432"
+              - name: PGUSER
+                value:
+              - name: PGPASSWORD
+                value:
+              - name: ENCRYPTION_PASS_PHRASE
+                value:
+              - name: OPENSSL_CIPHER_TYPE
+                value: aes-256-cbc
+              - name: AWS_ACCESS_KEY_ID
+                value:
+              - name: AWS_SECRET_ACCESS_KEY
+                value:
+              - name: AWS_DEFAULT_REGION
+                value:
+              - name: S3_BUCKET_NAME
+                value:
+              - name: BACKUP_TYPE
+                value: daily
+            restartPolicy: OnFailure