Add an updated Postgres 12 and Repmgr 5.1 configuration

README.md

@@ -115,6 +115,8 @@ You can configure any node of the cluster(`postgres.conf`) or pgpool(`pgpool.con
 
 ### Postgres
 
+To provide your own config file (apart from options given in `$CONFIGS`) you can put it in `/postgresql.conf.base`. The postdock postgres image will use that file as the base when configuring postgresql.conf.
+
 For the rest - you better **follow** the advise and look into the [src/Postgres-latest.Dockerfile](./src/Postgres-latest.Dockerfile) file - it full of comments :)
 
 ### Pgpool

make/postgres/make.sh

@@ -1,4 +1,3 @@
-echo ">>> Making postgres"
 
 
 for VALS in "POSTGRES_VERSION=9.5 REPMGR_VERSION=3.2 REPMGR_SYNTAX_V3=1 REPMGR_PACKAGE_VERSION=3.3.2-1.pgdg80+1" \
@@ -7,7 +6,8 @@ for VALS in "POSTGRES_VERSION=9.5 REPMGR_VERSION=3.2 REPMGR_SYNTAX_V3=1 REPMGR_P
             "POSTGRES_VERSION=9.5 REPMGR_VERSION=4.0 REPMGR_SYNTAX_V4=1 REPMGR_PACKAGE_VERSION=4.0.6-2.pgdg80+1" \
             "POSTGRES_VERSION=9.6 REPMGR_VERSION=4.0 REPMGR_SYNTAX_V4=1 REPMGR_PACKAGE_VERSION=4.0.6-2.pgdg80+1" \
             "POSTGRES_VERSION=10 REPMGR_VERSION=4.0 REPMGR_SYNTAX_V4=1 REPMGR_PACKAGE_VERSION=4.0.6-2.pgdg+1" \
-            "POSTGRES_VERSION=11 REPMGR_VERSION=4.0 REPMGR_SYNTAX_V4=1 REPMGR_PACKAGE_VERSION=4.0.6-2.pgdg+1"; do
+            "POSTGRES_VERSION=11 REPMGR_VERSION=4.0 REPMGR_SYNTAX_V4=1 REPMGR_PACKAGE_VERSION=4.0.6-2.pgdg+1" \
+            "POSTGRES_VERSION=12 REPMGR_VERSION=5.1 REPMGR_SYNTAX_V4=1 REPMGR_PACKAGE_VERSION=5.1.0-2.buster+1"; do
     eval $VALS
     FILE_FROM="./src/includes/dockerfile/Postgres-$POSTGRES_VERSION-Repmgr-$REPMGR_VERSION.part.Dockerfile"
     FILE_FROM_EXT="./src/includes/dockerfile/Postgres-extended-$POSTGRES_VERSION-Repmgr-$REPMGR_VERSION.part.Dockerfile"

src/Postgres-12-Repmgr-5.1.Dockerfile

@@ -0,0 +1,128 @@
+##########################################################################
+##                         AUTO-GENERATED FILE                          ##
+##########################################################################
+
+FROM postgres:12
+
+RUN apt-get update --fix-missing && \
+    apt-get install -y postgresql-server-dev-$PG_MAJOR wget openssh-server barman-cli
+
+COPY ./dockerfile/bin /usr/local/bin/dockerfile
+RUN chmod -R +x /usr/local/bin/dockerfile && ln -s /usr/local/bin/dockerfile/functions/* /usr/local/bin/
+
+# Install repmgr
+RUN apt-get -y install curl ca-certificates gnupg
+RUN curl https://dl.2ndquadrant.com/default/release/get/deb | bash
+RUN apt-get update
+
+RUN apt-get -y install 2ndquadrant-repository-keys repmgr=5.1.0-2.buster+1
+
+# Inherited variables
+# ENV POSTGRES_PASSWORD monkey_pass
+# ENV POSTGRES_USER monkey_user
+# ENV POSTGRES_DB monkey_db
+
+# Name of the cluster you want to start
+ENV CLUSTER_NAME pg_cluster
+
+# special repmgr db for cluster info
+ENV REPLICATION_DB replication_db
+ENV REPLICATION_USER replication_user
+ENV REPLICATION_PASSWORD replication_pass
+ENV REPLICATION_PRIMARY_PORT 5432
+
+
+# Host for replication (REQUIRED, NO DEFAULT)
+# ENV REPLICATION_PRIMARY_HOST
+
+# Integer number of node (NO DEFAULT) 
+# ENV NODE_ID 1
+                # if not defined, will be generated from the last number in NODE_NAME variable
+                # e.g. NODE_NAME=node-1 will give node identifier 1002
+
+# Node name (REQUIRED, NO DEFAULT)
+# ENV NODE_NAME node-1
+
+# (default: `hostname` of the node)
+# ENV CLUSTER_NODE_NETWORK_NAME null
+
+# priority on electing new master
+ENV NODE_PRIORITY 100
+
+ENV CONFIGS_DELIMITER_SYMBOL ,
+ENV CONFIGS_ASSIGNMENT_SYMBOL :
+                                #CONFIGS_DELIMITER_SYMBOL and CONFIGS_ASSIGNMENT_SYMBOL are used to parse CONFIGS variable
+                                # if CONFIGS_DELIMITER_SYMBOL=| and CONFIGS_ASSIGNMENT_SYMBOL=>, valid configuration string is var1>val1|var2>val2
+
+ENV REPMGR_MAJOR 4
+ENV REPMGR_NODES_TABLE nodes
+ENV REPMGR_NODE_ID_COLUMN node_id
+ENV REPMGR_NODE_NAME_COLUMN node_name
+ENV REPMGR_CLUSTER_SHOW_MASTER_PATTERN primary
+ENV REPMGR_SHOW_NODES_TABLE show_nodes
+ENV REPMGR_NODE_ID_PARAM_NAME node_id
+ENV REPMGR_LOG_LEVEL_PARAM_NAME log_level
+ENV REPMGR_MASTER_RESPONSE_TIMEOUT_PARAM_NAME async_query_timeout
+
+# ENV CONFIGS "listen_addresses:'*'"
+                                    # in format variable1:value1[,variable2:value2[,...]] if CONFIGS_DELIMITER_SYMBOL=, and CONFIGS_ASSIGNMENT_SYMBOL=:
+                                    # used for pgpool.conf file
+
+ENV PARTNER_NODES ""
+                    # List (comma separated) of all nodes in the cluster, it allows master to be adaptive on restart
+                    # (can act as a new standby if new master has been already elected)
+
+ENV MASTER_ROLE_LOCK_FILE_NAME $PGDATA/master.lock
+                                                   # File will be put in $MASTER_ROLE_LOCK_FILE_NAME when:
+                                                   #    - node starts as a primary node/master
+                                                   #    - node promoted to a primary node/master
+                                                   # File does not exist
+                                                   #    - if node starts as a standby
+ENV STANDBY_ROLE_LOCK_FILE_NAME $PGDATA/standby.lock
+                                                  # File will be put in $STANDBY_ROLE_LOCK_FILE_NAME when:
+                                                  #    - event repmgrd_failover_follow happened
+                                                  # contains upstream NODE_ID
+                                                  # that basically used when standby changes upstream node set by default
+ENV REPMGR_WAIT_POSTGRES_START_TIMEOUT 90
+                                            # For how long in seconds repmgr will wait for postgres start on current node
+                                            # Should be big enough to perform post replication start which might take from a minute to a few
+ENV USE_REPLICATION_SLOTS 1
+                                # Use replication slots to make sure that WAL files will not be removed without beein synced to replicas
+                                # Recomended(not required though) to put 0 for replicas of the second and deeper levels
+ENV CLEAN_OVER_REWIND 0
+                        # Clean $PGDATA directory before start standby and not try to rewind
+ENV SSH_ENABLE 0
+                        # If you need SSH server running on the node
+
+#### Advanced options ####
+ENV REPMGR_DEGRADED_MONITORING_TIMEOUT 5
+ENV REPMGR_PID_FILE /tmp/repmgrd.pid
+ENV STOPPING_LOCK_FILE /tmp/stop.pid
+ENV MASTER_SYNC_LOCK_FILE /tmp/replication
+ENV STOPPING_TIMEOUT 5
+ENV CONNECT_TIMEOUT 2
+ENV RECONNECT_ATTEMPTS 3
+ENV RECONNECT_INTERVAL 5
+ENV MASTER_RESPONSE_TIMEOUT 20
+ENV LOG_LEVEL INFO
+ENV CHECK_PGCONNECT_TIMEOUT 10
+ENV REPMGR_SLOT_NAME_PREFIX repmgr_slot_
+ENV LAUNCH_RECOVERY_CHECK_INTERVAL 30
+
+COPY ./pgsql/bin /usr/local/bin/cluster
+RUN chmod -R +x /usr/local/bin/cluster
+RUN ln -s /usr/local/bin/cluster/functions/* /usr/local/bin/
+COPY ./pgsql/configs /var/cluster_configs
+
+ENV NOTVISIBLE "in users profile"
+
+COPY ./ssh /tmp/.ssh
+RUN mv /tmp/.ssh/sshd_start /usr/local/bin/sshd_start && chmod +x /usr/local/bin/sshd_start
+
+EXPOSE 22
+EXPOSE 5432
+
+VOLUME /var/lib/postgresql/data
+USER root
+
+ENTRYPOINT ["/usr/local/bin/cluster/entrypoint.sh"]

src/Postgres-latest.Dockerfile

@@ -1 +1 @@
-Postgres-11-Repmgr-4.0.Dockerfile
+Postgres-12-Repmgr-5.1.Dockerfile

src/includes/dockerfile/Postgres-12-Repmgr-5.1.part.Dockerfile

@@ -0,0 +1,139 @@
+FROM postgres:{{ POSTGRES_VERSION }}
+
+RUN apt-get update --fix-missing && \
+    apt-get install -y postgresql-server-dev-$PG_MAJOR wget openssh-server barman-cli
+
+COPY ./dockerfile/bin /usr/local/bin/dockerfile
+RUN chmod -R +x /usr/local/bin/dockerfile && ln -s /usr/local/bin/dockerfile/functions/* /usr/local/bin/
+
+# Install repmgr
+RUN apt-get -y install curl ca-certificates gnupg
+RUN curl https://dl.2ndquadrant.com/default/release/get/deb | bash
+RUN apt-get update
+{{ #REPMGR_LATEST }}
+RUN apt-get -y install 2ndquadrant-repository-keys repmgr
+{{ /REPMGR_LATEST }}{{ ^REPMGR_LATEST }}
+RUN apt-get -y install 2ndquadrant-repository-keys repmgr={{ REPMGR_PACKAGE_VERSION }}
+{{ /REPMGR_LATEST }}
+
+# Inherited variables
+# ENV POSTGRES_PASSWORD monkey_pass
+# ENV POSTGRES_USER monkey_user
+# ENV POSTGRES_DB monkey_db
+
+# Name of the cluster you want to start
+ENV CLUSTER_NAME pg_cluster
+
+# special repmgr db for cluster info
+ENV REPLICATION_DB replication_db
+ENV REPLICATION_USER replication_user
+ENV REPLICATION_PASSWORD replication_pass
+ENV REPLICATION_PRIMARY_PORT 5432
+
+
+# Host for replication (REQUIRED, NO DEFAULT)
+# ENV REPLICATION_PRIMARY_HOST
+
+# Integer number of node (NO DEFAULT) 
+# ENV NODE_ID 1
+                # if not defined, will be generated from the last number in NODE_NAME variable
+                # e.g. NODE_NAME=node-1 will give node identifier 1002
+
+# Node name (REQUIRED, NO DEFAULT)
+# ENV NODE_NAME node-1
+
+# (default: `hostname` of the node)
+# ENV CLUSTER_NODE_NETWORK_NAME null
+
+# priority on electing new master
+ENV NODE_PRIORITY 100
+
+ENV CONFIGS_DELIMITER_SYMBOL ,
+ENV CONFIGS_ASSIGNMENT_SYMBOL :
+                                #CONFIGS_DELIMITER_SYMBOL and CONFIGS_ASSIGNMENT_SYMBOL are used to parse CONFIGS variable
+                                # if CONFIGS_DELIMITER_SYMBOL=| and CONFIGS_ASSIGNMENT_SYMBOL=>, valid configuration string is var1>val1|var2>val2
+
+{{ #REPMGR_SYNTAX_V4 }}
+ENV REPMGR_MAJOR 4
+ENV REPMGR_NODES_TABLE nodes
+ENV REPMGR_NODE_ID_COLUMN node_id
+ENV REPMGR_NODE_NAME_COLUMN node_name
+ENV REPMGR_CLUSTER_SHOW_MASTER_PATTERN primary
+ENV REPMGR_SHOW_NODES_TABLE show_nodes
+ENV REPMGR_NODE_ID_PARAM_NAME node_id
+ENV REPMGR_LOG_LEVEL_PARAM_NAME log_level
+ENV REPMGR_MASTER_RESPONSE_TIMEOUT_PARAM_NAME async_query_timeout
+{{ /REPMGR_SYNTAX_V4 }}{{ #REPMGR_SYNTAX_V3 }}
+ENV REPMGR_MAJOR 3
+ENV REPMGR_NODES_TABLE repl_nodes
+ENV REPMGR_NODE_ID_COLUMN id
+ENV REPMGR_NODE_NAME_COLUMN name
+ENV REPMGR_CLUSTER_SHOW_MASTER_PATTERN * master
+ENV REPMGR_SHOW_NODES_TABLE repl_show_nodes
+ENV REPMGR_NODE_ID_PARAM_NAME node
+ENV REPMGR_LOG_LEVEL_PARAM_NAME loglevel
+ENV REPMGR_MASTER_RESPONSE_TIMEOUT_PARAM_NAME master_reponse_timeout
+{{ /REPMGR_SYNTAX_V3 }}
+
+# ENV CONFIGS "listen_addresses:'*'"
+                                    # in format variable1:value1[,variable2:value2[,...]] if CONFIGS_DELIMITER_SYMBOL=, and CONFIGS_ASSIGNMENT_SYMBOL=:
+                                    # used for pgpool.conf file
+
+ENV PARTNER_NODES ""
+                    # List (comma separated) of all nodes in the cluster, it allows master to be adaptive on restart
+                    # (can act as a new standby if new master has been already elected)
+
+ENV MASTER_ROLE_LOCK_FILE_NAME $PGDATA/master.lock
+                                                   # File will be put in $MASTER_ROLE_LOCK_FILE_NAME when:
+                                                   #    - node starts as a primary node/master
+                                                   #    - node promoted to a primary node/master
+                                                   # File does not exist
+                                                   #    - if node starts as a standby
+ENV STANDBY_ROLE_LOCK_FILE_NAME $PGDATA/standby.lock
+                                                  # File will be put in $STANDBY_ROLE_LOCK_FILE_NAME when:
+                                                  #    - event repmgrd_failover_follow happened
+                                                  # contains upstream NODE_ID
+                                                  # that basically used when standby changes upstream node set by default
+ENV REPMGR_WAIT_POSTGRES_START_TIMEOUT 90
+                                            # For how long in seconds repmgr will wait for postgres start on current node
+                                            # Should be big enough to perform post replication start which might take from a minute to a few
+ENV USE_REPLICATION_SLOTS 1
+                                # Use replication slots to make sure that WAL files will not be removed without beein synced to replicas
+                                # Recomended(not required though) to put 0 for replicas of the second and deeper levels
+ENV CLEAN_OVER_REWIND 0
+                        # Clean $PGDATA directory before start standby and not try to rewind
+ENV SSH_ENABLE 0
+                        # If you need SSH server running on the node
+
+#### Advanced options ####
+ENV REPMGR_DEGRADED_MONITORING_TIMEOUT 5
+ENV REPMGR_PID_FILE /tmp/repmgrd.pid
+ENV STOPPING_LOCK_FILE /tmp/stop.pid
+ENV MASTER_SYNC_LOCK_FILE /tmp/replication
+ENV STOPPING_TIMEOUT 5
+ENV CONNECT_TIMEOUT 2
+ENV RECONNECT_ATTEMPTS 3
+ENV RECONNECT_INTERVAL 5
+ENV MASTER_RESPONSE_TIMEOUT 20
+ENV LOG_LEVEL INFO
+ENV CHECK_PGCONNECT_TIMEOUT 10
+ENV REPMGR_SLOT_NAME_PREFIX repmgr_slot_
+ENV LAUNCH_RECOVERY_CHECK_INTERVAL 30
+
+COPY ./pgsql/bin /usr/local/bin/cluster
+RUN chmod -R +x /usr/local/bin/cluster
+RUN ln -s /usr/local/bin/cluster/functions/* /usr/local/bin/
+COPY ./pgsql/configs /var/cluster_configs
+
+ENV NOTVISIBLE "in users profile"
+
+COPY ./ssh /tmp/.ssh
+RUN mv /tmp/.ssh/sshd_start /usr/local/bin/sshd_start && chmod +x /usr/local/bin/sshd_start
+
+EXPOSE 22
+EXPOSE 5432
+
+VOLUME /var/lib/postgresql/data
+USER root
+
+ENTRYPOINT ["/usr/local/bin/cluster/entrypoint.sh"]

src/pgsql/bin/functions/postdock_polymorphic

@@ -24,7 +24,7 @@ elif [ "$REPMGR_MAJOR" == "4" ]; then
     PRELOAD_REPMGR_LIB="repmgr"
     # new required params
     echo "degraded_monitoring_timeout=$REPMGR_DEGRADED_MONITORING_TIMEOUT" >> $REPMGR_CONFIG_FILE
-    echo "data_directory=$PGDATA" >> $REPMGR_CONFIG_FILE
+    echo "data_directory='$PGDATA'" >> $REPMGR_CONFIG_FILE
     echo "async_query_timeout=$MASTER_RESPONSE_TIMEOUT" >> $REPMGR_CONFIG_FILE
 fi
 

src/pgsql/bin/functions/postgres_configure

@@ -1,14 +1,17 @@
 #!/usr/bin/env bash
 
 CONFIG_FILE=$PGDATA/postgresql.conf
+CONFIG_FILE_BASE=/postgresql.conf.base
 
 echo ">>> Configuring $CONFIG_FILE"
 
-if [ "$FORCE_RECONFIGURE" == 1 ] || [ ! -f $CONFIG_FILE ]; then
+if [[ (( "$FORCE_RECONFIGURE" == 1 ) || ( ! -f $CONFIG_FILE )) && ( ! -f $CONFIG_FILE_BASE ) ]]; then
     echo ">>>>>> Config file was replaced with standard one!"
     cp -f /var/cluster_configs/postgresql.conf $CONFIG_FILE
+elif [[ (( "$FORCE_RECONFIGURE" == 1 ) || ( ! -f $CONFIG_FILE )) && ( -f $CONFIG_FILE_BASE ) ]]; then
+    echo ">>>>>> Adding configs to given $CONFIG_FILE_BASE"
 else
-    echo ">>>>>> Will add configs to the exists file"
+    echo ">>>>>> Will add configs to the existing file"
 fi
 
 IFS=$CONFIGS_DELIMITER_SYMBOL read -ra CONFIG_PAIRS <<< "$CONFIGS"

src/pgsql/bin/postgres/entrypoint.sh

@@ -9,14 +9,6 @@ echo "*:$REPLICATION_PRIMARY_PORT:*:$REPLICATION_USER:$REPLICATION_PASSWORD" >>
 chmod 0600 $PG_HOME/.pgpass
 chown postgres:postgres $PG_HOME/.pgpass
 
-if ! has_pg_cluster; then
-    echo ">>> Cleaning data folder which might have some garbage..."
-    rm -rf $PGDATA/*
-else
-    postgres_configure
-fi
-
-
 export CURRENT_REPLICATION_PRIMARY_HOST=""
 CURRENT_MASTER=`cluster_master || echo ''`
 echo ">>> Auto-detected master name: '$CURRENT_MASTER'"
@@ -47,11 +39,12 @@ chown -R postgres $PGDATA && chmod -R 0700 $PGDATA
 source /usr/local/bin/cluster/repmgr/configure.sh
 
 echo ">>> Sending in background postgres start..."
+
 if [[ "$CURRENT_REPLICATION_PRIMARY_HOST" == "" ]]; then
+    /usr/local/bin/cluster/repmgr/start.sh &
     cp -f /usr/local/bin/cluster/postgres/primary/entrypoint.sh /docker-entrypoint-initdb.d/
-    /docker-entrypoint.sh postgres &
+    /docker-entrypoint.sh postgres
 else
+    # repmgr is started after initial sync
     /usr/local/bin/cluster/postgres/standby/entrypoint.sh
 fi
-
-/usr/local/bin/cluster/repmgr/start.sh

src/pgsql/bin/postgres/primary/entrypoint.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -e
+
 FORCE_RECONFIGURE=1 postgres_configure
 
 # We need to create postgres user explicitly,

src/pgsql/bin/postgres/standby/entrypoint.sh

@@ -23,7 +23,8 @@ fi
 
 rm -f $MASTER_ROLE_LOCK_FILE_NAME # that file should not be here anyways
 
-postgres_configure
+FORCE_RECONFIGURE=1 postgres_configure
 
 echo ">>> Starting postgres..."
-exec gosu postgres postgres &
+/usr/local/bin/cluster/repmgr/start.sh &
+exec gosu postgres postgres

src/pgsql/bin/repmgr/configure.sh

@@ -11,16 +11,16 @@ fi
 echo ">>> Setting up repmgr config file '$REPMGR_CONFIG_FILE'..."
 echo "
 event_notification_command='/usr/local/bin/cluster/repmgr/events/router.sh %n %e %s \"%t\" \"%d\"'
-ssh_options=-o \"StrictHostKeyChecking no\" -v
+ssh_options='-o \"StrictHostKeyChecking no\" -v'
 use_replication_slots=$USE_REPLICATION_SLOTS
-pg_bindir=/usr/lib/postgresql/$PG_MAJOR/bin
+pg_bindir='/usr/lib/postgresql/$PG_MAJOR/bin'
 
 $REPMGR_NODE_ID_PARAM_NAME=$(get_node_id)
 node_name=$NODE_NAME
 conninfo='user=$REPLICATION_USER password=$REPLICATION_PASSWORD host=$CLUSTER_NODE_NETWORK_NAME dbname=$REPLICATION_DB port=$REPLICATION_PRIMARY_PORT connect_timeout=$CONNECT_TIMEOUT'
 failover=automatic
-promote_command='PGPASSWORD=$REPLICATION_PASSWORD repmgr standby promote --log-level DEBUG --verbose'
-follow_command='PGPASSWORD=$REPLICATION_PASSWORD repmgr standby follow -W --log-level DEBUG --verbose'
+promote_command='PGPASSWORD=\'$REPLICATION_PASSWORD\' repmgr standby promote --log-level DEBUG --verbose'
+follow_command='PGPASSWORD=\'$REPLICATION_PASSWORD\' repmgr standby follow -W --log-level DEBUG --verbose'
 reconnect_attempts=$RECONNECT_ATTEMPTS
 reconnect_interval=$RECONNECT_INTERVAL
 $REPMGR_LOG_LEVEL_PARAM_NAME=$LOG_LEVEL