test(refactor): Add Toxiproxy Hspec test infrastructure

nix/tools/tests.nix

@@ -42,7 +42,8 @@ let
       }
       ''
         ${withTools.withPg} -f test/observability/fixtures/load.sql \
-          ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:observability -- "''${_arg_leftovers[@]}"
+          ${withTools.withToxiproxyPg} \
+            ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:observability -- "''${_arg_leftovers[@]}"
       '';
 
   testDoctests =

nix/tools/withTools.nix

@@ -8,6 +8,7 @@
 , python3Packages
 , writeText
 , writers
+, toxiproxy
 }:
 let
   withTmpDb =
@@ -54,16 +55,25 @@ let
 
           export PGDATA="$tmpdir/db"
           export PGHOST="$tmpdir/socket"
+          PGPORT=$(${randomPort})
+          export PGPORT
           export PGUSER
           export PGDATABASE
           export PGRST_DB_SCHEMAS
           export PGTZ
           export PGOPTIONS
 
           HBA_FILE="$tmpdir/pg_hba.conf"
-          echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE"
-          echo "local $PGDATABASE all trust" >> "$HBA_FILE"
-          echo "local replication all trust" >> "$HBA_FILE"
+          {
+            echo "local $PGDATABASE some_protected_user password"
+            echo "local $PGDATABASE all trust"
+            echo "local replication all trust"
+            echo "host $PGDATABASE some_protected_user localhost password"
+            echo "host $PGDATABASE all localhost trust"
+          } >> "$HBA_FILE"
+
+          UNIX_PGHOST="$PGHOST"
+          export TCP_PGHOST="localhost"
 
           log "Initializing database cluster..."
           # We try to make the database cluster as independent as possible from the host
@@ -80,7 +90,7 @@ let
           # On MacOS, it's 104 chars
           # See: https://serverfault.com/questions/641347/check-if-a-path-exceeds-maximum-for-unix-domain-socket
 
-          pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \
+          pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"$TCP_PGHOST\" -c hba_file=$HBA_FILE -k $UNIX_PGHOST -c log_statement=\"all\" " \
             >> "$setuplog"
 
           log "Creating a minimally privileged $PGUSER connection role..."
@@ -93,6 +103,7 @@ let
             replica_slot="replica_$RANDOM"
             replica_dir="$tmpdir/$replica_slot"
             replica_host="$tmpdir/socket_$replica_slot"
+            replica_port=$(${randomPort})
 
             mkdir -p "$replica_host"
 
@@ -106,15 +117,16 @@ let
             log "Starting replica on $replica_host"
 
             # We set a low max_standby_streaming_delay to make the replication conflict fail faster in tests (otherwise it waits for the default 30s)
-            pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" -c max_standby_streaming_delay=\"3s\" " \
+            pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"$TCP_PGHOST\" -c port=$replica_port -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" -c max_standby_streaming_delay=\"3s\" " \
               >> "$setuplog"
 
             >&2 echo "${commandName}: Replica enabled. You can connect to it with: psql 'postgres:///$PGDATABASE?host=$replica_host' -U postgres"
             >&2 echo "${commandName}: You can tail the replica logs with: tail -f $replica_dblog"
 
             export PGREPLICAHOST="$replica_host"
+            export PGREPLICAPORT="$replica_port"
             export PGREPLICASLOT="$replica_slot"
-            export PGRST_DB_URI="postgres:///$PGDATABASE?host=$PGREPLICAHOST,$PGHOST"
+            export PGRST_DB_URI="postgres:///$PGDATABASE?host=$PGREPLICAHOST,$PGHOST&port=$replica_port,$PGPORT"
           fi
 
           # shellcheck disable=SC2329
@@ -372,6 +384,100 @@ let
         libraries = [ python3Packages.pandas python3Packages.tabulate python3Packages.psutil ];
       }
       (builtins.readFile ./monitor_pid.py);
+
+  randomPort =
+    writers.writePython3 "postgrest-random-port"
+      { }
+      ''
+        import socket
+        s = socket.socket()
+        s.bind(("127.0.0.1", 0))
+        print(s.getsockname()[1])
+        s.close()
+      '';
+
+  withToxiproxyProxy =
+    checkedShellScript
+      {
+        name = "postgrest-with-toxiproxy-proxy";
+        docs = "Run <command> with Toxiproxy proxy created.";
+        args =
+          [
+            "ARG_POSITIONAL_SINGLE([command], [Command to run])"
+            "ARG_LEFTOVERS([command arguments])"
+            "ARG_OPTIONAL_SINGLE([listen], [l], [Proxy will listen on this address])"
+            "ARG_OPTIONAL_SINGLE([upstream], [u], [Proxy will forward to this address])"
+          ];
+        positionalCompletion = "_command";
+        workingDir = "/";
+        withPath = [ toxiproxy ];
+      }
+      ''
+        proxyname="tp$RANDOM"
+        toxiproxy-cli create -l "$_arg_listen" -u "$_arg_upstream" "$proxyname"
+
+        # shellcheck disable=SC2329
+        stop () {
+          toxiproxy-cli delete "$proxyname" || true
+        }
+        trap stop EXIT
+
+        (TOXI_PROXY_NAME="$proxyname" "$_arg_command" "''${_arg_leftovers[@]}")
+      '';
+
+  withToxiproxyPg =
+    checkedShellScript
+      {
+        name = "postgrest-with-toxiproxy-pg";
+        docs = "Run <command> with a Toxiproxy proxy to PostgreSQL.";
+        args =
+          [
+            "ARG_POSITIONAL_SINGLE([command], [Command to run])"
+            "ARG_LEFTOVERS([command arguments])"
+            "ARG_USE_ENV([TCP_PGHOST], [], [PG host name])"
+            "ARG_USE_ENV([PGPORT], [], [PG port])"
+          ];
+        positionalCompletion = "_command";
+        workingDir = "/";
+      }
+      ''
+        proxy_port=''$(${randomPort})
+
+        ${withToxiproxy} ${withToxiproxyProxy} -l "$TCP_PGHOST:$proxy_port" -u "$TCP_PGHOST:$PGPORT" \
+          env "TOXI_PGPORT=$proxy_port" "$_arg_command" "''${_arg_leftovers[@]}"
+      '';
+
+  withToxiproxy =
+    checkedShellScript
+      {
+        name = "postgrest-with-toxiproxy";
+        docs = "Run <command> with toxiproxy-server";
+        args =
+          [
+            "ARG_POSITIONAL_SINGLE([command], [Command to run])"
+            "ARG_LEFTOVERS([command arguments])"
+          ];
+        positionalCompletion = "_command";
+        workingDir = "/";
+        withPath = [ toxiproxy ];
+      }
+      ''
+        if ! test -v TOXI_PROXY; then
+          export TOXI_PROXY=""
+          LOG_LEVEL=error toxiproxy-server&
+          TOXIPROXY_PID=$!
+          sleep 1 # give the server a moment to start
+
+          # shellcheck disable=SC2329
+          stop () {
+            kill "$TOXIPROXY_PID" || true
+            wait "$TOXIPROXY_PID" || true
+          }
+          trap stop EXIT  
+        fi
+        ("$_arg_command" "''${_arg_leftovers[@]}")
+      '';
+
 in
 buildToolbox
 {
@@ -380,11 +486,12 @@ buildToolbox
     inherit
       withGit
       withPgAll
-      withPgrst;
+      withPgrst
+      withToxiproxyPg;
   } // builtins.listToAttrs (
     # Create a `postgrest-with-pg-` for each PostgreSQL version
     builtins.map (pg: { inherit (pg) name; value = withTmpDb pg; }) postgresqlVersions
   );
   # make latest withPg available for other nix files
-  extra = { inherit withPg; };
+  extra = { inherit withPg withToxiproxyPg; };
 }

postgrest.cabal

@@ -204,6 +204,21 @@ executable postgrest
   else
     ghc-options: -O2
 
+library test-utils
+  visibility:         private
+  default-language:   Haskell2010
+  default-extensions: OverloadedStrings
+                      NoImplicitPrelude
+  hs-source-dirs:     test/lib
+  exposed-modules:    Toxiproxy
+  build-depends:      base              >= 4.9 && < 4.22
+                    , aeson             >= 2.0.3 && < 2.3
+                    , containers        >= 0.5.7 && < 0.8
+                    , http-client       >= 0.7.19 && < 0.8
+                    , servant-client    >= 0.20.3.0 && < 0.21
+                    , servant           >= 0.20.3.0 && < 0.21
+                    , text              >= 1.2.2 && < 2.2
+
 test-suite spec
   type:               exitcode-stdio-1.0
   default-language:   Haskell2010
@@ -312,6 +327,7 @@ test-suite observability
                       Observation.JwtCache
                       Observation.MetricsSpec
                       Observation.SchemaCacheSpec
+                      Observation.ToxiSpec
   build-depends:      base              >= 4.9 && < 4.22
                     , base64-bytestring >= 1 && < 1.3
                     , bytestring        >= 0.10.8 && < 0.13
@@ -323,11 +339,15 @@ test-suite observability
                     , hspec-wai-json    >= 0.10 && < 0.12
                     , http-types        >= 0.12.3 && < 0.13
                     , jose-jwt          >= 0.9.6 && < 0.11
+                    , monad-control     >= 1.0.1 && < 1.1
                     , postgrest
                     , prometheus-client >= 1.1.1 && < 1.2.0
                     , protolude         >= 0.3.1 && < 0.4
                     , text              >= 1.2.2 && < 2.2
+                    , test-utils
+                    , transformers-base >= 0.4.4 && < 0.5
                     , wai               >= 3.2.1 && < 3.3
+                    , wai-extra         >= 3.1.8 && < 3.2
   ghc-options:        -threaded -O0 -Werror -Wall -fwarn-identities
                       -fno-spec-constr -optP-Wno-nonportable-include-path
                       -fwrite-ide-info

src/PostgREST/AppState.hs

@@ -6,6 +6,7 @@
 module PostgREST.AppState
   ( AppState
   , destroy
+  , flushPool
   , getConfig
   , getSchemaCache
   , getMainThreadId

test/io/conftest.py

@@ -9,8 +9,9 @@ def dburi():
     "Postgres database connection URI."
     dbname = os.environ["PGDATABASE"]
     host = os.environ["PGHOST"]
+    port = os.environ["PGPORT"]
     user = os.environ["PGUSER"]
-    return f"postgresql://?dbname={dbname}&host={host}&user={user}".encode()
+    return f"postgresql://?dbname={dbname}&host={host}&port={port}&user={user}".encode()
 
 
 @pytest.fixture
@@ -19,6 +20,7 @@ def baseenv():
     return {
         "PGDATABASE": os.environ["PGDATABASE"],
         "PGHOST": os.environ["PGHOST"],
+        "PGPORT": os.environ["PGPORT"],
         "PGUSER": os.environ["PGUSER"],
     }
 
@@ -51,6 +53,7 @@ def replicaenv(defaultenv):
             **defaultenv,
             **conf,
             "PGHOST": os.environ["PGREPLICAHOST"] + "," + os.environ["PGHOST"],
+            "PGPORT": os.environ["PGREPLICAPORT"] + "," + os.environ["PGPORT"],
             "PGREPLICASLOT": os.environ["PGREPLICASLOT"],
         },
     }
@@ -76,6 +79,7 @@ def metapostgrest():
     env = {
         "PGDATABASE": os.environ["PGDATABASE"],
         "PGHOST": os.environ["PGHOST"],
+        "PGPORT": os.environ["PGPORT"],
         "PGUSER": role,
         "PGRST_DB_ANON_ROLE": role,
         "PGRST_DB_CONFIG": "true",

test/io/test_auth.py

@@ -162,7 +162,7 @@ def test_jwt_errors(defaultenv):
 
 def test_fail_with_invalid_password(defaultenv):
     "Connecting with an invalid password should fail without retries."
-    uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&user=some_protected_user&password=invalid_pass'
+    uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&port={defaultenv["PGPORT"]}&user=some_protected_user&password=invalid_pass'
     env = {**defaultenv, "PGRST_DB_URI": uri}
     with run(env=env, wait_for_readiness=False) as postgrest:
         exitCode = wait_until_exit(postgrest)

test/io/test_io.py

@@ -2022,7 +2022,7 @@ def test_log_listener_connection_start(defaultenv):
         # Check for the listener start message containing host and port
         # Do not check if pg version is displayed properly as it is tricky to test it
         assert any(
-            f'"{defaultenv["PGHOST"]}:5432" and listening for database notifications on the "pgrst" channel'
+            f'"{defaultenv["PGHOST"]}:{defaultenv["PGPORT"]}" and listening for database notifications on the "pgrst" channel'
             in line
             for line in output
         )