From 75d51f9d6c2166e2ddafcb1ed89d9f2d73354fdf Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Mon, 28 Jul 2025 07:18:52 +0100 Subject: [PATCH 1/2] Basic PITR testcase This commit adds a simple PITR testcase with the pg_simple test runner that verifies that 1. PITR works 2. data checksums before and after recovery are the same. --- libstormweaver/src/scripting/luactx.cpp | 3 + scenarios/ci/pitr.lua | 88 +++++++++++++++++++++++++ scripts/pg_simple.lua | 8 ++- 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 scenarios/ci/pitr.lua diff --git a/libstormweaver/src/scripting/luactx.cpp b/libstormweaver/src/scripting/luactx.cpp index b009931..5e88fcf 100644 --- a/libstormweaver/src/scripting/luactx.cpp +++ b/libstormweaver/src/scripting/luactx.cpp @@ -287,6 +287,9 @@ LuaContext::LuaContext(std::shared_ptr logger) return std::filesystem::copy(from, to, std::filesystem::copy_options::recursive); }; + fs_usertype["absolute"] = [](std::string const &path) { + return std::filesystem::absolute(path).string(); + }; fs_usertype["delete_directory"] = [](std::string const &dir) { return std::filesystem::remove_all(dir); }; diff --git a/scenarios/ci/pitr.lua b/scenarios/ci/pitr.lua new file mode 100644 index 0000000..47e5a33 --- /dev/null +++ b/scenarios/ci/pitr.lua @@ -0,0 +1,88 @@ +require("pg_simple") + +function main(argv) + if fs.is_directory("backups") then + warning("Deleting old stormweaver run backups") + fs.delete_directory("backups") + end + + if fs.is_directory("archive") then + warning("Deleting old stormweaver run archive") + fs.delete_directory("archive") + end + + os.execute("mkdir archive") + + simple_pg_single(argv, function(args) + pgm:basebackup(1, "-D", "backups/backup_0", "-U", "stormweaver", "-c", "fast") + + times = {} + + for workloadIdx = 1, tonumber(args["repeat"]) + 1 do + t1:run() + t1:wait_completion() + pgm:get(1, 10) + + sleep(3000) + + t = os.time() + + w = pgm.primaryNode:make_worker("verification") + w:calculate_database_checksums("backups/" .. t .. ".checksum") + + table.insert(times, t) + + sleep(3000) + end + + fs.delete_directory("archive-copy") + os.execute("cp -r archive archive-copy") + + for workloadIdx = 1, tonumber(args["repeat"]) do + pgm:stop(1, 10) + + fs.delete_directory("archive") + os.execute("cp -r archive-copy archive") + + info("Restoring and verifying PITR #" .. tostring(workloadIdx) .. " - " .. tostring(times[workloadIdx])) + fs.delete_directory("datadirs/datadir_pr") + + os.execute("cp -r backups/backup_0 datadirs/datadir_pr") + os.execute("touch datadirs/datadir_pr/recovery.signal") + + pgm:get(1):add_config({ + restore_command = "'cp " .. fs.absolute("archive/%f") .. ' "%p"\'', + recovery_target_time = "'" .. os.date("%Y-%m-%d %X", tostring(times[workloadIdx])) .. "'", + }) + + pgm:start(1) + + recovered = false + for s = 1, 200 do + if os.execute("grep 'pausing at the end of recovery' datadirs/datadir_pr/logs/server.log") then + recovered = true + break + end + sleep(1000) + end + + if not recovered then + error("Couldn't complete PITR") + end + + backupChecksumFile = "backups/" .. tostring(times[workloadIdx]) .. ".checksum" + checksumFile = "datadirs/datadir_pr/db.checksum" + w = pgm.primaryNode:make_worker("reset") + + w:sql_connection():execute_query("SELECT pg_wal_replay_resume()") + w:reset_metadata() + w:discover_existing_schema() + w:calculate_database_checksums(checksumFile) + + ret = os.execute("/usr/bin/diff " .. checksumFile .. " " .. backupChecksumFile) + if not ret then + error("Backup checksum verification failed with incremental #" .. tostring(workloadIdx)) + end + end + end) +end diff --git a/scripts/pg_simple.lua b/scripts/pg_simple.lua index 2248f58..827058a 100644 --- a/scripts/pg_simple.lua +++ b/scripts/pg_simple.lua @@ -56,7 +56,13 @@ function simple_pg_single(argv, test) pgconfig = PgConf.new(conffile["default"]) pgm = PgManager.new(pgconfig) - additional_settings = { shared_preload_libraries = "", summarize_wal = "on" } + additional_settings = { + shared_preload_libraries = "", + summarize_wal = "on", + archive_mode = "on", + archive_command = "'cp %p " .. fs.absolute("archive/%f'"), + max_wal_senders = "3", + } if args["tde"] ~= "off" then info("Running tests with pg_tde = " .. args["tde"]) From 9d50a73e9b7176301615440b0d7f8e916dbecaa8 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Fri, 8 Aug 2025 07:51:01 +0100 Subject: [PATCH 2/2] fixup! Basic PITR testcase --- scenarios/ci/pitr.lua | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/scenarios/ci/pitr.lua b/scenarios/ci/pitr.lua index 47e5a33..02884be 100644 --- a/scenarios/ci/pitr.lua +++ b/scenarios/ci/pitr.lua @@ -1,5 +1,15 @@ require("pg_simple") +function tail_grep(pattern, file, timeout) + for s = 1, timeout do + if os.execute("grep '" .. pattern .. "' '" .. file .. "'") then + return true + end + sleep(1000) + end + return false +end + function main(argv) if fs.is_directory("backups") then warning("Deleting old stormweaver run backups") @@ -57,16 +67,7 @@ function main(argv) pgm:start(1) - recovered = false - for s = 1, 200 do - if os.execute("grep 'pausing at the end of recovery' datadirs/datadir_pr/logs/server.log") then - recovered = true - break - end - sleep(1000) - end - - if not recovered then + if not tail_grep("pausing at the end of recovery", "datadirs/datadir_pr/logs/server.log", 200) then error("Couldn't complete PITR") end @@ -79,7 +80,7 @@ function main(argv) w:discover_existing_schema() w:calculate_database_checksums(checksumFile) - ret = os.execute("/usr/bin/diff " .. checksumFile .. " " .. backupChecksumFile) + ret = os.execute("diff " .. checksumFile .. " " .. backupChecksumFile) if not ret then error("Backup checksum verification failed with incremental #" .. tostring(workloadIdx)) end