From 0bbc781b6b79d84dad55aa63eedea51c6fa2e8e9 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Tue, 27 Oct 2020 14:21:21 +0500 Subject: [PATCH 01/54] Disallow cancelation of syncronous commit V1 Currently we allow to cancel awaiting of syncronous commit. Some drivers cancel query after timeout. If application will retry idempotent query, it will get confirmation of written data. This can lead to split-brain in HA scenarios. To prevent it this we add synchronous_commit_cancelation setting disalowing cancelation of syncronous replication wait Version for PostgreSQL 16 --- src/backend/access/transam/xact.c | 1 + src/backend/replication/syncrep.c | 19 +++++++++++++------ src/backend/utils/misc/guc_tables.c | 9 +++++++++ src/backend/utils/misc/postgresql.conf.sample | 7 +++++++ src/include/access/xact.h | 2 ++ 5 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 91dbfcc0d78..95fa6ce039a 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -85,6 +85,7 @@ bool DefaultXactDeferrable = false; bool XactDeferrable; int synchronous_commit = SYNCHRONOUS_COMMIT_ON; +bool synchronous_commit_cancelation = false; /* * CheckXidAlive is a xid value pointing to a possibly ongoing (sub) diff --git a/src/backend/replication/syncrep.c b/src/backend/replication/syncrep.c index 2455841d6b4..07d37bdec13 100644 --- a/src/backend/replication/syncrep.c +++ b/src/backend/replication/syncrep.c @@ -302,8 +302,8 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) { ereport(WARNING, (errcode(ERRCODE_ADMIN_SHUTDOWN), - errmsg("canceling the wait for synchronous replication and terminating connection due to administrator command"), - errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); + errmsg("canceling the wait for synchronous replication and terminating connection due to administrator command"), + errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); whereToSendOutput = DestNone; SyncRepCancelWait(); break; @@ -318,11 +318,18 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) if (QueryCancelPending) { QueryCancelPending = false; + if (synchronous_commit_cancelation) + { + ereport(WARNING, + (errmsg("canceling wait for synchronous replication due to user request"), + errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); + SyncRepCancelWait(); + break; + } + ereport(WARNING, - (errmsg("canceling wait for synchronous replication due to user request"), - errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); - SyncRepCancelWait(); - break; + (errmsg("canceling wait for synchronous replication due requested, but cancelation is not allowed"), + errdetail("The COMMIT record has already flushed to WAL locally and might not have been replicated to the standby. We must wait here."))); } /* diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 093567535bd..bcb224b6a8d 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -1198,6 +1198,15 @@ struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"synchronous_commit_cancelation", PGC_USERSET, WAL_SETTINGS, + gettext_noop("Allow to cancel waiting for replication of transaction commited localy."), + NULL + }, + &synchronous_commit_cancelation, + false, NULL, NULL, NULL + }, + { {"log_checkpoints", PGC_SIGHUP, LOGGING_WHAT, gettext_noop("Logs each checkpoint."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index c8ffcdf7b16..abfa4f769e5 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -755,6 +755,8 @@ # - Other Defaults - #dynamic_library_path = '$libdir' +#extension_destdir = '' # prepend path when loading extensions + # and shared objects (added by Debian) #gin_fuzzy_search_limit = 0 @@ -817,6 +819,11 @@ #include_if_exists = '...' # include file only if it exists #include = '...' # include file +#------------------------------------------------------------------------------ +# MDB +#------------------------------------------------------------------------------ + +#synchronous_commit_cancelation = off #------------------------------------------------------------------------------ # CUSTOMIZED OPTIONS diff --git a/src/include/access/xact.h b/src/include/access/xact.h index 7d3b9446e62..cab43d021f2 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -81,6 +81,8 @@ typedef enum /* Synchronous commit level */ extern PGDLLIMPORT int synchronous_commit; +/* Allow cancelation of queries waiting for sync replication but commited locally */ +extern bool synchronous_commit_cancelation; /* used during logical streaming of a transaction */ extern PGDLLIMPORT TransactionId CheckXidAlive; From 2a564cac0fc441e8d7e7c877fd1faef570c998a0 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Sun, 6 Dec 2020 10:02:44 +0500 Subject: [PATCH 02/54] Extend multixact SLRU --- src/include/access/multixact.h | 4 ++-- src/include/access/subtrans.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/include/access/multixact.h b/src/include/access/multixact.h index 246f757f6ab..31b16fb95e0 100644 --- a/src/include/access/multixact.h +++ b/src/include/access/multixact.h @@ -30,8 +30,8 @@ #define MaxMultiXactOffset ((MultiXactOffset) 0xFFFFFFFF) /* Number of SLRU buffers to use for multixact */ -#define NUM_MULTIXACTOFFSET_BUFFERS 8 -#define NUM_MULTIXACTMEMBER_BUFFERS 16 +#define NUM_MULTIXACTOFFSET_BUFFERS 32 +#define NUM_MULTIXACTMEMBER_BUFFERS 64 /* * Possible multixact lock modes ("status"). The first four modes are for diff --git a/src/include/access/subtrans.h b/src/include/access/subtrans.h index 46a473c77f5..48a6166a118 100644 --- a/src/include/access/subtrans.h +++ b/src/include/access/subtrans.h @@ -12,7 +12,7 @@ #define SUBTRANS_H /* Number of SLRU buffers to use for subtrans */ -#define NUM_SUBTRANS_BUFFERS 32 +#define NUM_SUBTRANS_BUFFERS 64 extern void SubTransSetParent(TransactionId xid, TransactionId parent); extern TransactionId SubTransGetParent(TransactionId xid); From 834af5b03315246c1f703e720defc9088946abeb Mon Sep 17 00:00:00 2001 From: Jakub Wartak Date: Thu, 23 Jun 2022 08:18:26 +0000 Subject: [PATCH 03/54] Use fadvise to prefetch WAL in xlogrecovery --- src/backend/access/transam/xlogrecovery.c | 12 ++++++++++++ src/backend/utils/activity/wait_event.c | 3 +++ src/include/utils/wait_event.h | 1 + 3 files changed, 16 insertions(+) diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 512abcc6ab7..39d07245241 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -3388,6 +3388,18 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, Assert(targetPageOff == readOff); Assert(reqLen <= readLen); +#if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) + /* + * Prefetch next wal blocks to avoid page misses on next read iterations. + */ +#define RACHUNK (128*1024) + if (readOff % RACHUNK == 0) { + pgstat_report_wait_start(WAIT_EVENT_WAL_PREFETCH); + posix_fadvise(readFile, readOff + RACHUNK, RACHUNK, POSIX_FADV_WILLNEED); + pgstat_report_wait_end(); + } +#endif + xlogreader->seg.ws_tli = curFileTLI; /* diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c index 05b71f9bc2a..f5f61873971 100644 --- a/src/backend/utils/activity/wait_event.c +++ b/src/backend/utils/activity/wait_event.c @@ -750,6 +750,9 @@ pgstat_get_wait_io(WaitEventIO w) case WAIT_EVENT_WAL_READ: event_name = "WALRead"; break; + case WAIT_EVENT_WAL_PREFETCH: + event_name = "WALPrefetch"; + break; case WAIT_EVENT_WAL_SYNC: event_name = "WALSync"; break; diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h index 2adc5df2e79..547fb0cd817 100644 --- a/src/include/utils/wait_event.h +++ b/src/include/utils/wait_event.h @@ -232,6 +232,7 @@ typedef enum WAIT_EVENT_WAL_INIT_SYNC, WAIT_EVENT_WAL_INIT_WRITE, WAIT_EVENT_WAL_READ, + WAIT_EVENT_WAL_PREFETCH, WAIT_EVENT_WAL_SYNC, WAIT_EVENT_WAL_SYNC_METHOD_ASSIGN, WAIT_EVENT_WAL_WRITE, From 890d9acf3166d42db5ec014231c1ed9ac2f4d55d Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 13 Apr 2021 16:26:33 +0300 Subject: [PATCH 04/54] Mdb-admin patch and regression tests Introduces 3 functions: extern bool mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId); extern void check_mdb_admin_is_member_of_role(Oid member, Oid role); extern bool mdb_admin_is_member_of_role(Oid member, Oid role); To check mdb admin belongship and role-to-role ownership transfer correctness. Our mdb_admin ACL model is the following: * Any roles user or/and roles can be granted with mdb_admin * mdb_admin member can tranfser ownershup of relations, namespaces and functions to other roles, if target role in neither: superuser, pg_read_server_files, pg_write_server_files nor pg_execute_server_program. * Allow mdb_admin to create LEAKPROOF functions * mdb admin sets session replication role * [MDB-16648 + MDB-17910]: Allow mdb admin to kill specific superuser queries MDB-27288: allow mdb_admin to kill autovac + tests mdb-27228: fix expected output --- src/backend/catalog/namespace.c | 19 +- src/backend/commands/alter.c | 11 +- src/backend/commands/functioncmds.c | 20 ++- src/backend/commands/schemacmds.c | 19 +- src/backend/commands/tablecmds.c | 16 +- src/backend/storage/ipc/signalfuncs.c | 34 +++- src/backend/utils/adt/acl.c | 163 +++++++++++++++++- src/include/utils/acl.h | 9 + src/test/Makefile | 2 +- src/test/mdb_admin/.gitignore | 2 + src/test/mdb_admin/Makefile | 23 +++ src/test/mdb_admin/t/signals.pl | 74 ++++++++ .../regress/expected/create_function_sql.out | 4 +- src/test/regress/expected/mdb_admin.out | 100 +++++++++++ src/test/regress/expected/test_setup.out | 1 + src/test/regress/parallel_schedule | 4 + src/test/regress/sql/mdb_admin.sql | 87 ++++++++++ src/test/regress/sql/test_setup.sql | 2 + 18 files changed, 563 insertions(+), 27 deletions(-) create mode 100644 src/test/mdb_admin/.gitignore create mode 100644 src/test/mdb_admin/Makefile create mode 100644 src/test/mdb_admin/t/signals.pl create mode 100644 src/test/regress/expected/mdb_admin.out create mode 100644 src/test/regress/sql/mdb_admin.sql diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c index e0d17eb569f..17588056007 100644 --- a/src/backend/catalog/namespace.c +++ b/src/backend/catalog/namespace.c @@ -2941,6 +2941,8 @@ LookupExplicitNamespace(const char *nspname, bool missing_ok) Oid namespaceId; AclResult aclresult; + HeapTuple tuple; + Oid ownerId; /* check for pg_temp alias */ if (strcmp(nspname, "pg_temp") == 0) { @@ -2958,7 +2960,22 @@ LookupExplicitNamespace(const char *nspname, bool missing_ok) if (missing_ok && !OidIsValid(namespaceId)) return InvalidOid; - aclresult = object_aclcheck(NamespaceRelationId, namespaceId, GetUserId(), ACL_USAGE); + tuple = SearchSysCache1(NAMESPACEOID, ObjectIdGetDatum(namespaceId)); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_SCHEMA), + errmsg("schema with OID %u does not exist", namespaceId))); + + ownerId = ((Form_pg_namespace) GETSTRUCT(tuple))->nspowner; + + ReleaseSysCache(tuple); + + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), ownerId)) { + aclresult = object_aclcheck(NamespaceRelationId, namespaceId, GetUserId(), ACL_USAGE); + } else { + aclresult = ACLCHECK_OK; + } + if (aclresult != ACLCHECK_OK) aclcheck_error(aclresult, OBJECT_SCHEMA, nspname); diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index 85f11a1ec61..efb344d087c 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -1008,7 +1008,8 @@ AlterObjectOwner_internal(Relation rel, Oid objectId, Oid new_ownerId) if (!superuser()) { /* must be owner */ - if (!has_privs_of_role(GetUserId(), old_ownerId)) + if (!has_privs_of_role(GetUserId(), old_ownerId) + && !mdb_admin_allow_bypass_owner_checks(GetUserId(), old_ownerId)) { char *objname; char namebuf[NAMEDATALEN]; @@ -1028,14 +1029,16 @@ AlterObjectOwner_internal(Relation rel, Oid objectId, Oid new_ownerId) aclcheck_error(ACLCHECK_NOT_OWNER, get_object_type(classId, objectId), objname); } - /* Must be able to become new owner */ - check_can_set_role(GetUserId(), new_ownerId); + + if (!mdb_admin_is_member_of_role(GetUserId(), new_ownerId)) { + /* Must be able to become new owner */ + check_can_set_role(GetUserId(), new_ownerId); + } /* New owner must have CREATE privilege on namespace */ if (OidIsValid(namespaceId)) { AclResult aclresult; - aclresult = object_aclcheck(NamespaceRelationId, namespaceId, new_ownerId, ACL_CREATE); if (aclresult != ACLCHECK_OK) diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index f63b5ef420b..4d2d71d6a05 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -1135,9 +1135,13 @@ CreateFunction(ParseState *pstate, CreateFunctionStmt *stmt) * by security barrier views or row-level security policies. */ if (isLeakProof && !superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("only superuser can define a leakproof function"))); + { + Oid role = get_role_oid("mdb_admin", true); + if (!is_member_of_role(GetUserId(), role)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("only superuser or mdb_admin can define a leakproof function"))); + } if (transformDefElem) { @@ -1420,9 +1424,13 @@ AlterFunction(ParseState *pstate, AlterFunctionStmt *stmt) { procForm->proleakproof = boolVal(leakproof_item->arg); if (procForm->proleakproof && !superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("only superuser can define a leakproof function"))); + { + Oid role = get_role_oid("mdb_admin", true); + if (!is_member_of_role(GetUserId(), role)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("only superuser or mdb_admin can define a leakproof function"))); + } } if (cost_item) { diff --git a/src/backend/commands/schemacmds.c b/src/backend/commands/schemacmds.c index 6eb3dc6bab6..e6704cb81c2 100644 --- a/src/backend/commands/schemacmds.c +++ b/src/backend/commands/schemacmds.c @@ -383,12 +383,16 @@ AlterSchemaOwner_internal(HeapTuple tup, Relation rel, Oid newOwnerId) AclResult aclresult; /* Otherwise, must be owner of the existing object */ - if (!object_ownercheck(NamespaceRelationId, nspForm->oid, GetUserId())) + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), nspForm->nspowner) + && !object_ownercheck(NamespaceRelationId, nspForm->oid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_SCHEMA, NameStr(nspForm->nspname)); - /* Must be able to become new owner */ - check_can_set_role(GetUserId(), newOwnerId); + + if (!mdb_admin_is_member_of_role(GetUserId(), newOwnerId)) { + /* Must be able to become new owner */ + check_can_set_role(GetUserId(), newOwnerId); + } /* * must have create-schema rights @@ -399,8 +403,14 @@ AlterSchemaOwner_internal(HeapTuple tup, Relation rel, Oid newOwnerId) * schemas. Because superusers will always have this right, we need * no special case for them. */ - aclresult = object_aclcheck(DatabaseRelationId, MyDatabaseId, GetUserId(), + + if (mdb_admin_allow_bypass_owner_checks(GetUserId(), nspForm->nspowner)) { + aclresult = ACLCHECK_OK; + } else { + aclresult = object_aclcheck(DatabaseRelationId, MyDatabaseId, GetUserId(), ACL_CREATE); + } + if (aclresult != ACLCHECK_OK) aclcheck_error(aclresult, OBJECT_DATABASE, get_database_name(MyDatabaseId)); @@ -431,7 +441,6 @@ AlterSchemaOwner_internal(HeapTuple tup, Relation rel, Oid newOwnerId) CatalogTupleUpdate(rel, &newtuple->t_self, newtuple); heap_freetuple(newtuple); - /* Update owner dependency reference */ changeDependencyOnOwner(NamespaceRelationId, nspForm->oid, newOwnerId); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 7aae9d208c4..59c07128f97 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -14234,13 +14234,18 @@ ATExecChangeOwner(Oid relationOid, Oid newOwnerId, bool recursing, LOCKMODE lock AclResult aclresult; /* Otherwise, must be owner of the existing object */ - if (!object_ownercheck(RelationRelationId, relationOid, GetUserId())) + + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), tuple_class->relowner) + && !object_ownercheck(RelationRelationId, relationOid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(get_rel_relkind(relationOid)), RelationGetRelationName(target_rel)); - /* Must be able to become new owner */ - check_can_set_role(GetUserId(), newOwnerId); + if (!mdb_admin_is_member_of_role(GetUserId(), newOwnerId)) { + /* Must be able to become new owner */ + check_can_set_role(GetUserId(), newOwnerId); + } + /* New owner must have CREATE privilege on namespace */ aclresult = object_aclcheck(NamespaceRelationId, namespaceOid, newOwnerId, ACL_CREATE); @@ -17386,7 +17391,7 @@ RangeVarCallbackForAlterRelation(const RangeVar *rv, Oid relid, Oid oldrelid, Form_pg_class classform; AclResult aclresult; char relkind; - + tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tuple)) return; /* concurrently dropped */ @@ -17394,7 +17399,8 @@ RangeVarCallbackForAlterRelation(const RangeVar *rv, Oid relid, Oid oldrelid, relkind = classform->relkind; /* Must own relation. */ - if (!object_ownercheck(RelationRelationId, relid, GetUserId())) + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), classform->relowner) + && !object_ownercheck(RelationRelationId, relid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(get_rel_relkind(relid)), rv->relname); /* No system table modifications unless explicitly allowed. */ diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index b595c2d6912..f41d6583489 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -20,6 +20,7 @@ #include "miscadmin.h" #include "pgstat.h" #include "postmaster/syslogger.h" +#include "postmaster/bgworker.h" #include "storage/pmsignal.h" #include "storage/proc.h" #include "storage/procarray.h" @@ -49,6 +50,8 @@ static int pg_signal_backend(int pid, int sig) { PGPROC *proc = BackendPidGetProc(pid); + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; /* * BackendPidGetProc returns NULL if the pid isn't valid; but by the time @@ -74,14 +77,41 @@ pg_signal_backend(int pid, int sig) return SIGNAL_BACKEND_ERROR; } + + local_beentry = pgstat_get_local_beentry_by_backend_id(proc->backendId); + if (local_beentry != NULL) + beentry = &local_beentry->backendStatus; /* * Only allow superusers to signal superuser-owned backends. Any process * not advertising a role might have the importance of a superuser-owned * backend, so treat it that way. */ if ((!OidIsValid(proc->roleId) || superuser_arg(proc->roleId)) && - !superuser()) - return SIGNAL_BACKEND_NOSUPERUSER; + !superuser()) { + Oid role; + char * appname; + + if (local_beentry == NULL) { + return SIGNAL_BACKEND_NOSUPERUSER; + } + + role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + appname = local_beentry->backendStatus.st_appname; + + // only allow mdb_admin to kill su queries + if (!is_member_of_role(GetUserId(), role)) { + return SIGNAL_BACKEND_NOSUPERUSER; + } + + /* mdb admin allowed to kill proc with application name 'MDB' or autovacuum */ + if (local_beentry->backendStatus.st_backendType == B_AUTOVAC_WORKER) { + // ok + } else if (appname != NULL && strcmp(appname, "MDB") == 0) { + // ok + } else { + return SIGNAL_BACKEND_NOSUPERUSER; + } + } /* Users can signal backends they have role membership in. */ if (!has_privs_of_role(GetUserId(), proc->roleId) && diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index d2278374d97..0cf2676fc32 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -123,6 +123,8 @@ static AclResult pg_role_aclcheck(Oid role_oid, Oid roleid, AclMode mode); static void RoleMembershipCacheCallback(Datum arg, int cacheid, uint32 hashvalue); +static bool has_privs_of_unwanted_system_role(Oid role); + /* * Test whether an identifier char can be left unquoted in ACLs. @@ -4974,6 +4976,58 @@ roles_is_member_of(Oid roleid, enum RoleRecurseType type, * * See also member_can_set_role, below. */ + +/* +* This is basically original postgresql privs-check function +*/ + +static bool +has_privs_of_role_strict(Oid member, Oid role) +{ + /* Fast path for simple case */ + if (member == role) + return true; + + /* Superusers have every privilege, so are part of every role */ + if (superuser_arg(member)) + return true; + + /* + * Find all the roles that member has the privileges of, including + * multi-level recursion, then see if target role is any one of them. + */ + return list_member_oid(roles_is_member_of(member, ROLERECURSE_PRIVS, + InvalidOid, NULL), + role); +} + +/* +* Check that role is either one of "dangerous" system role +* or has "strict" (not through mdb_admin) +* privs of this role +*/ + +static bool +has_privs_of_unwanted_system_role(Oid role) { + if (has_privs_of_role_strict(role, ROLE_PG_READ_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_WRITE_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_READ_ALL_DATA)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_WRITE_ALL_DATA)) { + return true; + } + + return false; +} + bool has_privs_of_role(Oid member, Oid role) { @@ -4994,6 +5048,47 @@ has_privs_of_role(Oid member, Oid role) role); } +// -- non-upstream patch begin +/* + * Is userId allowed to bypass ownership check + * and tranfer onwership to ownerId role? + */ +bool +mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) +{ + Oid mdb_admin_roleoid; + /* + * Never allow nobody to grant objects to + * superusers. + * This can result in various CVE. + * For paranoic reasons, check this even before + * membership of mdb_admin role. + */ + if (superuser_arg(ownerId)) { + return false; + } + + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + /* Is userId actually member of mdb admin? */ + if (!is_member_of_role(userId, mdb_admin_roleoid)) { + /* if no, disallow. */ + return false; + } + + /* + * Now, we need to check if ownerId + * is some dangerous role to trasfer membership to. + * + * For now, we check that ownerId does not have + * priviledge to execute server program or/and + * read/write server files, or/and pg read/write all data + */ + + /* All checks passed, hope will not be hacked here (again) */ + return !has_privs_of_unwanted_system_role(ownerId); +} + +// -- non-upstream patch end /* * Can member use SET ROLE to this role? * @@ -5041,6 +5136,73 @@ check_can_set_role(Oid member, Oid role) GetUserNameFromId(role, false)))); } +// -- mdb admin patch +/* + * check_mdb_admin_is_member_of_role + * is_member_of_role with a standard permission-violation error if not in usual case + * Is case `member` in mdb_admin we check that role is neither of superuser, pg_read/write + * server files nor pg_execute_server_program or pg_read/write all data + */ +void +check_mdb_admin_is_member_of_role(Oid member, Oid role) +{ + Oid mdb_admin_roleoid; + /* fast path - if we are superuser, its ok */ + if (superuser_arg(member)) { + return; + } + + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + /* Is userId actually member of mdb admin? */ + if (is_member_of_role(member, mdb_admin_roleoid)) { + + /* role is mdb admin */ + if (superuser_arg(role)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("cannot transfer ownership to superuser \"%s\"", + GetUserNameFromId(role, false)))); + } + + if (has_privs_of_unwanted_system_role(role)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("forbidden to transfer ownership to this system role in Cloud"))); + } + } else { + /* if no, check membership transfer in usual way. */ + + if (!is_member_of_role(member, role)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be member of role \"%s\"", + GetUserNameFromId(role, false)))); + } + } +} + +bool mdb_admin_is_member_of_role(Oid member, Oid role) { + Oid mdb_admin_roleoid; + /* fast path - if we are superuser, its ok */ + if (superuser_arg(member)) { + return true; + } + + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + /* Is userId actually member of mdb admin? */ + if (!is_member_of_role(member, mdb_admin_roleoid)) { + return false; + } + /* role is mdb admin */ + if (superuser_arg(role)) { + return false; + } + + return !has_privs_of_unwanted_system_role(role); +} + +// -- mdb admin patch + /* * Is member a member of role (directly or indirectly)? * @@ -5215,7 +5377,6 @@ select_best_grantor(Oid roleId, AclMode privileges, */ roles_list = roles_is_member_of(roleId, ROLERECURSE_PRIVS, InvalidOid, NULL); - /* initialize candidate result as default */ *grantorId = roleId; *grantOptions = ACL_NO_RIGHTS; diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index aba1afa971c..312a32557d3 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -214,6 +214,15 @@ extern void check_can_set_role(Oid member, Oid role); extern bool is_member_of_role(Oid member, Oid role); extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); + +// -- non-upstream patch begin +extern bool mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId); + +extern void check_mdb_admin_is_member_of_role(Oid member, Oid role); + +extern bool mdb_admin_is_member_of_role(Oid member, Oid role); +// -- non-upstream patch end + extern Oid select_best_admin(Oid member, Oid role); extern Oid get_role_oid(const char *rolname, bool missing_ok); extern Oid get_role_oid_or_public(const char *rolname); diff --git a/src/test/Makefile b/src/test/Makefile index dbd3192874d..9563bdcdd0e 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -12,7 +12,7 @@ subdir = src/test top_builddir = ../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = perl regress isolation modules authentication recovery subscription +SUBDIRS = perl regress isolation modules authentication recovery subscription mdb_admin ifeq ($(with_icu),yes) SUBDIRS += icu diff --git a/src/test/mdb_admin/.gitignore b/src/test/mdb_admin/.gitignore new file mode 100644 index 00000000000..871e943d50e --- /dev/null +++ b/src/test/mdb_admin/.gitignore @@ -0,0 +1,2 @@ +# Generated by test suite +/tmp_check/ diff --git a/src/test/mdb_admin/Makefile b/src/test/mdb_admin/Makefile new file mode 100644 index 00000000000..e4e82367da9 --- /dev/null +++ b/src/test/mdb_admin/Makefile @@ -0,0 +1,23 @@ +#------------------------------------------------------------------------- +# +# Makefile for src/test/mdb_admin +# +# Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# +# src/test/mdb_admin/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/test/mdb_admin +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +check: + $(prove_check) + +installcheck: + $(prove_installcheck) + +clean distclean maintainer-clean: + rm -rf tmp_check diff --git a/src/test/mdb_admin/t/signals.pl b/src/test/mdb_admin/t/signals.pl new file mode 100644 index 00000000000..a4b3f07fb89 --- /dev/null +++ b/src/test/mdb_admin/t/signals.pl @@ -0,0 +1,74 @@ + +# Copyright (c) 2024-2024, MDB, Mother Russia + +# Minimal test testing streaming replication +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize primary node +my $node_primary = PostgreSQL::Test::Cluster->new('primary'); +$node_primary->init(); +$node_primary->start; + +# Create some content on primary and check its presence in standby nodes +$node_primary->safe_psql('postgres', + " + CREATE DATABASE regress; + CREATE ROLE mdb_admin; + CREATE ROLE mdb_reg_lh_1; + CREATE ROLE mdb_reg_lh_2; + GRANT pg_signal_backend TO mdb_admin; + GRANT pg_signal_backend TO mdb_reg_lh_1; + GRANT mdb_admin TO mdb_reg_lh_2; +"); + +# Create some content on primary and check its presence in standby nodes +$node_primary->safe_psql('regress', + " + CREATE TABLE tab_int(i int); + INSERT INTO tab_int SELECT * FROm generate_series(1, 1000000); + ALTER SYSTEM SET autovacuum_vacuum_cost_limit TO 1; + ALTER SYSTEM SET autovacuum_vacuum_cost_delay TO 100; + ALTER SYSTEM SET autovacuum_naptime TO 1; +"); + +$node_primary->restart; + +sleep 1; + +my $res_pid = $node_primary->safe_psql('regress', + " + SELECT pid FROM pg_stat_activity WHERE backend_type = 'autovacuum worker' and datname = 'regress';; +"); + + +print "pid is $res_pid\n"; + +ok(1); + + +my ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_1; + SELECT pg_terminate_backend($res_pid); +"); + +# print ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1, "\n"); + +ok($res_reg_lh_1 != 0, "should fail for non-mdb_admin"); +like($stderr_reg_lh_1, qr/Only roles with the SUPERUSER attribute may terminate processes of roles with the SUPERUSER attribute./, "matches"); + +my ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_2; + SELECT pg_terminate_backend($res_pid); +"); + +ok($res_reg_lh_2 == 0, "should success for mdb_admin"); + +# print ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2, "\n"); + +done_testing(); diff --git a/src/test/regress/expected/create_function_sql.out b/src/test/regress/expected/create_function_sql.out index 50aca5940ff..96b40fdb56d 100644 --- a/src/test/regress/expected/create_function_sql.out +++ b/src/test/regress/expected/create_function_sql.out @@ -166,10 +166,10 @@ SET SESSION AUTHORIZATION regress_unpriv_user; SET search_path TO temp_func_test, public; ALTER FUNCTION functest_E_1(int) NOT LEAKPROOF; ALTER FUNCTION functest_E_2(int) LEAKPROOF; -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function CREATE FUNCTION functest_E_3(int) RETURNS bool LANGUAGE 'sql' LEAKPROOF AS 'SELECT $1 < 200'; -- fail -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function RESET SESSION AUTHORIZATION; -- -- CALLED ON NULL INPUT | RETURNS NULL ON NULL INPUT | STRICT diff --git a/src/test/regress/expected/mdb_admin.out b/src/test/regress/expected/mdb_admin.out new file mode 100644 index 00000000000..20722750c05 --- /dev/null +++ b/src/test/regress/expected/mdb_admin.out @@ -0,0 +1,100 @@ +CREATE ROLE regress_mdb_admin_user1; +CREATE ROLE regress_mdb_admin_user2; +CREATE ROLE regress_mdb_admin_user3; +CREATE ROLE regress_superuser WITH SUPERUSER; +GRANT mdb_admin TO regress_mdb_admin_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user3; +-- mdb admin trasfers ownership to another role +SET ROLE regress_mdb_admin_user2; +CREATE FUNCTION regress_mdb_admin_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; +CREATE SCHEMA regress_mdb_admin_schema; +GRANT CREATE ON SCHEMA regress_mdb_admin_schema TO regress_mdb_admin_user3; +CREATE TABLE regress_mdb_admin_schema.regress_mdb_admin_table(); +CREATE TABLE regress_mdb_admin_table(); +CREATE VIEW regress_mdb_admin_view as SELECT 1; +SET ROLE regress_mdb_admin_user1; +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_mdb_admin_user3; +ALTER VIEW regress_mdb_admin_view OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; +-- mdb admin fails to transfer ownership to superusers and particular system roles +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER TABLE regress_mdb_admin_table OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +-- end tests +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user3; +DROP VIEW regress_mdb_admin_view; +DROP FUNCTION regress_mdb_admin_add; +DROP TABLE regress_mdb_admin_schema.regress_mdb_admin_table; +DROP TABLE regress_mdb_admin_table; +DROP SCHEMA regress_mdb_admin_schema; +DROP ROLE regress_mdb_admin_user1; +DROP ROLE regress_mdb_admin_user2; +DROP ROLE regress_mdb_admin_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/expected/test_setup.out b/src/test/regress/expected/test_setup.out index 5d9e6bf12bc..c9a4dd96a38 100644 --- a/src/test/regress/expected/test_setup.out +++ b/src/test/regress/expected/test_setup.out @@ -243,3 +243,4 @@ create function fipshash(text) returns text strict immutable parallel safe leakproof return substr(encode(sha256($1::bytea), 'hex'), 1, 32); +CREATE ROLE mdb_admin; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 84e3710fb68..635518e1ec4 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -11,6 +11,10 @@ # required setup steps test: test_setup +# mdb admin simple checks + +test: mdb_admin + # ---------- # The first group of parallel tests # ---------- diff --git a/src/test/regress/sql/mdb_admin.sql b/src/test/regress/sql/mdb_admin.sql new file mode 100644 index 00000000000..b6b048e5692 --- /dev/null +++ b/src/test/regress/sql/mdb_admin.sql @@ -0,0 +1,87 @@ +CREATE ROLE regress_mdb_admin_user1; +CREATE ROLE regress_mdb_admin_user2; +CREATE ROLE regress_mdb_admin_user3; + +CREATE ROLE regress_superuser WITH SUPERUSER; + +GRANT mdb_admin TO regress_mdb_admin_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user3; + +-- mdb admin trasfers ownership to another role + +SET ROLE regress_mdb_admin_user2; +CREATE FUNCTION regress_mdb_admin_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; + +CREATE SCHEMA regress_mdb_admin_schema; +GRANT CREATE ON SCHEMA regress_mdb_admin_schema TO regress_mdb_admin_user3; +CREATE TABLE regress_mdb_admin_schema.regress_mdb_admin_table(); +CREATE TABLE regress_mdb_admin_table(); +CREATE VIEW regress_mdb_admin_view as SELECT 1; +SET ROLE regress_mdb_admin_user1; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_mdb_admin_user3; +ALTER VIEW regress_mdb_admin_view OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; + + +-- mdb admin fails to transfer ownership to superusers and particular system roles + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; +ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_superuser; +ALTER TABLE regress_mdb_admin_table OWNER TO regress_superuser; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_superuser; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_execute_server_program; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_execute_server_program; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_execute_server_program; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_execute_server_program; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_execute_server_program; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_server_files; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_server_files; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_server_files; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_server_files; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_server_files; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_server_files; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_server_files; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_server_files; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_all_data; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_all_data; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_all_data; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_all_data; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_all_data; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_all_data; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_all_data; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_all_data; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; + +-- end tests + +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user3; + +DROP VIEW regress_mdb_admin_view; +DROP FUNCTION regress_mdb_admin_add; +DROP TABLE regress_mdb_admin_schema.regress_mdb_admin_table; +DROP TABLE regress_mdb_admin_table; +DROP SCHEMA regress_mdb_admin_schema; +DROP ROLE regress_mdb_admin_user1; +DROP ROLE regress_mdb_admin_user2; +DROP ROLE regress_mdb_admin_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/sql/test_setup.sql b/src/test/regress/sql/test_setup.sql index 1b2d434683b..18723170507 100644 --- a/src/test/regress/sql/test_setup.sql +++ b/src/test/regress/sql/test_setup.sql @@ -299,3 +299,5 @@ create function fipshash(text) returns text strict immutable parallel safe leakproof return substr(encode(sha256($1::bytea), 'hex'), 1, 32); + +CREATE ROLE mdb_admin; From ee2a2c9e78182eb08fda194dfa6d22054ca7cbed Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Sun, 6 Dec 2020 10:01:59 +0500 Subject: [PATCH 05/54] MDB replication role patch MDB replication role regression tests Patch allows user with mdb_replication role to use pg_create_logical_replication_slot, pg_replication_slot_advance, pg_drop_replication_slot functions to manage logical replication slots. Also, users with mdb_admin (which is memner of pg_create_subscription) can create subscribptions. Slot names starting with MDB.* are forbidden. Add run as owner tap tests More test cases in mdb_102 --- .../test_decoding/expected/permissions.out | 94 ++++- contrib/test_decoding/sql/permissions.sql | 29 ++ src/backend/commands/subscriptioncmds.c | 10 +- src/backend/postmaster/syslogger.c | 2 + .../replication/logical/logicalfuncs.c | 6 +- src/backend/replication/logical/relation.c | 7 + src/backend/replication/logical/tablesync.c | 7 + src/backend/replication/logical/worker.c | 31 +- src/backend/replication/slot.c | 48 +++ src/backend/replication/slotfuncs.c | 14 +- src/backend/replication/walsender.c | 28 ++ src/backend/utils/init/postinit.c | 14 +- src/include/replication/slot.h | 27 ++ src/include/replication/walsender.h | 4 + .../t/101_logical_decoding_mdb_replication.pl | 159 +++++++++ src/test/regress/expected/mdb_replication.out | 35 ++ src/test/regress/expected/test_setup.out | 3 + src/test/regress/parallel_schedule | 4 + src/test/regress/sql/mdb_replication.sql | 41 +++ src/test/regress/sql/test_setup.sql | 4 + .../t/101_mdb_rep_changes_nonsuperuser.pl | 336 ++++++++++++++++++ .../t/102_mdb_check_permissions.pl | 202 +++++++++++ 22 files changed, 1089 insertions(+), 16 deletions(-) create mode 100644 src/test/recovery/t/101_logical_decoding_mdb_replication.pl create mode 100644 src/test/regress/expected/mdb_replication.out create mode 100644 src/test/regress/sql/mdb_replication.sql create mode 100644 src/test/subscription/t/101_mdb_rep_changes_nonsuperuser.pl create mode 100644 src/test/subscription/t/102_mdb_check_permissions.pl diff --git a/contrib/test_decoding/expected/permissions.out b/contrib/test_decoding/expected/permissions.out index d6eaba8c55d..6ba2594f579 100644 --- a/contrib/test_decoding/expected/permissions.out +++ b/contrib/test_decoding/expected/permissions.out @@ -2,6 +2,9 @@ SET synchronous_commit = on; -- setup CREATE ROLE regress_lr_normal; +CREATE ROLE mdb_replication; +CREATE ROLE regress_lr_mdb_replication; +GRANT mdb_replication TO regress_lr_mdb_replication; CREATE ROLE regress_lr_superuser SUPERUSER; CREATE ROLE regress_lr_replication REPLICATION; CREATE TABLE lr_test(data text); @@ -28,6 +31,24 @@ SELECT pg_drop_replication_slot('regression_slot'); (1 row) +-- superuser can control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); + data +------ +(0 rows) + +SELECT pg_drop_replication_slot('mdb_regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + RESET ROLE; -- replication user can control replication SET ROLE regress_lr_replication; @@ -50,20 +71,85 @@ SELECT pg_drop_replication_slot('regression_slot'); (1 row) +-- replication user can control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); + data +------ +(0 rows) + +SELECT pg_drop_replication_slot('mdb_regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +RESET ROLE; +-- mdb_replication user can control replication +SET ROLE regress_lr_mdb_replication; +SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +INSERT INTO lr_test VALUES('lr_mdb_replication_init'); +ERROR: permission denied for table lr_test +SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); + data +------ +(0 rows) + +SELECT pg_drop_replication_slot('regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + +-- mdb_replication user can't control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot2', 'test_decoding'); +ERROR: slot name "mdb_regression_slot2" is reserved +DETAIL: Slot names starting with "mdb" are reserved. +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +ERROR: slot name "mdb_regression_slot" is reserved +DETAIL: Slot names starting with "mdb" are reserved. +SELECT pg_drop_replication_slot('mdb_regression_slot'); +ERROR: slot name "mdb_regression_slot" is reserved +DETAIL: Slot names starting with "mdb" are reserved. +RESET ROLE; +-- cleanup +SET ROLE regress_lr_superuser; +SELECT pg_drop_replication_slot('mdb_regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + RESET ROLE; -- plain user *can't* can control replication SET ROLE regress_lr_normal; SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding'); ERROR: permission denied to use replication slots -DETAIL: Only roles with the REPLICATION attribute may use replication slots. +DETAIL: must be superuser, replication role or mdb_replication to use replication slots INSERT INTO lr_test VALUES('lr_superuser_init'); ERROR: permission denied for table lr_test SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); ERROR: permission denied to use replication slots -DETAIL: Only roles with the REPLICATION attribute may use replication slots. +DETAIL: must be superuser, replication role or mdb_replication to use replication slots SELECT pg_drop_replication_slot('regression_slot'); ERROR: permission denied to use replication slots -DETAIL: Only roles with the REPLICATION attribute may use replication slots. +DETAIL: must be superuser, replication role or mdb_replication to use replication slots RESET ROLE; -- replication users can drop superuser created slots SET ROLE regress_lr_superuser; @@ -94,7 +180,7 @@ RESET ROLE; SET ROLE regress_lr_normal; SELECT pg_drop_replication_slot('regression_slot'); ERROR: permission denied to use replication slots -DETAIL: Only roles with the REPLICATION attribute may use replication slots. +DETAIL: must be superuser, replication role or mdb_replication to use replication slots RESET ROLE; -- all users can see existing slots SET ROLE regress_lr_superuser; diff --git a/contrib/test_decoding/sql/permissions.sql b/contrib/test_decoding/sql/permissions.sql index 312b5145937..a21c35effbb 100644 --- a/contrib/test_decoding/sql/permissions.sql +++ b/contrib/test_decoding/sql/permissions.sql @@ -3,6 +3,9 @@ SET synchronous_commit = on; -- setup CREATE ROLE regress_lr_normal; +CREATE ROLE mdb_replication; +CREATE ROLE regress_lr_mdb_replication; +GRANT mdb_replication TO regress_lr_mdb_replication; CREATE ROLE regress_lr_superuser SUPERUSER; CREATE ROLE regress_lr_replication REPLICATION; CREATE TABLE lr_test(data text); @@ -13,6 +16,10 @@ SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_d INSERT INTO lr_test VALUES('lr_superuser_init'); SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); SELECT pg_drop_replication_slot('regression_slot'); +-- superuser can control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +SELECT pg_drop_replication_slot('mdb_regression_slot'); RESET ROLE; -- replication user can control replication @@ -21,6 +28,28 @@ SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_d INSERT INTO lr_test VALUES('lr_superuser_init'); SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); SELECT pg_drop_replication_slot('regression_slot'); +-- replication user can control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +SELECT pg_drop_replication_slot('mdb_regression_slot'); +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); +RESET ROLE; + +-- mdb_replication user can control replication +SET ROLE regress_lr_mdb_replication; +SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding'); +INSERT INTO lr_test VALUES('lr_mdb_replication_init'); +SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +SELECT pg_drop_replication_slot('regression_slot'); +-- mdb_replication user can't control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot2', 'test_decoding'); +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +SELECT pg_drop_replication_slot('mdb_regression_slot'); +RESET ROLE; + +-- cleanup +SET ROLE regress_lr_superuser; +SELECT pg_drop_replication_slot('mdb_regression_slot'); RESET ROLE; -- plain user *can't* can control replication diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index eddda7455a9..23c10e8861f 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -580,7 +580,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, bits32 supported_opts; SubOpts opts = {0}; AclResult aclresult; - + /* * Parse and check options. * @@ -608,6 +608,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, * attempts to access arbitrary network destinations, so require the user * to have been specifically authorized to create subscriptions. */ + /* MDB: mdb_admin need to be granted with pg_create_subscription role */ if (!has_privs_of_role(owner, ROLE_PG_CREATE_SUBSCRIPTION)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -1839,8 +1840,11 @@ AlterSubscriptionOwner_internal(Relation rel, HeapTuple tup, Oid newOwnerId) errmsg("password_required=false is superuser-only"), errhint("Subscriptions with the password_required option set to false may only be created or modified by the superuser."))); - /* Must be able to become new owner */ - check_can_set_role(GetUserId(), newOwnerId); + /* if we are mdb_admin, check that we alter ownership to unprivileged role */ + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), newOwnerId)) { + /* else we must be able to become new owner */ + check_can_set_role(GetUserId(), newOwnerId); + } /* * current owner must have CREATE on database diff --git a/src/backend/postmaster/syslogger.c b/src/backend/postmaster/syslogger.c index 858a2f6b2b9..c7938f7996c 100644 --- a/src/backend/postmaster/syslogger.c +++ b/src/backend/postmaster/syslogger.c @@ -316,6 +316,8 @@ SysLoggerMain(int argc, char *argv[]) #ifndef WIN32 AddWaitEventToSet(wes, WL_SOCKET_READABLE, syslogPipe[0], NULL, NULL); #endif + fclose(stderr); + fclose(stdout); /* main worker loop */ for (;;) diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 55a24c02c94..63e2a9d9d37 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -31,6 +31,7 @@ #include "replication/logical.h" #include "replication/message.h" #include "storage/fd.h" +#include "utils/acl.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/inval.h" @@ -116,8 +117,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin List *options = NIL; DecodingOutputState *p; - CheckSlotPermissions(); - + CheckMDBReplSlotPermissions(); CheckLogicalDecodingRequirements(); if (PG_ARGISNULL(0)) @@ -126,6 +126,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin errmsg("slot name must not be null"))); name = PG_GETARG_NAME(0); + CheckMDBReservedName(NameStr(*name)); + if (PG_ARGISNULL(1)) upto_lsn = InvalidXLogRecPtr; else diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c index 6edbd5643ae..0ec706d6aab 100644 --- a/src/backend/replication/logical/relation.c +++ b/src/backend/replication/logical/relation.c @@ -19,6 +19,7 @@ #include "access/genam.h" #include "access/table.h" +#include "catalog/catalog.h" #include "catalog/namespace.h" #include "catalog/pg_am_d.h" #include "catalog/pg_subscription_rel.h" @@ -398,6 +399,12 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) entry->localrel = table_open(relid, NoLock); entry->localreloid = relid; + /* Don't allow catalog access */ + if (IsSystemClass(relid, entry->localrel->rd_rel)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("logical replication target relation \"%s.%s\" is a system class", + remoterel->nspname, remoterel->relname))); /* Check for supported relkind. */ CheckSubscriptionRelkind(entry->localrel->rd_rel->relkind, remoterel->nspname, remoterel->relname); diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 2d368ddfee6..8f6f4628481 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -121,6 +121,7 @@ #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/usercontext.h" +#include "utils/acl.h" typedef enum { @@ -1136,6 +1137,7 @@ copy_table(Relation rel) List *attnamelist; ParseState *pstate; List *options = NIL; + AclResult aclresult; /* Get the publisher relation info. */ fetch_remote_table_info(get_namespace_name(RelationGetNamespace(rel)), @@ -1148,6 +1150,11 @@ copy_table(Relation rel) relmapentry = logicalrep_rel_open(lrel.remoteid, NoLock); Assert(rel == relmapentry->localrel); + /* Check permission on table. */ + aclresult = pg_class_aclcheck(RelationGetRelid(rel), GetUserId(), ACL_INSERT); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->rd_rel->relkind), RelationGetRelationName(rel)); + /* Start copy on the publisher. */ initStringInfo(&cmd); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index d6bbffd7c8d..9f22df79173 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -209,6 +209,7 @@ #include "utils/syscache.h" #include "utils/timeout.h" #include "utils/usercontext.h" +#include "utils/acl.h" #define NAPTIME_PER_CYCLE 1000 /* max sleep time between cycles (1s) */ @@ -2394,6 +2395,7 @@ apply_handle_insert(StringInfo s) TupleTableSlot *remoteslot; MemoryContext oldctx; bool run_as_owner; + AclResult aclresult; /* * Quick return if we are skipping data modification changes or handling @@ -2428,6 +2430,11 @@ apply_handle_insert(StringInfo s) /* Set relation for error callback */ apply_error_callback_arg.rel = rel; + aclresult = pg_class_aclcheck(RelationGetRelid(rel->localrel), GetUserId(), + ACL_INSERT); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->localrel->rd_rel->relkind), + RelationGetRelationName(rel->localrel)); /* Initialize the executor state. */ edata = create_edata_for_relation(rel); @@ -2546,6 +2553,7 @@ apply_handle_update(StringInfo s) RTEPermissionInfo *target_perminfo; MemoryContext oldctx; bool run_as_owner; + AclResult aclresult; /* * Quick return if we are skipping data modification changes or handling @@ -2585,6 +2593,14 @@ apply_handle_update(StringInfo s) if (!run_as_owner) SwitchToUntrustedUser(rel->localrel->rd_rel->relowner, &ucxt); + /* MDB check acl on relation */ + aclresult = pg_class_aclcheck(RelationGetRelid(rel->localrel), GetUserId(), + ACL_UPDATE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->localrel->rd_rel->relkind), + RelationGetRelationName(rel->localrel)); + + /* Initialize the executor state. */ edata = create_edata_for_relation(rel); estate = edata->estate; @@ -2727,6 +2743,7 @@ apply_handle_delete(StringInfo s) TupleTableSlot *remoteslot; MemoryContext oldctx; bool run_as_owner; + AclResult aclresult; /* * Quick return if we are skipping data modification changes or handling @@ -2753,7 +2770,6 @@ apply_handle_delete(StringInfo s) /* Set relation for error callback */ apply_error_callback_arg.rel = rel; - /* Check if we can do the delete. */ check_relation_updatable(rel); @@ -2765,6 +2781,12 @@ apply_handle_delete(StringInfo s) if (!run_as_owner) SwitchToUntrustedUser(rel->localrel->rd_rel->relowner, &ucxt); + aclresult = pg_class_aclcheck(RelationGetRelid(rel->localrel), GetUserId(), + ACL_DELETE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->localrel->rd_rel->relkind), + RelationGetRelationName(rel->localrel)); + /* Initialize the executor state. */ edata = create_edata_for_relation(rel); estate = edata->estate; @@ -3168,6 +3190,7 @@ apply_handle_truncate(StringInfo s) List *relids_logged = NIL; ListCell *lc; LOCKMODE lockmode = AccessExclusiveLock; + AclResult aclresult; /* * Quick return if we are skipping data modification changes or handling @@ -3197,6 +3220,12 @@ apply_handle_truncate(StringInfo s) continue; } + aclresult = pg_class_aclcheck(RelationGetRelid(rel->localrel), GetUserId(), + ACL_TRUNCATE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->localrel->rd_rel->relkind), + RelationGetRelationName(rel->localrel)); + remote_rels = lappend(remote_rels, rel); TargetPrivilegesCheck(rel->localrel, ACL_TRUNCATE); rels = lappend(rels, rel->localrel); diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index aa017451ccc..b5271921e6d 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -52,6 +52,7 @@ #include "storage/proc.h" #include "storage/procarray.h" #include "utils/builtins.h" +#include "utils/acl.h" /* * Replication slot on-disk data structure. @@ -1222,6 +1223,53 @@ CheckSlotPermissions(void) "REPLICATION"))); } + + +/* + * Check whether the user has privilege to use replication slots. + */ +void +CheckRoleMDBReplSlotPermissions(bool role_has_rolreplication, bool is_member_of_mdb_replication) +{ + /* superuser can do it */ + if (superuser()) { + return; + } + + /* mdb_replication can do it */ + if (is_member_of_mdb_replication) { + return; + } + + + if (!role_has_rolreplication) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied to use replication slots"), + errdetail("must be superuser, replication role or mdb_replication to use replication slots"))); +} + + +void +CheckRoleUseMDBReservedName(const char *name, bool role_has_rolreplication) +{ + /* superuser can do it */ + if (superuser()) { + return; + } + /* ugly coding for speed (taken from IsReservedName) */ + if (name[0] == 'm' && + name[1] == 'd' && + name[2] == 'b' && + !role_has_rolreplication) + { + ereport(ERROR, + (errcode(ERRCODE_RESERVED_NAME), + errmsg("slot name \"%s\" is reserved", name), + errdetail("Slot names starting with \"mdb\" are reserved."))); + } +} + /* * Reserve WAL for the currently active slot. * diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 612bdd99b52..02240021e58 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -25,6 +25,7 @@ #include "utils/inval.h" #include "utils/pg_lsn.h" #include "utils/resowner.h" +#include "utils/acl.h" /* * Helper function for creating a new physical replication slot with @@ -78,6 +79,7 @@ pg_create_physical_replication_slot(PG_FUNCTION_ARGS) elog(ERROR, "return type must be a row type"); CheckSlotPermissions(); + CheckMDBReservedName(NameStr(*name)); CheckSlotRequirements(); @@ -180,7 +182,8 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "return type must be a row type"); - CheckSlotPermissions(); + CheckMDBReplSlotPermissions(); + CheckMDBReservedName(NameStr(*name)); CheckLogicalDecodingRequirements(); @@ -216,7 +219,9 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) { Name name = PG_GETARG_NAME(0); - CheckSlotPermissions(); + /* mdb replication allowed */ + CheckMDBReplSlotPermissions(); + CheckMDBReservedName(NameStr(*name)); CheckSlotRequirements(); @@ -594,7 +599,8 @@ pg_replication_slot_advance(PG_FUNCTION_ARGS) Assert(!MyReplicationSlot); - CheckSlotPermissions(); + CheckMDBReplSlotPermissions(); + CheckMDBReservedName(NameStr(*slotname)); if (XLogRecPtrIsInvalid(moveto)) ereport(ERROR, @@ -694,6 +700,8 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) elog(ERROR, "return type must be a row type"); CheckSlotPermissions(); + CheckMDBReservedName(NameStr(*src_name)); + CheckMDBReservedName(NameStr(*dst_name)); if (logical_slot) CheckLogicalDecodingRequirements(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index b37f1d19b27..36ea981addc 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -117,6 +117,13 @@ WalSnd *MyWalSnd = NULL; /* Global state */ bool am_walsender = false; /* Am I a walsender process? */ + +/* These variables defined in InitPostgres and used in walsender logic for priv +* check without CatCache search. +*/ +bool role_has_rolreplication = false; /* has replication privelege */ +bool member_of_mdb_replication = false; /* member of mdb replication role */ + bool am_cascading_walsender = false; /* Am I cascading WAL to another * standby? */ bool am_db_walsender = false; /* Connected to a database? */ @@ -210,6 +217,20 @@ typedef struct /* The size of our buffer of time samples. */ #define LAG_TRACKER_BUFFER_SIZE 8192 +static void +check_permissions(void) +{ + /* superuser can do it */ + if (superuser()) { + return; + } + /* else should have REPLICATION role option */ + if (!role_has_rolreplication) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + (errmsg("must be superuser or replication role to use replication slots")))); +} + /* A mechanism for tracking replication lag. */ typedef struct { @@ -1075,6 +1096,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) if (cmd->kind == REPLICATION_KIND_PHYSICAL) { + check_permissions(); ReplicationSlotCreate(cmd->slotname, false, cmd->temporary ? RS_TEMPORARY : RS_PERSISTENT, false); @@ -1082,6 +1104,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) else { CheckLogicalDecodingRequirements(); + CheckRoleMDBReplSlotPermissions(role_has_rolreplication, member_of_mdb_replication); + CheckRoleUseMDBReservedName(cmd->slotname, role_has_rolreplication); /* * Initially create persistent slot as ephemeral - that allows us to @@ -1261,6 +1285,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) static void DropReplicationSlot(DropReplicationSlotCmd *cmd) { + CheckRoleUseMDBReservedName(cmd->slotname, role_has_rolreplication); ReplicationSlotDrop(cmd->slotname, !cmd->wait); } @@ -1274,6 +1299,8 @@ StartLogicalReplication(StartReplicationCmd *cmd) StringInfoData buf; QueryCompletion qc; + CheckRoleUseMDBReservedName(cmd->slotname, role_has_rolreplication); + /* make sure that our requirements are still fulfilled */ CheckLogicalDecodingRequirements(); @@ -1833,6 +1860,7 @@ exec_replication_command(const char *cmd_string) break; case T_BaseBackupCmd: + check_permissions(); cmdtag = "BASE_BACKUP"; set_ps_display(cmdtag); PreventInTransactionBlock(true, cmdtag); diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 35aa077a850..3c7e9620129 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -728,7 +728,7 @@ InitPostgres(const char *in_dbname, Oid dboid, bool am_superuser; char *fullpath; char dbname[NAMEDATALEN]; - int nfree = 0; + int nfree; elog(DEBUG3, "InitPostgres"); @@ -962,8 +962,16 @@ InitPostgres(const char *in_dbname, Oid dboid, if (am_walsender) { Assert(!bootstrap); - - if (!has_rolreplication(GetUserId())) + /* define this variable for later use in permission checks function. */ + /* we cannot use has_rolreplication directly because catcache search is prohibited + * in no active tx state. + */ + role_has_rolreplication = has_rolreplication(GetUserId()); + member_of_mdb_replication = is_member_of_role(GetUserId(), get_role_oid("mdb_replication", true)); + + /* has_rolreplication returns true in case of superuser_arg(role) */ + /* should have REPLICATION role or be a member of mdb_replication to start walsender */ + if (!role_has_rolreplication && !member_of_mdb_replication) ereport(FATAL, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied to start WAL sender"), diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index 96e9181aeb2..0f20a4596de 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -9,6 +9,8 @@ #ifndef SLOT_H #define SLOT_H +#include "utils/acl.h" +#include "miscadmin.h" #include "access/xlog.h" #include "access/xlogreader.h" #include "storage/condition_variable.h" @@ -248,4 +250,29 @@ extern void CheckPointReplicationSlots(void); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); +/* +* Base function for CheckMDBReplSlotPermissions, but does not +* perform cat cache search (for no-transaction state case), +* using pre-defined bool variables. +*/ +extern void CheckRoleMDBReplSlotPermissions(bool role_has_rolreplication, bool is_member_of_mdb_replication); + +/* +* Same as CheckMDBReservedName, but does not +* perform cat cache search (for no-transaction state case), +* using pre-defined bool variables, defined +* in InitPostrges. +*/ +extern void CheckRoleUseMDBReservedName(const char* name, bool role_has_rolreplication); + +inline void CheckMDBReservedName(const char* name) { + CheckRoleUseMDBReservedName(name, has_rolreplication(GetUserId())); +} + +inline void CheckMDBReplSlotPermissions(void) { + Oid role; + role = get_role_oid("mdb_replication", /* missing ok*/ true); + return CheckRoleMDBReplSlotPermissions(has_rolreplication(GetUserId()), is_member_of_role(GetUserId(), role)); +} + #endif /* SLOT_H */ diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index 9df7e50f943..48799551486 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -30,6 +30,10 @@ extern PGDLLIMPORT bool am_cascading_walsender; extern PGDLLIMPORT bool am_db_walsender; extern PGDLLIMPORT bool wake_wal_senders; +/* for mdb permission checks */ +extern PGDLLIMPORT bool role_has_rolreplication; +extern PGDLLIMPORT bool member_of_mdb_replication; + /* user-settable parameters */ extern PGDLLIMPORT int max_wal_senders; extern PGDLLIMPORT int wal_sender_timeout; diff --git a/src/test/recovery/t/101_logical_decoding_mdb_replication.pl b/src/test/recovery/t/101_logical_decoding_mdb_replication.pl new file mode 100644 index 00000000000..2c577ab439a --- /dev/null +++ b/src/test/recovery/t/101_logical_decoding_mdb_replication.pl @@ -0,0 +1,159 @@ +# Testing of logical decoding using SQL interface and/or pg_recvlogical +# +# Most logical decoding tests are in contrib/test_decoding. This module +# is for work that doesn't fit well there, like where server restarts +# are required. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 10; +use Config; + + +# Initialize master node +my $node_master = PostgreSQL::Test::Cluster->new('master'); +$node_master->init(allows_streaming => 1); +$node_master->append_conf( + 'postgresql.conf', qq( +wal_level = logical +)); +$node_master->start; +my $backup_name = 'master_backup'; + +$node_master->safe_psql('postgres', + "SET password_encryption='md5'; CREATE ROLE normal LOGIN PASSWORD 'pass';"); +$node_master->safe_psql('postgres', + "GRANT CREATE ON DATABASE postgres TO normal;"); +$node_master->safe_psql('postgres', + "GRANT ALL ON SCHEMA public TO normal;"); +$node_master->safe_psql('postgres', + "CREATE ROLE mdb_replication NOLOGIN;"); +$node_master->safe_psql('postgres', + "GRANT mdb_replication TO normal;"); + +$node_master->safe_psql('postgres', + qq[CREATE TABLE decoding_test(x integer, y text);], extra_params => [ '-U', 'normal' ]); + +$node_master->safe_psql('postgres', + qq[SELECT pg_create_logical_replication_slot('test_slot', 'test_decoding');], + extra_params => [ '-U', 'normal' ]); + +$node_master->safe_psql('postgres', + qq[INSERT INTO decoding_test(x,y) SELECT s, s::text FROM generate_series(1,10) s;], + extra_params => [ '-U', 'normal' ] +); + +# Basic decoding works +my ($result) = $node_master->safe_psql('postgres', + qq[SELECT pg_logical_slot_get_changes('test_slot', NULL, NULL);], + extra_params => [ '-U', 'normal' ]); +is(scalar(my @foobar = split /^/m, $result), + 12, 'Decoding produced 12 rows inc BEGIN/COMMIT'); + +# If we immediately crash the server we might lose the progress we just made +# and replay the same changes again. But a clean shutdown should never repeat +# the same changes when we use the SQL decoding interface. +$node_master->restart('fast'); + +# There are no new writes, so the result should be empty. +$result = $node_master->safe_psql('postgres', + qq[SELECT pg_logical_slot_get_changes('test_slot', NULL, NULL);], + extra_params => [ '-U', 'normal' ]); +chomp($result); +is($result, '', 'Decoding after fast restart repeats no rows'); + +# Insert some rows and verify that we get the same results from pg_recvlogical +# and the SQL interface. +$node_master->safe_psql('postgres', + qq[INSERT INTO decoding_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;], + extra_params => [ '-U', 'normal' ] +); + +my $expected = q{BEGIN +table public.decoding_test: INSERT: x[integer]:1 y[text]:'1' +table public.decoding_test: INSERT: x[integer]:2 y[text]:'2' +table public.decoding_test: INSERT: x[integer]:3 y[text]:'3' +table public.decoding_test: INSERT: x[integer]:4 y[text]:'4' +COMMIT}; + +my $stdout_sql = $node_master->safe_psql('postgres', + qq[SELECT data FROM pg_logical_slot_peek_changes('test_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');], + extra_params => [ '-U', 'normal' ] +); +is($stdout_sql, $expected, 'got expected output from SQL decoding session'); + +my $endpos = $node_master->safe_psql('postgres', + "SELECT lsn FROM pg_logical_slot_peek_changes('test_slot', NULL, NULL) ORDER BY lsn DESC LIMIT 1;", + extra_params => [ '-U', 'normal' ] +); +print "waiting to replay $endpos\n"; + +my $stdout_recv = $node_master->pg_recvlogical_upto( + 'postgres', 'test_slot', $endpos, 180, + 'include-xids' => '0', + 'skip-empty-xacts' => '1'); +chomp($stdout_recv); +is($stdout_recv, $expected, + 'got same expected output from pg_recvlogical decoding session'); + +$node_master->poll_query_until('postgres', + "SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'test_slot' AND active_pid IS NULL)" +) or die "slot never became inactive"; + +$stdout_recv = $node_master->pg_recvlogical_upto( + 'postgres', 'test_slot', $endpos, 180, + 'include-xids' => '0', + 'skip-empty-xacts' => '1'); +chomp($stdout_recv); +is($stdout_recv, '', + 'pg_recvlogical acknowledged changes, nothing pending on slot'); + +$node_master->safe_psql('postgres', 'CREATE DATABASE otherdb'); + +is( $node_master->psql( + 'otherdb', + "SELECT lsn FROM pg_logical_slot_peek_changes('test_slot', NULL, NULL) ORDER BY lsn DESC LIMIT 1;", + extra_params => [ '-U', 'normal' ] + ), + 3, + 'replaying logical slot from another database fails'); + +$node_master->safe_psql('otherdb', + qq[SELECT pg_create_logical_replication_slot('otherdb_slot', 'test_decoding');], + extra_params => [ '-U', 'normal' ] +); + +# make sure you can't drop a slot while active +SKIP: +{ + + # some Windows Perls at least don't like IPC::Run's start/kill_kill regime. + skip "Test fails on Windows perl", 2 if $Config{osname} eq 'MSWin32'; + + my $pg_recvlogical = IPC::Run::start( + [ + 'pg_recvlogical', '-d', $node_master->connstr('otherdb'), + '-S', 'otherdb_slot', '-f', '-', '--start', '-U' , 'normal' + ]); + $node_master->poll_query_until('otherdb', + "SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'otherdb_slot' AND active_pid IS NOT NULL)" + ) or die "slot never became active"; + is($node_master->psql('postgres', 'DROP DATABASE otherdb'), + 3, 'dropping a DB with active logical slots fails'); + $pg_recvlogical->kill_kill; + is($node_master->slot('otherdb_slot')->{'slot_name'}, + undef, 'logical slot still exists'); +} + +$node_master->poll_query_until('otherdb', + "SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'otherdb_slot' AND active_pid IS NULL)", +) or die "slot never became inactive"; + +is($node_master->psql('postgres', 'DROP DATABASE otherdb'), + 0, 'dropping a DB with inactive logical slots succeeds'); +is($node_master->slot('otherdb_slot')->{'slot_name'}, + undef, 'logical slot was actually dropped with DB'); + +# done with the node +$node_master->stop; diff --git a/src/test/regress/expected/mdb_replication.out b/src/test/regress/expected/mdb_replication.out new file mode 100644 index 00000000000..2c8f7bac409 --- /dev/null +++ b/src/test/regress/expected/mdb_replication.out @@ -0,0 +1,35 @@ +CREATE ROLE regress_mdb_repl_no_priv LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl_no_priv2 LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl_su LOGIN SUPERUSER; +CREATE ROLE regress_mdb_repl_pgrad LOGIN NOSUPERUSER; +GRANT pg_read_all_data TO regress_mdb_repl_pgrad; +GRANT mdb_admin to regress_mdb_repl; +GRANT CREATE ON DATABASE REGRESSION TO regress_mdb_repl; +GRANT CREATE ON DATABASE REGRESSION TO regress_mdb_repl_no_priv; +-- ok - member of mdb_admin +SET SESSION AUTHORIZATION regress_mdb_repl; +CREATE SUBSCRIPTION regress_mdbsub CONNECTION 'dbname=doesnotexist password=regress_fakepassword' PUBLICATION foo WITH (slot_name = NONE, connect = false); +WARNING: subscription was created, but is not connected +HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. +-- ok - we are allowed to change ownership under mdb_admin +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_no_priv2; +-- should fail - we are not allowed to change ownership to priviledged role +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_su; +ERROR: must be owner of subscription regress_mdbsub +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_pgrad; +ERROR: must be owner of subscription regress_mdbsub +RESET SESSION AUTHORIZATION; +DROP SUBSCRIPTION regress_mdbsub; +-- should fail - not member of pg_subcription_users or mdb_admin +SET SESSION AUTHORIZATION regress_mdb_repl_no_priv; +CREATE SUBSCRIPTION regress_mdbsub_no_priv CONNECTION 'dbname=doesnotexist password=regress_fakepassword' PUBLICATION foo WITH (slot_name = NONE, connect = false); +ERROR: permission denied to create subscription +DETAIL: Only roles with privileges of the "pg_create_subscription" role may create subscriptions. +-- reset to su, cleanup +RESET SESSION AUTHORIZATION; +REVOKE ALL ON DATABASE REGRESSION FROM regress_mdb_repl; +REVOKE ALL ON DATABASE REGRESSION FROM regress_mdb_repl_no_priv; +DROP ROLE regress_mdb_repl; +DROP ROLE regress_mdb_repl_no_priv; +DROP ROLE regress_mdb_repl_no_priv2; diff --git a/src/test/regress/expected/test_setup.out b/src/test/regress/expected/test_setup.out index c9a4dd96a38..ffd15883d10 100644 --- a/src/test/regress/expected/test_setup.out +++ b/src/test/regress/expected/test_setup.out @@ -244,3 +244,6 @@ create function fipshash(text) strict immutable parallel safe leakproof return substr(encode(sha256($1::bytea), 'hex'), 1, 32); CREATE ROLE mdb_admin; +CREATE ROLE mdb_superuser; +CREATE ROLE mdb_replication; +GRANT pg_create_subscription TO mdb_admin; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 635518e1ec4..a9b4d09e3ca 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -15,6 +15,10 @@ test: test_setup test: mdb_admin +test: mdb_superuser + +test: mdb_replication + # ---------- # The first group of parallel tests # ---------- diff --git a/src/test/regress/sql/mdb_replication.sql b/src/test/regress/sql/mdb_replication.sql new file mode 100644 index 00000000000..31bcb460879 --- /dev/null +++ b/src/test/regress/sql/mdb_replication.sql @@ -0,0 +1,41 @@ +CREATE ROLE regress_mdb_repl_no_priv LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl_no_priv2 LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl_su LOGIN SUPERUSER; +CREATE ROLE regress_mdb_repl_pgrad LOGIN NOSUPERUSER; + +GRANT pg_read_all_data TO regress_mdb_repl_pgrad; + +GRANT mdb_admin to regress_mdb_repl; + +GRANT CREATE ON DATABASE REGRESSION TO regress_mdb_repl; + +GRANT CREATE ON DATABASE REGRESSION TO regress_mdb_repl_no_priv; + +-- ok - member of mdb_admin +SET SESSION AUTHORIZATION regress_mdb_repl; +CREATE SUBSCRIPTION regress_mdbsub CONNECTION 'dbname=doesnotexist password=regress_fakepassword' PUBLICATION foo WITH (slot_name = NONE, connect = false); + +-- ok - we are allowed to change ownership under mdb_admin +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_no_priv2; + +-- should fail - we are not allowed to change ownership to priviledged role +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_su; +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_pgrad; + +RESET SESSION AUTHORIZATION; +DROP SUBSCRIPTION regress_mdbsub; + +-- should fail - not member of pg_subcription_users or mdb_admin +SET SESSION AUTHORIZATION regress_mdb_repl_no_priv; +CREATE SUBSCRIPTION regress_mdbsub_no_priv CONNECTION 'dbname=doesnotexist password=regress_fakepassword' PUBLICATION foo WITH (slot_name = NONE, connect = false); +-- reset to su, cleanup + +RESET SESSION AUTHORIZATION; + +REVOKE ALL ON DATABASE REGRESSION FROM regress_mdb_repl; +REVOKE ALL ON DATABASE REGRESSION FROM regress_mdb_repl_no_priv; + +DROP ROLE regress_mdb_repl; +DROP ROLE regress_mdb_repl_no_priv; +DROP ROLE regress_mdb_repl_no_priv2; \ No newline at end of file diff --git a/src/test/regress/sql/test_setup.sql b/src/test/regress/sql/test_setup.sql index 18723170507..1e2e50f9009 100644 --- a/src/test/regress/sql/test_setup.sql +++ b/src/test/regress/sql/test_setup.sql @@ -301,3 +301,7 @@ create function fipshash(text) return substr(encode(sha256($1::bytea), 'hex'), 1, 32); CREATE ROLE mdb_admin; +CREATE ROLE mdb_superuser; +CREATE ROLE mdb_replication; + +GRANT pg_create_subscription TO mdb_admin; diff --git a/src/test/subscription/t/101_mdb_rep_changes_nonsuperuser.pl b/src/test/subscription/t/101_mdb_rep_changes_nonsuperuser.pl new file mode 100644 index 00000000000..0d3ae411ddf --- /dev/null +++ b/src/test/subscription/t/101_mdb_rep_changes_nonsuperuser.pl @@ -0,0 +1,336 @@ +# Basic logical replication test +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->start; + +# Prepare publisher +$node_publisher->safe_psql('postgres', + "CREATE ROLE normal_mdb_repl LOGIN PASSWORD 'regress_lolkekpassword'"); +$node_publisher->safe_psql('postgres', + "CREATE ROLE mdb_replication"); +$node_publisher->safe_psql('postgres', + "GRANT mdb_replication TO normal_mdb_repl"); +$node_publisher->safe_psql('postgres', + "GRANT CREATE ON DATABASE postgres TO normal_mdb_repl"); +$node_publisher->safe_psql('postgres', + "GRANT ALL ON SCHEMA public TO normal_mdb_repl;"); + + +# Delete pg_hba.conf from the given node, add a new entry to it +# and then execute a reload to refresh it. +sub reset_pg_hba +{ + my $node = shift; + my $hba_method = shift; + + unlink($node->data_dir . '/pg_hba.conf'); + $node->append_conf('pg_hba.conf', "local all normal_mdb_repl $hba_method"); + $node->append_conf('pg_hba.conf', "local all all trust"); + $node->reload; + return; +} + +# Prepare subscriber +$node_subscriber->safe_psql('postgres', + "CREATE ROLE normal_mdb_admin LOGIN"); +$node_subscriber->safe_psql('postgres', + "CREATE ROLE mdb_admin"); +$node_subscriber->safe_psql('postgres', + "GRANT pg_create_subscription TO mdb_admin"); +$node_subscriber->safe_psql('postgres', + "GRANT mdb_admin TO normal_mdb_admin"); +$node_subscriber->safe_psql('postgres', + "GRANT CREATE ON DATABASE postgres TO normal_mdb_admin"); +$node_subscriber->safe_psql('postgres', + "GRANT ALL ON SCHEMA public TO normal_mdb_admin;"); + +# Create some preexisting content on publisher +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_notrep AS SELECT generate_series(1,10) AS a", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_ins AS SELECT generate_series(1,1002) AS a", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_full AS SELECT generate_series(1,10) AS a", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', "CREATE TABLE tab_full2 (x text)", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_full2 VALUES ('a'), ('b'), ('b')", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_rep (a int primary key)", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_mixed (a int primary key, b text)", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_mixed (a, b) VALUES (1, 'foo')", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_include (a int, b text, CONSTRAINT covering PRIMARY KEY(a) INCLUDE(b))", + extra_params => [ '-U', 'normal_mdb_repl' ] +); + + +# Setup structure on subscriber +$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_notrep (a int)", extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_ins (a int)", extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_full (a int)", extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_full2 (x text)", extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab_rep (a int primary key)", + extra_params => [ '-U', 'normal_mdb_admin' ]); + +# different column count and order than on publisher +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab_mixed (c text, b text, a int primary key)", + extra_params => [ '-U', 'normal_mdb_admin' ]); + +# replication of the table with included index +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab_include (a int, b text, CONSTRAINT covering PRIMARY KEY(a) INCLUDE(b))", + extra_params => [ '-U', 'normal_mdb_admin' ] +); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres user=normal_mdb_repl'; +$node_publisher->safe_psql('postgres', "CREATE PUBLICATION tap_pub", extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub_ins_only WITH (publish = insert)", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "ALTER PUBLICATION tap_pub ADD TABLE tab_rep, tab_full, tab_full2, tab_mixed, tab_include", + extra_params => [ '-U', 'normal_mdb_repl' ] +); +$node_publisher->safe_psql('postgres', + "ALTER PUBLICATION tap_pub_ins_only ADD TABLE tab_ins", + extra_params => [ '-U', 'normal_mdb_repl' ]); + + +# require password for all connection on publisher +reset_pg_hba($node_publisher, 'scram-sha-256'); + +$ENV{"PGPASSWORD"} = 'regress_lolkekpassword'; + +my $appname = 'tap_sub'; +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr password=regress_lolkekpassword application_name=$appname' PUBLICATION tap_pub, tap_pub_ins_only", + extra_params => [ '-U', 'normal_mdb_admin' ] +); + +$node_publisher->wait_for_catchup($appname); + +# Also wait for initial table sync to finish +my $synced_query = + "SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +my $result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_notrep"); +is($result, qq(0), 'check non-replicated table is empty on subscriber'); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_ins"); +is($result, qq(1002), 'check initial data was copied to subscriber'); + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_ins SELECT generate_series(1,50)"); +$node_publisher->safe_psql('postgres', "DELETE FROM tab_ins WHERE a > 20"); +$node_publisher->safe_psql('postgres', "UPDATE tab_ins SET a = -a"); + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_rep SELECT generate_series(1,50)"); +$node_publisher->safe_psql('postgres', "DELETE FROM tab_rep WHERE a > 20"); +$node_publisher->safe_psql('postgres', "UPDATE tab_rep SET a = -a"); + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_mixed VALUES (2, 'bar')"); + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_include SELECT generate_series(1,50)"); +$node_publisher->safe_psql('postgres', + "DELETE FROM tab_include WHERE a > 20"); +$node_publisher->safe_psql('postgres', "UPDATE tab_include SET a = -a"); + +$node_publisher->wait_for_catchup($appname); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_ins"); +is($result, qq(1052|1|1002), 'check replicated inserts on subscriber'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_rep"); +is($result, qq(20|-20|-1), 'check replicated changes on subscriber'); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT c, b, a FROM tab_mixed"); +is( $result, qq(|foo|1 +|bar|2), 'check replicated changes with different column order'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_include"); +is($result, qq(20|-20|-1), + 'check replicated changes with primary key index with included columns'); + +# insert some duplicate rows +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_full SELECT generate_series(1,10)"); + +# add REPLICA IDENTITY FULL so we can update +$node_publisher->safe_psql('postgres', + "ALTER TABLE tab_full REPLICA IDENTITY FULL"); +$node_subscriber->safe_psql('postgres', + "ALTER TABLE tab_full REPLICA IDENTITY FULL"); +$node_publisher->safe_psql('postgres', + "ALTER TABLE tab_full2 REPLICA IDENTITY FULL"); +$node_subscriber->safe_psql('postgres', + "ALTER TABLE tab_full2 REPLICA IDENTITY FULL"); +$node_publisher->safe_psql('postgres', + "ALTER TABLE tab_ins REPLICA IDENTITY FULL"); +$node_subscriber->safe_psql('postgres', + "ALTER TABLE tab_ins REPLICA IDENTITY FULL"); + +# and do the updates +$node_publisher->safe_psql('postgres', "UPDATE tab_full SET a = a * a"); +$node_publisher->safe_psql('postgres', + "UPDATE tab_full2 SET x = 'bb' WHERE x = 'b'"); + +$node_publisher->wait_for_catchup($appname); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_full"); +is($result, qq(20|1|100), + 'update works with REPLICA IDENTITY FULL and duplicate tuples'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT x FROM tab_full2 ORDER BY 1"); +is( $result, qq(a +bb +bb), + 'update works with REPLICA IDENTITY FULL and text datums'); + +# check that change of connection string and/or publication list causes +# restart of subscription workers. Not all of these are registered as tests +# as we need to poll for a change but the test suite will fail none the less +# when something goes wrong. +my $oldpid = $node_publisher->safe_psql('postgres', + "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname';" +); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub CONNECTION 'application_name=$appname $publisher_connstr password=regress_lolkekpassword '", + extra_params => [ '-U', 'normal_mdb_admin' ] +); +$node_publisher->poll_query_until('postgres', + "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname';" +) or die "Timed out while waiting for apply to restart"; + +$oldpid = $node_publisher->safe_psql('postgres', + "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname';" +); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub SET PUBLICATION tap_pub_ins_only WITH (copy_data = false)", + extra_params => [ '-U', 'normal_mdb_admin' ] +); +$node_publisher->poll_query_until('postgres', + "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname';" +) or die "Timed out while waiting for apply to restart"; + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_ins SELECT generate_series(1001,1100)"); +$node_publisher->safe_psql('postgres', "DELETE FROM tab_rep"); + +# Restart the publisher and check the state of the subscriber which +# should be in a streaming state after catching up. +$node_publisher->stop('fast'); +$node_publisher->start; + +$node_publisher->wait_for_catchup($appname); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_ins"); +is($result, qq(1152|1|1100), + 'check replicated inserts after subscription publication change'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_rep"); +is($result, qq(20|-20|-1), + 'check changes skipped after subscription publication change'); + +# check alter publication (relcache invalidation etc) +$node_publisher->safe_psql('postgres', + "ALTER PUBLICATION tap_pub_ins_only SET (publish = 'insert, delete')"); +$node_publisher->safe_psql('postgres', + "ALTER PUBLICATION tap_pub_ins_only ADD TABLE tab_full"); +$node_publisher->safe_psql('postgres', "DELETE FROM tab_ins WHERE a > 0"); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION WITH (copy_data = false)", + extra_params => [ '-U', 'normal_mdb_admin' ] +); +$node_publisher->safe_psql('postgres', "INSERT INTO tab_full VALUES(0)"); + +$node_publisher->wait_for_catchup($appname); + +# note that data are different on provider and subscriber +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_ins"); +is($result, qq(1052|1|1002), + 'check replicated deletes after alter publication'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_full"); +is($result, qq(21|0|100), 'check replicated insert after alter publication'); + +# check restart on rename +$oldpid = $node_publisher->safe_psql('postgres', + "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname';" +); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub RENAME TO tap_sub_renamed", + extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_publisher->poll_query_until('postgres', + "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname';" +) or die "Timed out while waiting for apply to restart"; + +# check all the cleanup +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub_renamed", extra_params => [ '-U', 'normal_mdb_admin' ]); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_subscription"); +is($result, qq(0), 'check subscription was dropped on subscriber'); + +$result = $node_publisher->safe_psql('postgres', + "SELECT count(*) FROM pg_replication_slots"); +is($result, qq(0), 'check replication slot was dropped on publisher'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_subscription_rel"); +is($result, qq(0), + 'check subscription relation status was dropped on subscriber'); + +$result = $node_publisher->safe_psql('postgres', + "SELECT count(*) FROM pg_replication_slots"); +is($result, qq(0), 'check replication slot was dropped on publisher'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_replication_origin"); +is($result, qq(0), 'check replication origin was dropped on subscriber'); + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); + +done_testing(); diff --git a/src/test/subscription/t/102_mdb_check_permissions.pl b/src/test/subscription/t/102_mdb_check_permissions.pl new file mode 100644 index 00000000000..1efd1cccf6e --- /dev/null +++ b/src/test/subscription/t/102_mdb_check_permissions.pl @@ -0,0 +1,202 @@ + +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +# Test that logical replication respects permissions +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use Test::More; + +my ($node_publisher, $node_subscriber, $publisher_connstr, $result, $offset); +$offset = 0; + +sub publish_insert +{ + my ($tbl, $new_i) = @_; + $node_publisher->safe_psql( + 'postgres', qq( + SET SESSION AUTHORIZATION regress_user1; + INSERT INTO $tbl (i) VALUES ($new_i); + )); +} + +sub publish_update +{ + my ($tbl, $old_i, $new_i) = @_; + $node_publisher->safe_psql( + 'postgres', qq( + SET SESSION AUTHORIZATION regress_user1; + UPDATE $tbl SET i = $new_i WHERE i = $old_i; + )); +} + +sub publish_delete +{ + my ($tbl, $old_i) = @_; + $node_publisher->safe_psql( + 'postgres', qq( + SET SESSION AUTHORIZATION regress_user1; + DELETE FROM $tbl WHERE i = $old_i; + )); +} + + +sub publish_truncate +{ + my ($tbl) = @_; + $node_publisher->safe_psql( + 'postgres', qq( + SET SESSION AUTHORIZATION regress_user1; + TRUNCATE $tbl; + )); +} + +sub expect_replication +{ + my ($tbl, $cnt, $min, $max, $testname) = @_; + $node_publisher->wait_for_catchup('app_test_mdb_admin_sub'); + $result = $node_subscriber->safe_psql( + 'postgres', qq( + SELECT COUNT(i), MIN(i), MAX(i) FROM $tbl)); + is($result, "$cnt|$min|$max", $testname); +} + +sub expect_failure +{ + my ($tbl, $cnt, $min, $max, $re, $testname) = @_; + $offset = $node_subscriber->wait_for_log($re, $offset); + $result = $node_subscriber->safe_psql( + 'postgres', qq( + SELECT COUNT(i), MIN(i), MAX(i) FROM $tbl)); + is($result, "$cnt|$min|$max", $testname); +} + +# Create publisher and subscriber nodes with schemas owned and published by +# "regress_alice" but subscribed and replicated by different role +# "regress_admin". For partitioned tables, layout the partitions differently +# on the publisher than on the subscriber. +# +$node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_publisher->init(allows_streaming => 'logical'); +$node_subscriber->init; +$node_publisher->start; +$node_subscriber->start; +$publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; + +for my $node ($node_publisher, $node_subscriber) +{ + $node->safe_psql( + 'postgres', qq( + CREATE ROLE mdb_admin; + GRANT pg_create_subscription TO mdb_admin; + CREATE ROLE mdb_replication; + + CREATE ROLE regress_user1 NOSUPERUSER LOGIN PASSWORD 'regress_lolkekpassword'; + CREATE ROLE regress_user2 NOSUPERUSER LOGIN PASSWORD 'regress_lolkekpassword'; + + GRANT CREATE ON DATABASE postgres TO regress_user1; + GRANT CREATE ON DATABASE postgres TO regress_user2; + SET SESSION AUTHORIZATION regress_user1; + + CREATE SCHEMA sh; + CREATE TABLE sh.tt (i INTEGER); + + ALTER TABLE sh.tt REPLICA IDENTITY FULL; + + GRANT USAGE ON SCHEMA sh TO regress_user2; + GRANT INSERT, SELECT, UPDATE, DELETE, TRUNCATE ON TABLE sh.tt TO regress_user2; + )); +} + +$node_publisher->safe_psql( + 'postgres', qq( +GRANT mdb_replication TO regress_user2; +SET SESSION AUTHORIZATION regress_user1; + +CREATE PUBLICATION tap_pub + FOR TABLE sh.tt; +)); + + +# Delete pg_hba.conf from the given node, add a new entry to it +# and then execute a reload to refresh it. +sub reset_pg_hba +{ + my $node = shift; + my $hba_method = shift; + + unlink($node->data_dir . '/pg_hba.conf'); + $node->append_conf('pg_hba.conf', "local all regress_user2 $hba_method"); + $node->append_conf('pg_hba.conf', "local all all trust"); + $node->reload; + return; +} + +# require password for all connection on publisher +reset_pg_hba($node_publisher, 'scram-sha-256'); + +$ENV{"PGPASSWORD"} = 'regress_lolkekpassword'; + +$node_subscriber->safe_psql( + 'postgres', qq( +GRANT mdb_admin TO regress_user2; +SET SESSION AUTHORIZATION regress_user2; + +CREATE SUBSCRIPTION test_mdb_admin_sub CONNECTION '$publisher_connstr application_name=app_test_mdb_admin_sub user=regress_user2 password=regress_lolkekpassword' PUBLICATION tap_pub WITH(run_as_owner=true); +)); + +# Wait for initial sync to finish +$node_subscriber->wait_for_subscription_sync($node_publisher, 'app_test_mdb_admin_sub'); + +# Verify that "regress_admin" can replicate into the tables +# +publish_insert("sh.tt", 2); +publish_insert("sh.tt", 8); +publish_insert("sh.tt", 16); +expect_replication("sh.tt", 3, 2, 16, + "mdb_admin/mdb_replication role replicates into sh.tt"); + +# check that no replication possible if permission revoked +$node_subscriber->safe_psql( + 'postgres', qq( + REVOKE INSERT ON TABLE sh.tt FROM regress_user2 +)); + + + +publish_insert("sh.tt", 47); + +expect_failure( + "sh.tt", + 3, + 2, + 16, + qr/ERROR: permission denied for table tt/, + "role without INSERT grant for table fails to replicate insert"); + + +# check that no replication possible if permission revoked +$node_subscriber->safe_psql( + 'postgres', qq( + GRANT INSERT ON TABLE sh.tt TO regress_user2 +)); + + +expect_replication("sh.tt", 4, 2, 47, + "mdb_admin/mdb_replication role replicates after grants again"); + +publish_update("sh.tt", 2 => 7); +publish_delete("sh.tt", 16); + +expect_replication("sh.tt", 3, 7, 47, + "mdb_admin/mdb_replication role replicates update/delete into sh.tt"); + +publish_truncate("sh.tt"); + +publish_insert("sh.tt", 47); + +expect_replication("sh.tt", 1, 47, 47, + "mdb_admin/mdb_replication role replicates truncate sh.tt"); + +done_testing(); From afe958ab74725728962600a0af7537f7142a5e06 Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Tue, 21 Feb 2023 19:34:14 +0000 Subject: [PATCH 06/54] Role mdb_superuser: feature and regress testsing This commit introduces new mdb internal role mdb_superuser. Role is capaple of: GRANT/REVOKE any set of priviledges to/from any object in database. Has power of pg_database_owner in any database, including: DROP any object in database (except system catalog and stuff) Role is NOT capaple of: Create database, role, extension or alter other roles with such priviledges. Transfer ownership to /pass has_priv of roles: PG_READ_ALL_DATA PG_WRITE_ALL_DATA PG_EXECUTE_SERVER_PROGRAM PG_READ_SERVER_FILES PG_WRITE_SERVER_FILES Fix configure.ac USE_MDBLOCALES option handling Apply autoreconf stuff Set missing ok parameter ito true while acquiring mdb_superuser oid In regress tests, nobody creates mdb_superuser role, so missing ok is fine Fix spelling Applied suggestion Allow mdb_superuser to have power of pg_database_owner Allow mdb_superuser to alter objects and grant ACl to objects, owned by pg_database_owner. Also, when acl check, allow mdb_superuser use pg_database_owner role power to pass check regression test fixes --- src/backend/commands/functioncmds.c | 4 +- src/backend/utils/adt/acl.c | 34 ++++- src/include/pg_config.h.in | 6 + src/include/utils/acl.h | 1 + src/test/regress/expected/mdb_superuser.out | 108 +++++++++++++++ src/test/regress/sql/mdb_superuser.sql | 138 ++++++++++++++++++++ 6 files changed, 288 insertions(+), 3 deletions(-) create mode 100644 src/test/regress/expected/mdb_superuser.out create mode 100644 src/test/regress/sql/mdb_superuser.sql diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index 4d2d71d6a05..65c9e135a54 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -1136,7 +1136,7 @@ CreateFunction(ParseState *pstate, CreateFunctionStmt *stmt) */ if (isLeakProof && !superuser()) { - Oid role = get_role_oid("mdb_admin", true); + Oid role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); if (!is_member_of_role(GetUserId(), role)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -1425,7 +1425,7 @@ AlterFunction(ParseState *pstate, AlterFunctionStmt *stmt) procForm->proleakproof = boolVal(leakproof_item->arg); if (procForm->proleakproof && !superuser()) { - Oid role = get_role_oid("mdb_admin", true); + Oid role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); if (!is_member_of_role(GetUserId(), role)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 0cf2676fc32..7795e1e7ec3 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5003,7 +5003,7 @@ has_privs_of_role_strict(Oid member, Oid role) /* * Check that role is either one of "dangerous" system role -* or has "strict" (not through mdb_admin) +* or has "strict" (not through mdb_admin or mdb_superuser) * privs of this role */ @@ -5031,6 +5031,8 @@ has_privs_of_unwanted_system_role(Oid role) { bool has_privs_of_role(Oid member, Oid role) { + Oid mdb_superuser_roleoid; + /* Fast path for simple case */ if (member == role) return true; @@ -5039,6 +5041,23 @@ has_privs_of_role(Oid member, Oid role) if (superuser_arg(member)) return true; + mdb_superuser_roleoid = get_role_oid("mdb_superuser", true /*if nodoby created mdb_superuser role in this database*/); + + if (is_member_of_role(member, mdb_superuser_roleoid)) { + /* if target role is superuser, disallow */ + if (!superuser_arg(role)) { + /* we want mdb_roles_admin to bypass + * has_priv_of_roles test + * if target role is neither superuser nor + * some dangerous system role + */ + if (!has_privs_of_unwanted_system_role(role)) { + return true; + } + } + } + + /* * Find all the roles that member has the privileges of, including * multi-level recursion, then see if target role is any one of them. @@ -5048,6 +5067,8 @@ has_privs_of_role(Oid member, Oid role) role); } +// -- mdb_superuser patch + // -- non-upstream patch begin /* * Is userId allowed to bypass ownership check @@ -5355,6 +5376,7 @@ select_best_grantor(Oid roleId, AclMode privileges, List *roles_list; int nrights; ListCell *l; + Oid mdb_superuser_roleoid; /* * The object owner is always treated as having all grant options, so if @@ -5369,6 +5391,16 @@ select_best_grantor(Oid roleId, AclMode privileges, return; } + mdb_superuser_roleoid = get_role_oid("mdb_superuser", true /*if nodoby created mdb_superuser role in this database*/); + + if (is_member_of_role(GetUserId(), mdb_superuser_roleoid) + && has_privs_of_role(GetUserId(), ownerId)) { + *grantorId = mdb_superuser_roleoid; + AclMode mdb_superuser_allowed_privs = needed_goptions; + *grantOptions = mdb_superuser_allowed_privs; + return; + } + /* * Otherwise we have to do a careful search to see if roleId has the * privileges of any suitable role. Note: we can hang onto the result of diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index d06c49a8942..015c034c7cb 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -276,6 +276,9 @@ /* Define to 1 if you have the `m' library (-lm). */ #undef HAVE_LIBM +/* Define to 1 if you have the `mdblocales' library (-lmdblocales). */ +#undef HAVE_LIBMDBLOCALES + /* Define to 1 if you have the `pam' library (-lpam). */ #undef HAVE_LIBPAM @@ -728,6 +731,9 @@ /* Define to 1 to build with LZ4 support. (--with-lz4) */ #undef USE_LZ4 +/* Define to 1 to build with MDB locales. (--with-mdblocales) */ +#undef USE_MDBLOCALES + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 312a32557d3..c472cea1487 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -211,6 +211,7 @@ extern int aclmembers(const Acl *acl, Oid **roleids); extern bool has_privs_of_role(Oid member, Oid role); extern bool member_can_set_role(Oid member, Oid role); extern void check_can_set_role(Oid member, Oid role); +extern bool has_privs_of_role_strict(Oid member, Oid role); extern bool is_member_of_role(Oid member, Oid role); extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); diff --git a/src/test/regress/expected/mdb_superuser.out b/src/test/regress/expected/mdb_superuser.out new file mode 100644 index 00000000000..e8dbe2fe1cc --- /dev/null +++ b/src/test/regress/expected/mdb_superuser.out @@ -0,0 +1,108 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; +GRANT mdb_admin TO mdb_superuser; +CREATE ROLE regress_superuser WITH SUPERUSER; +GRANT mdb_superuser TO regress_mdb_superuser_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user3; +SET ROLE regress_mdb_superuser_user2; +CREATE FUNCTION regress_mdb_superuser_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; +CREATE SCHEMA regress_mdb_superuser_schema; +CREATE TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table(); +CREATE TABLE regress_mdb_superuser_table(); +CREATE VIEW regress_mdb_superuser_view as SELECT 1; +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; +ERROR: permission denied for table regress_mdb_superuser_table +SET ROLE regress_mdb_superuser_user1; +-- mdb_superuser can grant to other role +GRANT USAGE, CREATE ON SCHEMA regress_mdb_superuser_schema TO regress_mdb_superuser_user3; +GRANT ALL PRIVILEGES ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; +REVOKE ALL PRIVILEGES ON TABLE regress_mdb_superuser_table FROM regress_mdb_superuser_user3; +GRANT INSERT, SELECT ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; +-- grant works +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; +SET ROLE mdb_superuser; +-- mdb_superuser drop object of other role +DROP TABLE regress_mdb_superuser_table; +-- mdb admin fails to transfer ownership to superusers and system roles +RESET SESSION AUTHORIZATION; +CREATE TABLE regress_superuser_table(); +SET ROLE pg_read_server_files; +CREATE TABLE regress_pgrsf_table(); +SET ROLE pg_write_server_files; +CREATE TABLE regress_pgwsf_table(); +SET ROLE pg_execute_server_program; +CREATE TABLE regress_pgxsp_table(); +SET ROLE pg_read_all_data; +CREATE TABLE regress_pgrad_table(); +SET ROLE pg_write_all_data; +CREATE TABLE regress_pgrwd_table(); +SET ROLE mdb_superuser; +-- cannot read all data (fail) +SELECT * FROM pg_authid; +ERROR: permission denied for table pg_authid +-- can drop superuser objects, because has power of pg_database_owner +DROP TABLE regress_superuser_table; +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; +-- does allowed to creare database, role or extension +-- or grant such priviledge +CREATE DATABASE regress_db_fail; +ERROR: permission denied to create database +CREATE ROLE regress_role_fail; +ERROR: permission denied to create role +DETAIL: Only roles with the CREATEROLE attribute may create roles. +ALTER ROLE mdb_superuser WITH CREATEROLE; +ERROR: permission denied to alter role +DETAIL: Only roles with the CREATEROLE attribute and the ADMIN option on role "mdb_superuser" may alter this role. +ALTER ROLE mdb_superuser WITH CREATEDB; +ERROR: permission denied to alter role +DETAIL: Only roles with the CREATEROLE attribute and the ADMIN option on role "mdb_superuser" may alter this role. +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEROLE; +ERROR: permission denied to alter role +DETAIL: Only roles with the CREATEROLE attribute and the ADMIN option on role "regress_mdb_superuser_user2" may alter this role. +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEDB; +ERROR: permission denied to alter role +DETAIL: Only roles with the CREATEROLE attribute and the ADMIN option on role "regress_mdb_superuser_user2" may alter this role. +-- mdb_superuser more powerfull than pg_database_owner +RESET SESSION AUTHORIZATION; +CREATE DATABASE regress_check_owner OWNER regress_mdb_superuser_user2; +\c regress_check_owner; +SET ROLE regress_mdb_superuser_user2; +CREATE SCHEMA regtest; +CREATE TABLE regtest.regtest(); +-- this should fail +SET ROLE regress_mdb_superuser_user3; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user3; +ERROR: permission denied for schema regtest +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user3; +ERROR: permission denied for schema regtest +SET ROLE regress_mdb_superuser_user1; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; +\c regression +DROP DATABASE regress_check_owner; +-- end tests +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; +DROP VIEW regress_mdb_superuser_view; +DROP FUNCTION regress_mdb_superuser_add; +DROP TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table; +DROP TABLE regress_mdb_superuser_table; +ERROR: table "regress_mdb_superuser_table" does not exist +DROP SCHEMA regress_mdb_superuser_schema; +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; diff --git a/src/test/regress/sql/mdb_superuser.sql b/src/test/regress/sql/mdb_superuser.sql new file mode 100644 index 00000000000..77dc489f715 --- /dev/null +++ b/src/test/regress/sql/mdb_superuser.sql @@ -0,0 +1,138 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; + +GRANT mdb_admin TO mdb_superuser; + +CREATE ROLE regress_superuser WITH SUPERUSER; + +GRANT mdb_superuser TO regress_mdb_superuser_user1; + +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user3; + + +SET ROLE regress_mdb_superuser_user2; + +CREATE FUNCTION regress_mdb_superuser_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; + +CREATE SCHEMA regress_mdb_superuser_schema; +CREATE TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table(); +CREATE TABLE regress_mdb_superuser_table(); +CREATE VIEW regress_mdb_superuser_view as SELECT 1; + +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; + +SET ROLE regress_mdb_superuser_user1; + +-- mdb_superuser can grant to other role +GRANT USAGE, CREATE ON SCHEMA regress_mdb_superuser_schema TO regress_mdb_superuser_user3; +GRANT ALL PRIVILEGES ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; +REVOKE ALL PRIVILEGES ON TABLE regress_mdb_superuser_table FROM regress_mdb_superuser_user3; + +GRANT INSERT, SELECT ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; + +-- grant works +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; + +SET ROLE mdb_superuser; + +-- mdb_superuser drop object of other role +DROP TABLE regress_mdb_superuser_table; +-- mdb admin fails to transfer ownership to superusers and system roles + +RESET SESSION AUTHORIZATION; + +CREATE TABLE regress_superuser_table(); + +SET ROLE pg_read_server_files; + +CREATE TABLE regress_pgrsf_table(); + +SET ROLE pg_write_server_files; + +CREATE TABLE regress_pgwsf_table(); + +SET ROLE pg_execute_server_program; + +CREATE TABLE regress_pgxsp_table(); + +SET ROLE pg_read_all_data; + +CREATE TABLE regress_pgrad_table(); + +SET ROLE pg_write_all_data; + +CREATE TABLE regress_pgrwd_table(); + +SET ROLE mdb_superuser; + +-- cannot read all data (fail) +SELECT * FROM pg_authid; + +-- can drop superuser objects, because has power of pg_database_owner +DROP TABLE regress_superuser_table; +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; + + +-- does allowed to creare database, role or extension +-- or grant such priviledge + +CREATE DATABASE regress_db_fail; +CREATE ROLE regress_role_fail; + +ALTER ROLE mdb_superuser WITH CREATEROLE; +ALTER ROLE mdb_superuser WITH CREATEDB; + +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEROLE; +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEDB; + +-- mdb_superuser more powerfull than pg_database_owner + +RESET SESSION AUTHORIZATION; +CREATE DATABASE regress_check_owner OWNER regress_mdb_superuser_user2; + +\c regress_check_owner; + +SET ROLE regress_mdb_superuser_user2; +CREATE SCHEMA regtest; +CREATE TABLE regtest.regtest(); + +-- this should fail + +SET ROLE regress_mdb_superuser_user3; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user3; +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user3; + +SET ROLE regress_mdb_superuser_user1; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; + +\c regression +DROP DATABASE regress_check_owner; + +-- end tests + +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; + +DROP VIEW regress_mdb_superuser_view; +DROP FUNCTION regress_mdb_superuser_add; +DROP TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table; +DROP TABLE regress_mdb_superuser_table; +DROP SCHEMA regress_mdb_superuser_schema; +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; From 95221469f474a61e65d6f9ce7ea85349011c9422 Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 22 Feb 2022 23:26:51 +0300 Subject: [PATCH 07/54] provide [mdb -postgresql] restict grant roles in YC[MDB-16990] --- src/backend/access/common/Makefile | 3 +- src/backend/access/common/yc_checker.c | 21 +++++++++++++ src/backend/commands/user.c | 20 ++++++++++++ src/backend/utils/misc/guc.c | 1 + src/backend/utils/misc/guc_tables.c | 22 +++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 2 ++ src/include/access/yc_checker.h | 31 +++++++++++++++++++ src/test/modules/test_misc/t/003_check_guc.pl | 2 ++ 8 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 src/backend/access/common/yc_checker.c create mode 100644 src/include/access/yc_checker.h diff --git a/src/backend/access/common/Makefile b/src/backend/access/common/Makefile index b9aff0ccfdc..cb9315262bc 100644 --- a/src/backend/access/common/Makefile +++ b/src/backend/access/common/Makefile @@ -28,6 +28,7 @@ OBJS = \ toast_compression.o \ toast_internals.o \ tupconvert.o \ - tupdesc.o + tupdesc.o \ + yc_checker.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/common/yc_checker.c b/src/backend/access/common/yc_checker.c new file mode 100644 index 00000000000..a7e7822cc2d --- /dev/null +++ b/src/backend/access/common/yc_checker.c @@ -0,0 +1,21 @@ +/*------------------------------------------------------------------------- + * + * yc_checker.c + * yc routines + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/access/common/yc_checker.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "access/yc_checker.h" + +/* GUC variables */ + +YCGrantCheckerType yc_grant_checker_type = YC_GRANT_CHECKER_OFF; diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c index 39f909662f5..c47db9cf56c 100644 --- a/src/backend/commands/user.c +++ b/src/backend/commands/user.c @@ -16,6 +16,7 @@ #include "access/htup_details.h" #include "access/table.h" #include "access/xact.h" +#include "access/yc_checker.h" #include "catalog/binary_upgrade.h" #include "catalog/catalog.h" #include "catalog/dependency.h" @@ -1690,6 +1691,25 @@ AddRoleMems(Oid currentUserId, const char *rolename, Oid roleid, Assert(list_length(memberSpecs) == list_length(memberIds)); + if (!superuser()) { + switch (yc_grant_checker_type) { + case YC_GRANT_CHECKER_WARN: + ereport(WARNING, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("Roles granted using SQL might be eventually revoked. Please, use Yandex Cloud console, cli or terraform to grant role in postgresql cluster."))); + break; + case YC_GRANT_CHECKER_CRIT: + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("Roles granted using SQL might be eventually revoked. Please, use Yandex Cloud console, cli or terraform to grant role in postgresql cluster."))); + break; + case YC_GRANT_CHECKER_OFF: + default: + break; + + } + } + /* Validate grantor (and resolve implicit grantor if not specified). */ grantorId = check_role_grantor(currentUserId, roleid, grantorId, true); diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index d955397e94c..1ea44bca7f8 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -3335,6 +3335,7 @@ set_config_option_ext(const char *name, const char *value, void *newextra = NULL; bool prohibitValueChange = false; bool makeDefault; + Oid role; if (elevel == 0) { diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index bcb224b6a8d..721188a88ba 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -85,6 +85,10 @@ #include "utils/inval.h" #include "utils/xml.h" +/* MDB patch */ +#include "access/yc_checker.h" +/**/ + /* This value is normally passed in from the Makefile */ #ifndef PG_KRB_SRVTAB #define PG_KRB_SRVTAB "" @@ -479,6 +483,14 @@ static const struct config_enum_entry file_extend_method_options[] = { {NULL, 0, false} }; +static const struct config_enum_entry yc_grant_checker_options[] = { + {"off", YC_GRANT_CHECKER_OFF, false}, + {"warn", YC_GRANT_CHECKER_WARN, false}, + {"crit", YC_GRANT_CHECKER_CRIT, false}, + {NULL, 0, false} +}; + + /* * Options for enum values stored in other modules */ @@ -4652,6 +4664,16 @@ struct config_enum ConfigureNamesEnum[] = NULL, NULL, NULL }, + { + {"ycmdb.yc_grant_checker", PGC_SUSET, CLIENT_CONN_STATEMENT, + gettext_noop("Enables YC MDB runtime checker, which check if user is ok to grant roles to other users."), + NULL + }, + ((int *) &yc_grant_checker_type), + YC_GRANT_CHECKER_OFF, + yc_grant_checker_options, + NULL, NULL, NULL + }, { {"default_transaction_isolation", PGC_USERSET, CLIENT_CONN_STATEMENT, gettext_noop("Sets the transaction isolation level of each new transaction."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index abfa4f769e5..bcb63b681b9 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -825,6 +825,8 @@ #synchronous_commit_cancelation = off +#ycmdb.yc_grant_checker = off + #------------------------------------------------------------------------------ # CUSTOMIZED OPTIONS #------------------------------------------------------------------------------ diff --git a/src/include/access/yc_checker.h b/src/include/access/yc_checker.h new file mode 100644 index 00000000000..9c9a6b6ad4c --- /dev/null +++ b/src/include/access/yc_checker.h @@ -0,0 +1,31 @@ +# +/*------------------------------------------------------------------------- + * + * yc_checker.h + * + * Header file for YC MDB specific only GUC variables, + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/yc_checker.h + * + *------------------------------------------------------------------------- + */ +#ifndef PG_YC_CHECKER_H +#define PG_YC_CHECKER_H + +/* Possible values for yc_grant_checker_type */ +typedef enum +{ + YC_GRANT_CHECKER_OFF, + YC_GRANT_CHECKER_WARN, + YC_GRANT_CHECKER_CRIT, +} YCGrantCheckerType; + +/* GUC variables */ + +extern PGDLLIMPORT YCGrantCheckerType yc_grant_checker_type; + + +#endif /* PG_YC_CHECKER_H */ diff --git a/src/test/modules/test_misc/t/003_check_guc.pl b/src/test/modules/test_misc/t/003_check_guc.pl index a5d680e82e7..5feadc558ed 100644 --- a/src/test/modules/test_misc/t/003_check_guc.pl +++ b/src/test/modules/test_misc/t/003_check_guc.pl @@ -69,6 +69,8 @@ } } +push @gucs_in_file, "ycmdb.yc_grant_checker"; + close $contents; # Cross-check that all the GUCs found in the sample file match the ones From 5b52b5a9f9f2e8713eb37f67663ece0f9f58450c Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 11 May 2022 23:11:42 +0300 Subject: [PATCH 08/54] MDB-16955 : disallow to kill repl mon in cloud --- src/backend/postmaster/bgworker.c | 28 +++++++++++++++++++++++++++ src/backend/storage/ipc/signalfuncs.c | 11 +++++++++++ src/include/postmaster/bgworker.h | 1 + 3 files changed, 40 insertions(+) diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 0dd22b23511..79f4856db27 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -1309,3 +1309,31 @@ GetBackgroundWorkerTypeByPid(pid_t pid) return result; } + + +bool +GetBackgroundWorkerFindByPidCmp(pid_t pid, const char *target) +{ + int slotno; + bool result = false; + + LWLockAcquire(BackgroundWorkerLock, LW_SHARED); + + for (slotno = 0; slotno < BackgroundWorkerData->total_slots; slotno++) + { + BackgroundWorkerSlot *slot = &BackgroundWorkerData->slot[slotno]; + + if (slot->pid > 0 && slot->pid == pid) + { + if (slot->worker.bgw_type) { + result = strcmp(target, slot->worker.bgw_type) == 0; + } else { + result = false; + } + break; + } + } + + LWLockRelease(BackgroundWorkerLock); + return result; +} diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index f41d6583489..e8b40f17cf4 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -113,6 +113,17 @@ pg_signal_backend(int pid, int sig) } } + if (!superuser_arg(GetUserId()) && local_beentry != NULL) { + PgBackendStatus *beentry = &local_beentry->backendStatus; + // MDB-16955 : disallow to kill repl mon in cloud + if (beentry != NULL && beentry->st_backendType == B_BG_WORKER) + { + if (GetBackgroundWorkerFindByPidCmp(beentry->st_procpid, "repl_mon")) { + return SIGNAL_BACKEND_NOPERMISSION; + } + } + } + /* Users can signal backends they have role membership in. */ if (!has_privs_of_role(GetUserId(), proc->roleId) && !has_privs_of_role(GetUserId(), ROLE_PG_SIGNAL_BACKEND)) diff --git a/src/include/postmaster/bgworker.h b/src/include/postmaster/bgworker.h index 845d4498e65..6ab5091df2d 100644 --- a/src/include/postmaster/bgworker.h +++ b/src/include/postmaster/bgworker.h @@ -125,6 +125,7 @@ extern BgwHandleStatus WaitForBackgroundWorkerStartup(BackgroundWorkerHandle *ha extern BgwHandleStatus WaitForBackgroundWorkerShutdown(BackgroundWorkerHandle *); extern const char *GetBackgroundWorkerTypeByPid(pid_t pid); +extern bool GetBackgroundWorkerFindByPidCmp(pid_t pid, const char *target); /* Terminate a bgworker */ extern void TerminateBackgroundWorker(BackgroundWorkerHandle *handle); From 42cfeb32c0884668215dab152221f7573b7db8f9 Mon Sep 17 00:00:00 2001 From: "Andrey M. Borodin" Date: Mon, 12 Sep 2022 11:47:30 +0500 Subject: [PATCH 09/54] Demonstrate and fix lock of all SQL queries by pg_stat_statements --- .../pg_stat_statements/pg_stat_statements.c | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index e488334a967..92282d2b586 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -1266,8 +1266,12 @@ pgss_store(const char *query, uint64 queryId, key.queryid = queryId; key.toplevel = (exec_nested_level == 0); - /* Lookup the hash table entry with shared lock. */ - LWLockAcquire(pgss->lock, LW_SHARED); + /* + * Lookup the hash table entry with shared lock. + * If exclusive lock is taken - just give up. + */ + if (!LWLockConditionalAcquire(pgss->lock, LW_SHARED)) + return; entry = (pgssEntry *) hash_search(pgss_hash, &key, HASH_FIND, NULL); @@ -1292,7 +1296,13 @@ pgss_store(const char *query, uint64 queryId, norm_query = generate_normalized_query(jstate, query, query_location, &query_len); - LWLockAcquire(pgss->lock, LW_SHARED); + /* exclusive lock may be taken while we were doing this */ + /* XXX: Andrey: I'm not sure we should drop here shared lock at all */ + if (!LWLockConditionalAcquire(pgss->lock, LW_SHARED)) + { + pfree(norm_query); + return; + } } /* Append new query text to file with only shared lock held */ @@ -1308,7 +1318,9 @@ pgss_store(const char *query, uint64 queryId, /* Need exclusive lock to make a new hashtable entry - promote */ LWLockRelease(pgss->lock); - LWLockAcquire(pgss->lock, LW_EXCLUSIVE); + /* This renders impossible to enter another concurrent query */ + if (!LWLockConditionalAcquire(pgss->lock, LW_EXCLUSIVE)) + return; /* * A garbage collection may have occurred while we weren't holding the From 43243329911feeb1a048276e0b5db263ed3b7beb Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Wed, 18 Jan 2023 18:27:40 +0000 Subject: [PATCH 10/54] MDB-21297: forbit usage of COPY TO PROGRAMM and COPY FROM PROGRAMM to non-superuser It is well known that some of PostgreSQL-related security issues (CVE) was related to COPY FROM/TO PROGRAM exploits for priviledge escalation or other unwanted behaviour or consequences. Thus, proper usage of this feature needed. For now, simply forbit this. Add mdb copy test --- src/backend/commands/copy.c | 29 ++++++++++++------------ src/test/regress/expected/mdb_copy.out | 29 ++++++++++++++++++++++++ src/test/regress/parallel_schedule | 2 ++ src/test/regress/sql/mdb_copy.sql | 31 ++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 src/test/regress/expected/mdb_copy.out create mode 100644 src/test/regress/sql/mdb_copy.sql diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index a469825bdce..db385a69603 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -80,37 +80,38 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt, { if (stmt->is_program) { - if (!has_privs_of_role(GetUserId(), ROLE_PG_EXECUTE_SERVER_PROGRAM)) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to COPY to or from an external program"), - errdetail("Only roles with privileges of the \"%s\" role may COPY to or from an external program.", - "pg_execute_server_program"), - errhint("Anyone can COPY to stdout or from stdin. " - "psql's \\copy command also works for anyone."))); + // -- non-upstream patch begin + /* + * MDB-21297: forbit usage of COPY TO PROGRAM and COPY FROM PROGRAM to non-su + */ + + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("forbidden to COPY to or from an external program or file in Yandex Cloud"), + errhint("Anyone can COPY to stdout or from stdin. " + "psql's \\copy command also works for anyone."))); + + // --- non-upstream patch end } else { if (is_from && !has_privs_of_role(GetUserId(), ROLE_PG_READ_SERVER_FILES)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to COPY from a file"), - errdetail("Only roles with privileges of the \"%s\" role may COPY from a file.", - "pg_read_server_files"), + errmsg("must be superuser or have privileges of the pg_read_server_files role to COPY from a file"), errhint("Anyone can COPY to stdout or from stdin. " "psql's \\copy command also works for anyone."))); if (!is_from && !has_privs_of_role(GetUserId(), ROLE_PG_WRITE_SERVER_FILES)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to COPY to a file"), - errdetail("Only roles with privileges of the \"%s\" role may COPY to a file.", - "pg_write_server_files"), + errmsg("must be superuser or have privileges of the pg_write_server_files role to COPY to a file"), errhint("Anyone can COPY to stdout or from stdin. " "psql's \\copy command also works for anyone."))); } } + if (stmt->relation) { LOCKMODE lockmode = is_from ? RowExclusiveLock : AccessShareLock; diff --git a/src/test/regress/expected/mdb_copy.out b/src/test/regress/expected/mdb_copy.out new file mode 100644 index 00000000000..df6d70d1e7d --- /dev/null +++ b/src/test/regress/expected/mdb_copy.out @@ -0,0 +1,29 @@ +CREATE ROLE regress_mdb_copy_r1 LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_copy_r1_mdb_adm LOGIN NOSUPERUSER; +GRANT mdb_admin TO regress_mdb_copy_r1_mdb_adm; +CREATE ROLE regress_mdb_copy_r1_su LOGIN SUPERUSER; +-- should fail +SET ROLE regress_mdb_copy_r1; +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +ERROR: forbidden to COPY to or from an external program or file in Yandex Cloud +HINT: Anyone can COPY to stdout or from stdin. psql's \copy command also works for anyone. +DROP TABLE tt; +-- should fail +SET ROLE regress_mdb_copy_r1_mdb_adm; +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +ERROR: forbidden to COPY to or from an external program or file in Yandex Cloud +HINT: Anyone can COPY to stdout or from stdin. psql's \copy command also works for anyone. +DROP TABLE tt; +-- fail, no one can do it +SET ROLE regress_mdb_copy_r1_su; +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +ERROR: forbidden to COPY to or from an external program or file in Yandex Cloud +HINT: Anyone can COPY to stdout or from stdin. psql's \copy command also works for anyone. +DROP TABLE tt; +RESET SESSION AUTHORIZATION; +DROP ROLE regress_mdb_copy_r1; +DROP ROLE regress_mdb_copy_r1_mdb_adm; +DROP ROLE regress_mdb_copy_r1_su; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index a9b4d09e3ca..23b862c0b3b 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -19,6 +19,8 @@ test: mdb_superuser test: mdb_replication +test: mdb_copy + # ---------- # The first group of parallel tests # ---------- diff --git a/src/test/regress/sql/mdb_copy.sql b/src/test/regress/sql/mdb_copy.sql new file mode 100644 index 00000000000..c734b2814c5 --- /dev/null +++ b/src/test/regress/sql/mdb_copy.sql @@ -0,0 +1,31 @@ +CREATE ROLE regress_mdb_copy_r1 LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_copy_r1_mdb_adm LOGIN NOSUPERUSER; +GRANT mdb_admin TO regress_mdb_copy_r1_mdb_adm; +CREATE ROLE regress_mdb_copy_r1_su LOGIN SUPERUSER; + +-- should fail +SET ROLE regress_mdb_copy_r1; + +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +DROP TABLE tt; + +-- should fail +SET ROLE regress_mdb_copy_r1_mdb_adm; + +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +DROP TABLE tt; + +-- fail, no one can do it +SET ROLE regress_mdb_copy_r1_su; + +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +DROP TABLE tt; + +RESET SESSION AUTHORIZATION; + +DROP ROLE regress_mdb_copy_r1; +DROP ROLE regress_mdb_copy_r1_mdb_adm; +DROP ROLE regress_mdb_copy_r1_su; From 7d959846a1bc59c8513d6096a9b8590394063b49 Mon Sep 17 00:00:00 2001 From: usernamedt Date: Mon, 13 Feb 2023 15:00:31 +0800 Subject: [PATCH 11/54] Implement mdb-locales patch Do not use mdb_locales and mdb_newlocale when configured without them. Added ifdef codepath build without mdb-locales feature Add mdb locales patch, restore COPY from/to files, enable regress. Squashed commit of the following: commit 9f8ea4a5f42e0fd6077061bafbb428e88499896f Author: reshke kirill Date: Wed Feb 22 09:27:53 2023 +0000 Add mdb locales patch, restore COPY from/to files, enable regress. This commit does several things. * Enables back COPY from/to FILE functionality, becuase it is used by pg_regress * Enables pg_regress tests in deb build itself. * Add mdb locales function and checks that package build with mdb-locales support Add configure ac target to define USE_MDBLOCALES properly Refactor optional setlocale, fix minor issues --- configure | 126 +++++++++++++++++++++-- configure.ac | 17 +++ src/backend/utils/adt/Makefile | 3 +- src/backend/utils/adt/mdb.c | 36 +++++++ src/backend/utils/adt/pg_locale.c | 63 +++++++----- src/backend/utils/mb/mbutils.c | 3 +- src/bin/initdb/initdb.c | 11 +- src/bin/pg_upgrade/check.c | 2 + src/common/exec.c | 4 +- src/include/catalog/pg_proc.dat | 5 +- src/include/common/mdb_locale.h | 24 +++++ src/include/pg_config.h.in | 3 + src/interfaces/ecpg/ecpglib/connect.c | 3 +- src/interfaces/ecpg/ecpglib/descriptor.c | 8 +- src/interfaces/ecpg/ecpglib/execute.c | 7 +- src/interfaces/libpq/Makefile | 2 +- src/pl/plperl/plperl.c | 19 ++-- src/port/chklocale.c | 10 +- src/test/locale/test-ctype.c | 4 +- src/test/regress/expected/misc.out | 7 ++ src/test/regress/sql/misc.sql | 5 + 21 files changed, 295 insertions(+), 67 deletions(-) create mode 100644 src/backend/utils/adt/mdb.c create mode 100644 src/include/common/mdb_locale.h diff --git a/configure b/configure index b0308a2b9e7..d46f5e6d966 100755 --- a/configure +++ b/configure @@ -684,6 +684,7 @@ BISON MKDIR_P LN_S TAR +USE_MDBLOCALES install_bin INSTALL_DATA INSTALL_SCRIPT @@ -802,6 +803,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -872,6 +874,8 @@ with_system_tzdata with_zlib with_lz4 with_zstd +with_gnu_ld +with_mdblocales with_ssl with_openssl enable_largefile @@ -946,6 +950,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1198,6 +1203,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1335,7 +1349,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1488,6 +1502,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1584,6 +1599,8 @@ Optional Packages: --without-zlib do not use Zlib --with-lz4 build with LZ4 support --with-zstd build with ZSTD support + --with-gnu-ld assume the C compiler uses GNU ld [default=no] + --without-mdblocales build without MDB locales --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -2792,7 +2809,6 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu - ac_aux_dir= for ac_dir in config "$srcdir"/config; do if test -f "$ac_dir/install-sh"; then @@ -2823,6 +2839,7 @@ ac_configure="$SHELL $ac_aux_dir/configure" # Please don't use this var. +ac_configure_args=$(echo "$ac_configure_args" | sed -e "s/ -f\(debug\|file\)-prefix-map=[^' ]*//g") cat >>confdefs.h <<_ACEOF #define CONFIGURE_ARGS "$ac_configure_args" @@ -10132,6 +10149,40 @@ case $INSTALL in esac +# +# MDB locales +# + + + + +# Check whether --with-mdblocales was given. +if test "${with_mdblocales+set}" = set; then : + withval=$with_mdblocales; + case $withval in + yes) + +$as_echo "#define USE_MDBLOCALES 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-mdblocales option" "$LINENO" 5 + ;; + esac + +else + with_mdblocales=yes + +$as_echo "#define USE_MDBLOCALES 1" >>confdefs.h + +fi + + + + if test -z "$TAR"; then for ac_prog in tar do @@ -12706,6 +12757,56 @@ fi fi +if test "$with_mdblocales" = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mdb_setlocale in -lmdblocales" >&5 +$as_echo_n "checking for mdb_setlocale in -lmdblocales... " >&6; } +if ${ac_cv_lib_mdblocales_mdb_setlocale+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lmdblocales $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char mdb_setlocale (); +int +main () +{ +return mdb_setlocale (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_mdblocales_mdb_setlocale=yes +else + ac_cv_lib_mdblocales_mdb_setlocale=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mdblocales_mdb_setlocale" >&5 +$as_echo "$ac_cv_lib_mdblocales_mdb_setlocale" >&6; } +if test "x$ac_cv_lib_mdblocales_mdb_setlocale" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBMDBLOCALES 1 +_ACEOF + + LIBS="-lmdblocales $LIBS" + +else + as_fn_error $? "mdblocales library not found" "$LINENO" 5 +fi + +fi + if test "$enable_spinlocks" = yes; then $as_echo "#define HAVE_SPINLOCKS 1" >>confdefs.h @@ -14078,6 +14179,17 @@ else fi +fi + +if test "$with_mdblocales" = yes; then + ac_fn_c_check_header_mongrel "$LINENO" "mdblocales.h" "ac_cv_header_mdblocales_h" "$ac_includes_default" +if test "x$ac_cv_header_mdblocales_h" = xyes; then : + +else + as_fn_error $? "mdblocales header not found." "$LINENO" 5 +fi + + fi if test "$with_gssapi" = yes ; then @@ -15443,7 +15555,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15489,7 +15601,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15513,7 +15625,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15558,7 +15670,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15582,7 +15694,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; diff --git a/configure.ac b/configure.ac index 9eefefd6804..b3d5efcde3d 100644 --- a/configure.ac +++ b/configure.ac @@ -1180,6 +1180,14 @@ case $INSTALL in esac AC_SUBST(install_bin) +# +# MDB locales +# + +PGAC_ARG_BOOL(with, mdblocales, yes, [build without MDB locales], + [AC_DEFINE([USE_MDBLOCALES], 1, [Define to 1 to build with MDB locales. (--with-mdblocales)])]) +AC_SUBST(USE_MDBLOCALES) + PGAC_PATH_PROGS(TAR, tar) AC_PROG_LN_S AC_PROG_MKDIR_P @@ -1335,6 +1343,11 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_mdblocales" = yes; then + AC_CHECK_LIB(mdblocales, mdb_setlocale, [], + [AC_MSG_ERROR([mdblocales library not found])]) +fi + if test "$enable_spinlocks" = yes; then AC_DEFINE(HAVE_SPINLOCKS, 1, [Define to 1 if you have spinlocks.]) else @@ -1568,6 +1581,10 @@ if test "$with_zstd" = yes; then AC_CHECK_HEADER(zstd.h, [], [AC_MSG_ERROR([zstd.h header file is required for ZSTD])]) fi +if test "$with_mdblocales" = yes; then + AC_CHECK_HEADER(mdblocales.h, [], [AC_MSG_ERROR([mdblocales header not found.])]) +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/utils/adt/Makefile b/src/backend/utils/adt/Makefile index 0de0bbb1b8a..930bede049c 100644 --- a/src/backend/utils/adt/Makefile +++ b/src/backend/utils/adt/Makefile @@ -118,7 +118,8 @@ OBJS = \ windowfuncs.o \ xid.o \ xid8funcs.o \ - xml.o + xml.o \ + mdb.o # See notes in src/backend/parser/Makefile about the following two rules jsonpath_gram.h: jsonpath_gram.c diff --git a/src/backend/utils/adt/mdb.c b/src/backend/utils/adt/mdb.c new file mode 100644 index 00000000000..cc61073fa58 --- /dev/null +++ b/src/backend/utils/adt/mdb.c @@ -0,0 +1,36 @@ +/*------------------------------------------------------------------------- + * + * mdb.c + * mdb routines + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/utils/adt/mdb.c + * + *------------------------------------------------------------------------- + */ + + +#include "postgres.h" +#include "fmgr.h" + +/* + * mdb_admin_enabled + * Check that mdb locale patch is enabled + */ +Datum +mdb_locale_enabled(PG_FUNCTION_ARGS) +{ + bool res; + +#if USE_MDBLOCALES + res = true; +#else + res = false; +#endif + + PG_RETURN_BOOL(res); +} diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c index bee165cd6a1..bcbde2ef6ea 100644 --- a/src/backend/utils/adt/pg_locale.c +++ b/src/backend/utils/adt/pg_locale.c @@ -69,6 +69,7 @@ #include "utils/pg_locale.h" #include "utils/relcache.h" #include "utils/syscache.h" +#include "common/mdb_locale.h" #ifdef USE_ICU #include @@ -181,7 +182,7 @@ pg_perm_setlocale(int category, const char *locale) const char *envvar; #ifndef WIN32 - result = setlocale(category, locale); + result = SETLOCALE(category, locale); #else /* @@ -199,7 +200,7 @@ pg_perm_setlocale(int category, const char *locale) } else #endif - result = setlocale(category, locale); + result = SETLOCALE(category, locale); #endif /* WIN32 */ if (result == NULL) @@ -296,7 +297,7 @@ check_locale(int category, const char *locale, char **canonname) if (canonname) *canonname = NULL; /* in case of failure */ - save = setlocale(category, NULL); + save = SETLOCALE(category, NULL); if (!save) return false; /* won't happen, we hope */ @@ -304,14 +305,14 @@ check_locale(int category, const char *locale, char **canonname) save = pstrdup(save); /* set the locale with setlocale, to see if it accepts it. */ - res = setlocale(category, locale); + res = SETLOCALE(category, locale); /* save canonical name if requested. */ if (res && canonname) *canonname = pstrdup(res); /* restore old value. */ - if (!setlocale(category, save)) + if (!SETLOCALE(category, save)) elog(WARNING, "failed to restore old locale \"%s\"", save); pfree(save); @@ -547,12 +548,12 @@ PGLC_localeconv(void) memset(&worklconv, 0, sizeof(worklconv)); /* Save prevailing values of monetary and numeric locales */ - save_lc_monetary = setlocale(LC_MONETARY, NULL); + save_lc_monetary = SETLOCALE(LC_MONETARY, NULL); if (!save_lc_monetary) elog(ERROR, "setlocale(NULL) failed"); save_lc_monetary = pstrdup(save_lc_monetary); - save_lc_numeric = setlocale(LC_NUMERIC, NULL); + save_lc_numeric = SETLOCALE(LC_NUMERIC, NULL); if (!save_lc_numeric) elog(ERROR, "setlocale(NULL) failed"); save_lc_numeric = pstrdup(save_lc_numeric); @@ -574,7 +575,7 @@ PGLC_localeconv(void) */ /* Save prevailing value of ctype locale */ - save_lc_ctype = setlocale(LC_CTYPE, NULL); + save_lc_ctype = SETLOCALE(LC_CTYPE, NULL); if (!save_lc_ctype) elog(ERROR, "setlocale(NULL) failed"); save_lc_ctype = pstrdup(save_lc_ctype); @@ -582,11 +583,11 @@ PGLC_localeconv(void) /* Here begins the critical section where we must not throw error */ /* use numeric to set the ctype */ - setlocale(LC_CTYPE, locale_numeric); + SETLOCALE(LC_CTYPE, locale_numeric); #endif /* Get formatting information for numeric */ - setlocale(LC_NUMERIC, locale_numeric); + SETLOCALE(LC_NUMERIC, locale_numeric); extlconv = localeconv(); /* Must copy data now in case setlocale() overwrites it */ @@ -596,11 +597,11 @@ PGLC_localeconv(void) #ifdef WIN32 /* use monetary to set the ctype */ - setlocale(LC_CTYPE, locale_monetary); + SETLOCALE(LC_CTYPE, locale_monetary); #endif /* Get formatting information for monetary */ - setlocale(LC_MONETARY, locale_monetary); + SETLOCALE(LC_MONETARY, locale_monetary); extlconv = localeconv(); /* Must copy data now in case setlocale() overwrites it */ @@ -630,12 +631,12 @@ PGLC_localeconv(void) * should fail. */ #ifdef WIN32 - if (!setlocale(LC_CTYPE, save_lc_ctype)) + if (!SETLOCALE(LC_CTYPE, save_lc_ctype)) elog(FATAL, "failed to restore LC_CTYPE to \"%s\"", save_lc_ctype); #endif - if (!setlocale(LC_MONETARY, save_lc_monetary)) + if (!SETLOCALE(LC_MONETARY, save_lc_monetary)) elog(FATAL, "failed to restore LC_MONETARY to \"%s\"", save_lc_monetary); - if (!setlocale(LC_NUMERIC, save_lc_numeric)) + if (!SETLOCALE(LC_NUMERIC, save_lc_numeric)) elog(FATAL, "failed to restore LC_NUMERIC to \"%s\"", save_lc_numeric); /* @@ -819,7 +820,7 @@ cache_locale_time(void) */ /* Save prevailing value of time locale */ - save_lc_time = setlocale(LC_TIME, NULL); + save_lc_time = SETLOCALE(LC_TIME, NULL); if (!save_lc_time) elog(ERROR, "setlocale(NULL) failed"); save_lc_time = pstrdup(save_lc_time); @@ -834,16 +835,16 @@ cache_locale_time(void) */ /* Save prevailing value of ctype locale */ - save_lc_ctype = setlocale(LC_CTYPE, NULL); + save_lc_ctype = SETLOCALE(LC_CTYPE, NULL); if (!save_lc_ctype) elog(ERROR, "setlocale(NULL) failed"); save_lc_ctype = pstrdup(save_lc_ctype); /* use lc_time to set the ctype */ - setlocale(LC_CTYPE, locale_time); + SETLOCALE(LC_CTYPE, locale_time); #endif - setlocale(LC_TIME, locale_time); + SETLOCALE(LC_TIME, locale_time); /* We use times close to current time as data for strftime(). */ timenow = time(NULL); @@ -892,10 +893,10 @@ cache_locale_time(void) * failure to do so is fatal. */ #ifdef WIN32 - if (!setlocale(LC_CTYPE, save_lc_ctype)) + if (!SETLOCALE(LC_CTYPE, save_lc_ctype)) elog(FATAL, "failed to restore LC_CTYPE to \"%s\"", save_lc_ctype); #endif - if (!setlocale(LC_TIME, save_lc_time)) + if (!SETLOCALE(LC_TIME, save_lc_time)) elog(FATAL, "failed to restore LC_TIME to \"%s\"", save_lc_time); /* @@ -1316,7 +1317,7 @@ lc_collate_is_c(Oid collation) if (result >= 0) return (bool) result; - localeptr = setlocale(LC_COLLATE, NULL); + localeptr = SETLOCALE(LC_COLLATE, NULL); if (!localeptr) elog(ERROR, "invalid LC_COLLATE setting"); @@ -1369,7 +1370,7 @@ lc_ctype_is_c(Oid collation) if (result >= 0) return (bool) result; - localeptr = setlocale(LC_CTYPE, NULL); + localeptr = SETLOCALE(LC_CTYPE, NULL); if (!localeptr) elog(ERROR, "invalid LC_CTYPE setting"); @@ -1563,8 +1564,10 @@ pg_newlocale_from_collation(Oid collid) /* Normal case where they're the same */ errno = 0; #ifndef WIN32 - loc = newlocale(LC_COLLATE_MASK | LC_CTYPE_MASK, collcollate, + + loc = NEWLOCALE(LC_COLLATE_MASK | LC_CTYPE_MASK, collcollate, NULL); + #else loc = _create_locale(LC_ALL, collcollate); #endif @@ -1578,11 +1581,11 @@ pg_newlocale_from_collation(Oid collid) locale_t loc1; errno = 0; - loc1 = newlocale(LC_COLLATE_MASK, collcollate, NULL); + loc1 = NEWLOCALE(LC_COLLATE_MASK, collcollate, NULL); if (!loc1) report_newlocale_failure(collcollate); errno = 0; - loc = newlocale(LC_CTYPE_MASK, collctype, loc1); + loc = NEWLOCALE(LC_CTYPE_MASK, collctype, loc1); if (!loc) report_newlocale_failure(collctype); #else @@ -1707,12 +1710,16 @@ get_collation_actual_version(char collprovider, const char *collcollate) { #if defined(__GLIBC__) /* Use the glibc version because we don't have anything better. */ - collversion = pstrdup(gnu_get_libc_version()); +#ifdef USE_MDBLOCALES + collversion = pstrdup(mdb_localesversion()); +#else + collversion = pstrdup(gnu_get_libc_version()); +#endif #elif defined(LC_VERSION_MASK) locale_t loc; /* Look up FreeBSD collation version. */ - loc = newlocale(LC_COLLATE_MASK, collcollate, NULL); + loc = NEWLOCALE(LC_COLLATE_MASK, collcollate, NULL); if (loc) { collversion = diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 36bcd9e25a5..acba6663fd9 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -43,6 +43,7 @@ #include "utils/relcache.h" #include "utils/syscache.h" #include "varatt.h" +#include "common/mdb_locale.h" /* * We maintain a simple linked list caching the fmgr lookup info for the @@ -1363,7 +1364,7 @@ pg_bind_textdomain_codeset(const char *domainname) int new_msgenc; #ifndef WIN32 - const char *ctype = setlocale(LC_CTYPE, NULL); + const char *ctype = SETLOCALE(LC_CTYPE, NULL); if (pg_strcasecmp(ctype, "C") == 0 || pg_strcasecmp(ctype, "POSIX") == 0) #endif diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index f2ee1d999b4..66ff614ef46 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -80,6 +80,7 @@ #include "getopt_long.h" #include "mb/pg_wchar.h" #include "miscadmin.h" +#include "common/mdb_locale.h" /* Ideally this would be in a .h file, but it hardly seems worth the trouble */ @@ -367,9 +368,9 @@ save_global_locale(int category) if (!save) pg_fatal("out of memory"); #else - save = setlocale(category, NULL); + save = SETLOCALE(category, NULL); if (!save) - pg_fatal("setlocale() failed"); + pg_fatal("[mdb]setlocale() failed"); save = pg_strdup(save); #endif return save; @@ -385,7 +386,7 @@ restore_global_locale(int category, save_locale_t save) if (!_wsetlocale(category, save)) pg_fatal("failed to restore old locale"); #else - if (!setlocale(category, save)) + if (!SETLOCALE(category, save)) pg_fatal("failed to restore old locale \"%s\"", save); #endif free(save); @@ -2128,7 +2129,7 @@ locale_date_order(const char *locale) save = save_global_locale(LC_TIME); - setlocale(LC_TIME, locale); + SETLOCALE(LC_TIME, locale); memset(&testtime, 0, sizeof(testtime)); testtime.tm_mday = 22; @@ -2191,7 +2192,7 @@ check_locale_name(int category, const char *locale, char **canonname) locale = ""; /* set the locale with setlocale, to see if it accepts it. */ - res = setlocale(category, locale); + res = SETLOCALE(category, locale); /* save canonical name if requested. */ if (res && canonname) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index f3bd6a2102f..6d91748d785 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -14,6 +14,8 @@ #include "fe_utils/string_utils.h" #include "mb/pg_wchar.h" #include "pg_upgrade.h" +#include "common/mdb_locale.h" + static void check_new_cluster_is_empty(void); static void check_is_install_user(ClusterInfo *cluster); diff --git a/src/common/exec.c b/src/common/exec.c index f209b934df7..41a4308d0cc 100644 --- a/src/common/exec.c +++ b/src/common/exec.c @@ -33,6 +33,8 @@ #include #include #include +#include "common/mdb_locale.h" + #ifdef EXEC_BACKEND #if defined(HAVE_SYS_PERSONALITY_H) @@ -441,7 +443,7 @@ set_pglocale_pgservice(const char *argv0, const char *app) /* don't set LC_ALL in the backend */ if (strcmp(app, PG_TEXTDOMAIN("postgres")) != 0) { - setlocale(LC_ALL, ""); + SETLOCALE(LC_ALL, ""); /* * One could make a case for reproducing here PostmasterMain()'s test diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 6996073989a..bbae8d6ee47 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12034,7 +12034,6 @@ proname => 'brin_minmax_multi_summary_send', provolatile => 's', prorettype => 'bytea', proargtypes => 'pg_brin_minmax_multi_summary', prosrc => 'brin_minmax_multi_summary_send' }, - { oid => '6291', descr => 'arbitrary value from among input values', proname => 'any_value', prokind => 'a', proisstrict => 'f', prorettype => 'anyelement', proargtypes => 'anyelement', @@ -12042,5 +12041,7 @@ { oid => '6292', descr => 'aggregate transition function', proname => 'any_value_transfn', prorettype => 'anyelement', proargtypes => 'anyelement anyelement', prosrc => 'any_value_transfn' }, - +{ oid => '16383', descr => 'contains', + proname => 'mdb_locale_enabled', prorettype => 'bool', + proargtypes => '', prosrc => 'mdb_locale_enabled' }, ] diff --git a/src/include/common/mdb_locale.h b/src/include/common/mdb_locale.h new file mode 100644 index 00000000000..61290b2d938 --- /dev/null +++ b/src/include/common/mdb_locale.h @@ -0,0 +1,24 @@ +/*------------------------------------------------------------------------- + * + * locale_mdb.h + * Generic headers for custom MDB-locales patch. + * + * IDENTIFICATION + * src/include/common/mdb_locale.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PG_MDB_LOCALE_H +#define PG_MDB_LOCALE_H + +#ifdef USE_MDBLOCALES +#include +#define SETLOCALE(category, locale) mdb_setlocale(category, locale) +#define NEWLOCALE(category, locale, base) mdb_newlocale(category, locale, base) +#else +#define SETLOCALE(category, locale) setlocale(category, locale) +#define NEWLOCALE(category, locale, base) newlocale(category, locale, base) +#endif + +#endif /* PG_MDB_LOCALE_H */ \ No newline at end of file diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 015c034c7cb..188416fefa2 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -655,6 +655,9 @@ /* A string containing the version number, platform, and C compiler */ #undef PG_VERSION_STR +/* Use mdb locales or not */ +#undef USE_MDBLOCALES + /* Define to 1 to allow profiling output to be saved separately for each process. */ #undef PROFILE_PID_DIR diff --git a/src/interfaces/ecpg/ecpglib/connect.c b/src/interfaces/ecpg/ecpglib/connect.c index 19261fe56c2..dc246660041 100644 --- a/src/interfaces/ecpg/ecpglib/connect.c +++ b/src/interfaces/ecpg/ecpglib/connect.c @@ -9,6 +9,7 @@ #include "ecpglib_extern.h" #include "ecpgtype.h" #include "sqlca.h" +#include "common/mdb_locale.h" #ifdef HAVE_USELOCALE locale_t ecpg_clocale = (locale_t) 0; @@ -521,7 +522,7 @@ ECPGconnect(int lineno, int c, const char *name, const char *user, const char *p #ifdef HAVE_USELOCALE if (!ecpg_clocale) { - ecpg_clocale = newlocale(LC_NUMERIC_MASK, "C", (locale_t) 0); + ecpg_clocale = NEWLOCALE(LC_NUMERIC_MASK, "C", (locale_t) 0); if (!ecpg_clocale) { #ifdef ENABLE_THREAD_SAFETY diff --git a/src/interfaces/ecpg/ecpglib/descriptor.c b/src/interfaces/ecpg/ecpglib/descriptor.c index 883a210a812..3781e9740ed 100644 --- a/src/interfaces/ecpg/ecpglib/descriptor.c +++ b/src/interfaces/ecpg/ecpglib/descriptor.c @@ -15,6 +15,8 @@ #include "sql3types.h" #include "sqlca.h" #include "sqlda.h" +#include "common/mdb_locale.h" + static void descriptor_free(struct descriptor *desc); @@ -500,8 +502,8 @@ ECPGget_desc(int lineno, const char *desc_name, int index,...) #ifdef HAVE__CONFIGTHREADLOCALE stmt.oldthreadlocale = _configthreadlocale(_ENABLE_PER_THREAD_LOCALE); #endif - stmt.oldlocale = ecpg_strdup(setlocale(LC_NUMERIC, NULL), lineno); - setlocale(LC_NUMERIC, "C"); + stmt.oldlocale = ecpg_strdup(SETLOCALE(LC_NUMERIC, NULL), lineno); + SETLOCALE(LC_NUMERIC, "C"); #endif /* desperate try to guess something sensible */ @@ -514,7 +516,7 @@ ECPGget_desc(int lineno, const char *desc_name, int index,...) #else if (stmt.oldlocale) { - setlocale(LC_NUMERIC, stmt.oldlocale); + SETLOCALE(LC_NUMERIC, stmt.oldlocale); ecpg_free(stmt.oldlocale); } #ifdef HAVE__CONFIGTHREADLOCALE diff --git a/src/interfaces/ecpg/ecpglib/execute.c b/src/interfaces/ecpg/ecpglib/execute.c index 93926fd4fb1..bb9fe577276 100644 --- a/src/interfaces/ecpg/ecpglib/execute.c +++ b/src/interfaces/ecpg/ecpglib/execute.c @@ -31,6 +31,7 @@ #include "sqlca.h" #include "sqlda-compat.h" #include "sqlda-native.h" +#include "common/mdb_locale.h" /* * This function returns a newly malloced string that has ' and \ @@ -2000,13 +2001,13 @@ ecpg_do_prologue(int lineno, const int compat, const int force_indicator, #ifdef HAVE__CONFIGTHREADLOCALE stmt->oldthreadlocale = _configthreadlocale(_ENABLE_PER_THREAD_LOCALE); #endif - stmt->oldlocale = ecpg_strdup(setlocale(LC_NUMERIC, NULL), lineno); + stmt->oldlocale = ecpg_strdup(SETLOCALE(LC_NUMERIC, NULL), lineno); if (stmt->oldlocale == NULL) { ecpg_do_epilogue(stmt); return false; } - setlocale(LC_NUMERIC, "C"); + SETLOCALE(LC_NUMERIC, "C"); #endif /* @@ -2220,7 +2221,7 @@ ecpg_do_epilogue(struct statement *stmt) uselocale(stmt->oldlocale); #else if (stmt->oldlocale) - setlocale(LC_NUMERIC, stmt->oldlocale); + SETLOCALE(LC_NUMERIC, stmt->oldlocale); #ifdef HAVE__CONFIGTHREADLOCALE /* diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 5cf1d6400e6..659551da57d 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -84,7 +84,7 @@ endif # that are built correctly for use in a shlib. SHLIB_LINK_INTERNAL = -lpgcommon_shlib -lpgport_shlib ifneq ($(PORTNAME), win32) -SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) +SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm -lmdblocales, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) else SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi32 -lssl -lsocket -lnsl -lresolv -lintl -lm $(PTHREAD_LIBS), $(LIBS)) $(LDAP_LIBS_FE) endif diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index 863864253f9..7fb585b4bd1 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -37,6 +37,7 @@ #include "utils/rel.h" #include "utils/syscache.h" #include "utils/typcache.h" +#include "common/mdb_locale.h" /* define our text domain for translations */ #undef TEXTDOMAIN @@ -740,15 +741,15 @@ plperl_init_interp(void) *save_numeric, *save_time; - loc = setlocale(LC_COLLATE, NULL); + loc = SETLOCALE(LC_COLLATE, NULL); save_collate = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_CTYPE, NULL); + loc = SETLOCALE(LC_CTYPE, NULL); save_ctype = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_MONETARY, NULL); + loc = SETLOCALE(LC_MONETARY, NULL); save_monetary = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_NUMERIC, NULL); + loc = SETLOCALE(LC_NUMERIC, NULL); save_numeric = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_TIME, NULL); + loc = SETLOCALE(LC_TIME, NULL); save_time = loc ? pstrdup(loc) : NULL; #define PLPERL_RESTORE_LOCALE(name, saved) \ @@ -4181,7 +4182,7 @@ static char * setlocale_perl(int category, char *locale) { dTHX; - char *RETVAL = setlocale(category, locale); + char *RETVAL = SETLOCALE(category, locale); if (RETVAL) { @@ -4196,7 +4197,7 @@ setlocale_perl(int category, char *locale) #ifdef LC_ALL if (category == LC_ALL) - newctype = setlocale(LC_CTYPE, NULL); + newctype = SETLOCALE(LC_CTYPE, NULL); else #endif newctype = RETVAL; @@ -4214,7 +4215,7 @@ setlocale_perl(int category, char *locale) #ifdef LC_ALL if (category == LC_ALL) - newcoll = setlocale(LC_COLLATE, NULL); + newcoll = SETLOCALE(LC_COLLATE, NULL); else #endif newcoll = RETVAL; @@ -4233,7 +4234,7 @@ setlocale_perl(int category, char *locale) #ifdef LC_ALL if (category == LC_ALL) - newnum = setlocale(LC_NUMERIC, NULL); + newnum = SETLOCALE(LC_NUMERIC, NULL); else #endif newnum = RETVAL; diff --git a/src/port/chklocale.c b/src/port/chklocale.c index 6fa6810a46a..aa60adb9224 100644 --- a/src/port/chklocale.c +++ b/src/port/chklocale.c @@ -18,6 +18,8 @@ #else #include "postgres_fe.h" #endif +#include "common/mdb_locale.h" + #ifdef HAVE_LANGINFO_H #include @@ -319,7 +321,7 @@ pg_get_encoding_from_locale(const char *ctype, bool write_message) pg_strcasecmp(ctype, "POSIX") == 0) return PG_SQL_ASCII; - save = setlocale(LC_CTYPE, NULL); + save = SETLOCALE(LC_CTYPE, NULL); if (!save) return -1; /* setlocale() broken? */ /* must copy result, or it might change after setlocale */ @@ -327,7 +329,7 @@ pg_get_encoding_from_locale(const char *ctype, bool write_message) if (!save) return -1; /* out of memory; unlikely */ - name = setlocale(LC_CTYPE, ctype); + name = SETLOCALE(LC_CTYPE, ctype); if (!name) { free(save); @@ -342,13 +344,13 @@ pg_get_encoding_from_locale(const char *ctype, bool write_message) sys = win32_langinfo(name); #endif - setlocale(LC_CTYPE, save); + SETLOCALE(LC_CTYPE, save); free(save); } else { /* much easier... */ - ctype = setlocale(LC_CTYPE, NULL); + ctype = SETLOCALE(LC_CTYPE, NULL); if (!ctype) return -1; /* setlocale() broken? */ diff --git a/src/test/locale/test-ctype.c b/src/test/locale/test-ctype.c index a3f896c5ecb..10c2b49cb92 100644 --- a/src/test/locale/test-ctype.c +++ b/src/test/locale/test-ctype.c @@ -23,6 +23,8 @@ the author shall be liable for any damage, etc. #include #include #include +#include "common/mdb_locale.h" + char *flag(int b); void describe_char(int c); @@ -62,7 +64,7 @@ main() short c; char *cur_locale; - cur_locale = setlocale(LC_ALL, ""); + cur_locale = SETLOCALE(LC_ALL, ""); if (cur_locale) fprintf(stderr, "Successfully set locale to \"%s\"\n", cur_locale); else diff --git a/src/test/regress/expected/misc.out b/src/test/regress/expected/misc.out index 6e816c57f1f..d2cbc8f87f4 100644 --- a/src/test/regress/expected/misc.out +++ b/src/test/regress/expected/misc.out @@ -396,3 +396,10 @@ SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; -- -- rewrite rules -- +--- mdb-related +SELECT mdb_locale_enabled(); + mdb_locale_enabled +-------------------- + t +(1 row) + diff --git a/src/test/regress/sql/misc.sql b/src/test/regress/sql/misc.sql index 165a2e175fb..597186ffe4b 100644 --- a/src/test/regress/sql/misc.sql +++ b/src/test/regress/sql/misc.sql @@ -273,3 +273,8 @@ SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; -- -- rewrite rules -- + +--- mdb-related + +SELECT mdb_locale_enabled(); + From 2ffab63bce49b58e9b5901de237ec3b566a7ebf8 Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Thu, 11 May 2023 08:36:59 +0000 Subject: [PATCH 12/54] MDB-23247: startup param for auth passthrough under unpriviledged user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accept new startup param _pq_.service_auth_role for service auth under any user, which is niether superuser nor have some dangerous system role priv Add tap-test for mdb service role auth 👍👌😉 Fix tests after rebase contrib tests 💅️️💅️️💅️️ now works MDB-23247: debug ouput for testing purposes lowered to DEBUG5 elog level Skip caching routines for pre-startup logic (SCRAM -service auth role) --- contrib/seg/Makefile | 4 +- src/backend/libpq/auth.c | 25 +++++++- src/backend/postmaster/postmaster.c | 21 ++++--- src/backend/utils/adt/acl.c | 70 ++++++++++++++++----- src/bin/psql/t/100_mdb.pl | 98 +++++++++++++++++++++++++++++ src/include/libpq/libpq-be.h | 4 ++ src/include/utils/acl.h | 6 +- src/interfaces/libpq/fe-connect.c | 9 +++ src/interfaces/libpq/fe-protocol3.c | 4 ++ src/interfaces/libpq/libpq-int.h | 2 + 10 files changed, 218 insertions(+), 25 deletions(-) create mode 100644 src/bin/psql/t/100_mdb.pl diff --git a/contrib/seg/Makefile b/contrib/seg/Makefile index a1e49bf051e..c75351b204d 100644 --- a/contrib/seg/Makefile +++ b/contrib/seg/Makefile @@ -13,8 +13,10 @@ DATA = seg--1.1.sql seg--1.1--1.2.sql seg--1.2--1.3.sql seg--1.3--1.4.sql \ PGFILEDESC = "seg - line segment data type" HEADERS = segdata.h +# MDB-23261 diasble security test, because we do not extension creation via sql for now +#REGRESS = security seg -REGRESS = security seg +REGRESS = seg EXTRA_CLEAN = y.tab.c y.tab.h diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index 183d9e731d6..c0f1f76a797 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -40,6 +40,7 @@ #include "utils/guc.h" #include "utils/memutils.h" #include "utils/timestamp.h" +#include "utils/acl.h" /*---------------------------------------------------------------- * Global authentication functions @@ -818,12 +819,32 @@ CheckPWChallengeAuth(Port *port, const char **logdetail) int auth_result; char *shadow_pass; PasswordType pwtype; + Oid mdb_service_authoid; + Oid useroid; + Oid service_auth_roleoid; Assert(port->hba->auth_method == uaSCRAM || port->hba->auth_method == uaMD5); - /* First look up the user's password. */ - shadow_pass = get_role_password(port->user_name, logdetail); + + + if (port->service_auth_role) { + mdb_service_authoid = get_role_oid("mdb_service_auth", true); + service_auth_roleoid = get_role_oid(port->service_auth_role, true); + useroid = get_role_oid(port->user_name, true); + + /* MDB-23247: check that given role name has priviledge for auth - passthrough*/ + if (!is_member_of_role(service_auth_roleoid, mdb_service_authoid) || has_privs_of_unwanted_system_role_prestartup(useroid)) + ereport(FATAL, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid auth request for role %s", + port->service_auth_role))); + + shadow_pass = get_role_password(port->service_auth_role, logdetail); + } else { + /* First look up the user's password. */ + shadow_pass = get_role_password(port->user_name, logdetail); + } /* * If the user does not exist, or has no password or it's expired, we diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 5d328b9807c..bab2f05a544 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -2189,6 +2189,8 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done) break; /* missing value, will complain below */ valptr = buf + valoffset; + elog(DEBUG5, "startup got %s %s", nameptr, valptr); + if (strcmp(nameptr, "database") == 0) port->database_name = pstrdup(valptr); else if (strcmp(nameptr, "user") == 0) @@ -2219,13 +2221,18 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done) } else if (strncmp(nameptr, "_pq_.", 5) == 0) { - /* - * Any option beginning with _pq_. is reserved for use as a - * protocol-level option, but at present no such options are - * defined. - */ - unrecognized_protocol_options = - lappend(unrecognized_protocol_options, pstrdup(nameptr)); + /* MDB-23247: parse service auth role from startup options */ + if (strcmp(nameptr, "_pq_.service_auth_role") == 0) { + port->service_auth_role = pstrdup(valptr); + } else { + /* + * Any option beginning with _pq_. is reserved for use as a + * protocol-level option, but at present no such options are + * defined. + */ + unrecognized_protocol_options = + lappend(unrecognized_protocol_options, pstrdup(nameptr)); + } } else { diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 7795e1e7ec3..12e40119bfa 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -123,8 +123,6 @@ static AclResult pg_role_aclcheck(Oid role_oid, Oid roleid, AclMode mode); static void RoleMembershipCacheCallback(Datum arg, int cacheid, uint32 hashvalue); -static bool has_privs_of_unwanted_system_role(Oid role); - /* * Test whether an identifier char can be left unquoted in ACLs. @@ -4853,7 +4851,7 @@ RoleMembershipCacheCallback(Datum arg, int cacheid, uint32 hashvalue) */ static List * roles_is_member_of(Oid roleid, enum RoleRecurseType type, - Oid admin_of, Oid *admin_role) + Oid admin_of, Oid *admin_role, bool no_cache) { Oid dba; List *roles_list; @@ -4869,7 +4867,6 @@ roles_is_member_of(Oid roleid, enum RoleRecurseType type, if (cached_role[type] == roleid && !OidIsValid(admin_of) && OidIsValid(cached_role[type])) return cached_roles[type]; - /* * Role expansion happens in a non-database backend when guc.c checks * ROLE_PG_READ_ALL_SETTINGS for a physical walsender SHOW command. In @@ -4961,7 +4958,10 @@ roles_is_member_of(Oid roleid, enum RoleRecurseType type, cached_role[type] = InvalidOid; /* just paranoia */ list_free(cached_roles[type]); cached_roles[type] = new_cached_roles; - cached_role[type] = roleid; + + if (!no_cache) { + cached_role[type] = roleid; + } /* And now we can return the answer */ return cached_roles[type]; @@ -4997,17 +4997,39 @@ has_privs_of_role_strict(Oid member, Oid role) * multi-level recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_PRIVS, - InvalidOid, NULL), + InvalidOid, NULL, false), + role); +} + + +static bool +has_privs_of_role_strict_no_cache(Oid member, Oid role) +{ + /* Fast path for simple case */ + if (member == role) + return true; + + /* Superusers have every privilege, so are part of every role */ + if (superuser_arg(member)) + return true; + + /* + * Find all the roles that member has the privileges of, including + * multi-level recursion, then see if target role is any one of them. + */ + return list_member_oid(roles_is_member_of(member, ROLERECURSE_PRIVS, + InvalidOid, NULL, true), role); } + /* * Check that role is either one of "dangerous" system role * or has "strict" (not through mdb_admin or mdb_superuser) * privs of this role */ -static bool +bool has_privs_of_unwanted_system_role(Oid role) { if (has_privs_of_role_strict(role, ROLE_PG_READ_SERVER_FILES)) { return true; @@ -5028,6 +5050,26 @@ has_privs_of_unwanted_system_role(Oid role) { return false; } +bool has_privs_of_unwanted_system_role_prestartup(Oid role) { + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_READ_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_WRITE_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { + return true; + } + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_READ_ALL_DATA)) { + return true; + } + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_WRITE_ALL_DATA)) { + return true; + } + + return false; +} + bool has_privs_of_role(Oid member, Oid role) { @@ -5063,7 +5105,7 @@ has_privs_of_role(Oid member, Oid role) * multi-level recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_PRIVS, - InvalidOid, NULL), + InvalidOid, NULL, false), role); } @@ -5140,7 +5182,7 @@ member_can_set_role(Oid member, Oid role) * multi-level recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_SETROLE, - InvalidOid, NULL), + InvalidOid, NULL, false), role); } @@ -5253,7 +5295,7 @@ is_member_of_role(Oid member, Oid role) * recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_MEMBERS, - InvalidOid, NULL), + InvalidOid, NULL, false), role); } @@ -5277,7 +5319,7 @@ is_member_of_role_nosuper(Oid member, Oid role) * recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_MEMBERS, - InvalidOid, NULL), + InvalidOid, NULL, false), role); } @@ -5299,7 +5341,7 @@ is_admin_of_role(Oid member, Oid role) if (member == role) return false; - (void) roles_is_member_of(member, ROLERECURSE_MEMBERS, role, &admin_role); + (void) roles_is_member_of(member, ROLERECURSE_MEMBERS, role, &admin_role, false); return OidIsValid(admin_role); } @@ -5321,7 +5363,7 @@ select_best_admin(Oid member, Oid role) if (member == role) return InvalidOid; - (void) roles_is_member_of(member, ROLERECURSE_PRIVS, role, &admin_role); + (void) roles_is_member_of(member, ROLERECURSE_PRIVS, role, &admin_role, false); return admin_role; } @@ -5408,7 +5450,7 @@ select_best_grantor(Oid roleId, AclMode privileges, * doesn't query any role memberships. */ roles_list = roles_is_member_of(roleId, ROLERECURSE_PRIVS, - InvalidOid, NULL); + InvalidOid, NULL, false); /* initialize candidate result as default */ *grantorId = roleId; *grantOptions = ACL_NO_RIGHTS; diff --git a/src/bin/psql/t/100_mdb.pl b/src/bin/psql/t/100_mdb.pl new file mode 100644 index 00000000000..d5b0342354c --- /dev/null +++ b/src/bin/psql/t/100_mdb.pl @@ -0,0 +1,98 @@ + +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +use strict; +use warnings; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize primary node. Force UTF-8 encoding, so that we can use non-ASCII +# characters in the passwords below. +my $node = PostgreSQL::Test::Cluster->new('primary'); +$node->init(extra => [ '--locale=C', '--encoding=UTF8' ]); +$node->start; + + +# Create test roles. +$node->safe_psql( + 'postgres', + "SET password_encryption='scram-sha-256'; +SET client_encoding='utf8'; +CREATE USER mdbsar_test_role LOGIN PASSWORD 'test'; + +CREATE ROLE mdb_service_auth LOGIN PASSWORD 'serv'; +CREATE ROLE sup LOGIN SUPERUSER PASSWORD '123'; +"); + + +# Delete pg_hba.conf from the given node, add a new entry to it +# and then execute a reload to refresh it. +sub reset_pg_hba +{ + my $node = shift; + my $hba_method = shift; + + unlink($node->data_dir . '/pg_hba.conf'); + $node->append_conf('pg_hba.conf', "local all all $hba_method"); + $node->reload; + return; +} + +# Require password from now on. +reset_pg_hba($node, 'scram-sha-256'); + + + +# Test access for a single role, useful to wrap all tests into one. +sub test_login +{ + local $Test::Builder::Level = $Test::Builder::Level + 1; + + my $node = shift; + my $role = shift; + my $password = shift; + my $expected_res = shift; + my $add_serv_role = shift; + my $status_string = 'failed'; + + $status_string = 'success' if ($expected_res eq 0); + + my $connstr = "user=$role"; + my $testname = + "authentication $status_string for role $role with password $password"; + + $ENV{"PGPASSWORD"} = $password; + if ($add_serv_role eq 1) + { + $ENV{"PGSERVICEAUTHROLE"} = 'mdb_service_auth'; + } + + if ($expected_res eq 0) + { + $node->connect_ok($connstr, $testname); + } + else + { + # No checks of the error message, only the status code. + $node->connect_fails($connstr, $testname); + } +} + + +test_login($node, 'mdb_service_auth', "serv", 0, 0); +test_login($node, 'mdbsar_test_role', "serv", 2, 0); +test_login($node, 'mdbsar_test_role', "test", 0, 0); + +test_login($node, 'sup', "123", 0, 0); + +test_login($node, 'mdb_service_auth', "serv", 0, 1); +test_login($node, 'mdbsar_test_role', "serv", 0, 1); +test_login($node, 'mdbsar_test_role', "test", 2, 1); + +test_login($node, 'sup', "serv", 2, 1); + +ok(1); +done_testing(); + diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h index 3b2ce9908f8..8ae518aed86 100644 --- a/src/include/libpq/libpq-be.h +++ b/src/include/libpq/libpq-be.h @@ -165,6 +165,10 @@ typedef struct Port */ char *database_name; char *user_name; + /* + * MDB-23247: service role name to perform auth - passthrough + */ + char *service_auth_role; char *cmdline_options; List *guc_options; diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index c472cea1487..0cd8dc4d7a9 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -211,8 +211,11 @@ extern int aclmembers(const Acl *acl, Oid **roleids); extern bool has_privs_of_role(Oid member, Oid role); extern bool member_can_set_role(Oid member, Oid role); extern void check_can_set_role(Oid member, Oid role); -extern bool has_privs_of_role_strict(Oid member, Oid role); extern bool is_member_of_role(Oid member, Oid role); + +// -- mdb patch +extern bool has_privs_of_unwanted_system_role_prestartup(Oid role); +// -- mdb patch end extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); @@ -227,6 +230,7 @@ extern bool mdb_admin_is_member_of_role(Oid member, Oid role); extern Oid select_best_admin(Oid member, Oid role); extern Oid get_role_oid(const char *rolname, bool missing_ok); extern Oid get_role_oid_or_public(const char *rolname); +extern bool has_privs_of_unwanted_system_role(Oid role); extern Oid get_rolespec_oid(const RoleSpec *role, bool missing_ok); extern void check_rolespec_name(const RoleSpec *role, const char *detail_msg); extern HeapTuple get_rolespec_tuple(const RoleSpec *role); diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 87453b11ea2..fe3dfd8e12b 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -362,6 +362,12 @@ static const internalPQconninfoOption PQconninfoOptions[] = { "Load-Balance-Hosts", "", 8, /* sizeof("disable") = 8 */ offsetof(struct pg_conn, load_balance_hosts)}, + /* MDB-23247: option for service log-in */ + {"_pq_.service_auth_role", "PGSERVICEAUTHROLE", + "", NULL, + "_pg__service_auth_role", "", 20, + offsetof(struct pg_conn, service_auth_role)}, + /* Terminating entry --- MUST BE LAST */ {NULL, NULL, NULL, NULL, NULL, NULL, 0} @@ -4459,6 +4465,9 @@ freePGconn(PGconn *conn) free(conn->rowBuf); free(conn->target_session_attrs); free(conn->load_balance_hosts); + if (conn->service_auth_role) { + free(conn->service_auth_role); + } termPQExpBuffer(&conn->errorMessage); termPQExpBuffer(&conn->workBuffer); diff --git a/src/interfaces/libpq/fe-protocol3.c b/src/interfaces/libpq/fe-protocol3.c index c42a52e162e..1e6928f8561 100644 --- a/src/interfaces/libpq/fe-protocol3.c +++ b/src/interfaces/libpq/fe-protocol3.c @@ -2336,6 +2336,10 @@ build_startup_packet(const PGconn *conn, char *packet, } } + /* MDB-23247: add service auth role to startup options */ + if (conn->service_auth_role && conn->service_auth_role[0]) + ADD_STARTUP_OPTION("_pq_.service_auth_role", conn->service_auth_role); + /* Add trailing terminator */ if (packet) packet[packet_len] = '\0'; diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index 66880f39aa4..570fb8f563a 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -412,6 +412,8 @@ struct pg_conn char *require_auth; /* name of the expected auth method */ char *load_balance_hosts; /* load balance over hosts */ + char *service_auth_role; /* MDB-23247: option for service log-in */ + /* Optional file to write trace info to */ FILE *Pfdebug; int traceFlags; From a1aaf2aeaca61319d898f5473ae574256c6aa214 Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Wed, 5 Jul 2023 07:06:41 +0000 Subject: [PATCH 13/54] Add mdb changelog Update mdb-patched.md Update mdb-pacthes.md --- MDB-PATCHES.md | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 MDB-PATCHES.md diff --git a/MDB-PATCHES.md b/MDB-PATCHES.md new file mode 100644 index 00000000000..a7d1efcbefc --- /dev/null +++ b/MDB-PATCHES.md @@ -0,0 +1,90 @@ + +15: + +7dfdf3a55ed: mdb_replication role patch +f6a3406fed8: Disallow cancelation of syncronous commit V1 +4289037edd2: Extend multixact SLRU +d6842b7f65c: Allow mdb_admin to create LEAKPROOF functions +146a82b3a75: mdb admin sets session replication role +68312eae50b: [MDB-16648]: Allow mdb admin to kill specific superuser queries +ab8f5243195: provide [mdb -postgresql] restict grant roles in YC[MDB-16990] +8ccf1aa7c51: Allow mdb admin to tranfers ownership on non-superuser objects regressioon tests for mdb admin functionality[MDB-16988] +bf4cad5ecdd: MDB-17910: check MDB reserved application name fix +64fed445a14: MDB-16955 : disallow to kill repl mon in cloud +afeb1eb4c2f: Fix mdb_replication role +5852300fb1d: pg_replication_slot_advance fix +c52bd070b56: Fix compilation errors + +0db90bbcbdd: Demonstrate and fix lock of all SQL queries by pg_stat_statements +530019f966d: MDB-21297: forbit usage of COPY TO PROGRAMM and COPY FROM PROGRAMM to non-superuser +8c860bf4d66: Reimplement mdb-admin, refactor mdb_admin check and usages. + +3a89cc36c74: Implement mdb-locales patch +dc7d503498b: Add mdb locales patch, restore COPY from/to files, enable regress. +96c30d707a7: Role mdb_superuser: feature and regress testsing +2d5f40ce3c9: Refactor optional setlocale, fix minor issues +41f04495a89: Update dependencies: bump libmdblocales, add mdb-locales +adc0b21d39f: Allow mdb_superuser to have power of pg_database_owner +ac90e1819fa: MDB-23247: startup param for auth passthrough under unpriviledged user +2bf6f042542: Add tap-test for mdb service role auth 👍👌😉 +9750b4efc44: Use fadvise to prefetch WAL in xlogrecovery +25f12802528: Fix tests after rebasecontrib tests 💅️️💅️️💅️️ now works +746dd65f557: MDB-23247: debug ouput for testing purposes lowered to DEBUG5 elog level + + + + + +16: + +/* misc */ + +/* on branch mdb-16 cherry-picked 'as is' */ +f6a3406fed8 -> 1effb23478e: Disallow cancelation of syncronous commit V1 +4289037edd2 -> b542d608604: Extend multixact SLRU + + + +/* mdb - admin + mdb_replication */ +7dfdf3a55ed: mdb_replication role patch + + +d6842b7f65c: Allow mdb_admin to create LEAKPROOF functions +146a82b3a75: mdb admin sets session replication role +68312eae50b: [MDB-16648]: Allow mdb admin to kill specific superuser queries +8ccf1aa7c51: Allow mdb admin to tranfers ownership on non-superuser objects regressioon tests for mdb admin functionality[MDB-16988] +8c860bf4d66: Reimplement mdb-admin, refactor mdb_admin check and usages. + +/* sqashed to */ +52435055d7b: Mdb-admin patch and regression tests +/*******/ + +/* as is */ +ab8f5243195->3fecc85426e: provide [mdb -postgresql] restict grant roles in YC[MDB-16990] + +/* pack of mdb patches */ +bf4cad5ecdd: MDB-17910: check MDB reserved application name fix +64fed445a14: MDB-16955 : disallow to kill repl mon in cloud +afeb1eb4c2f: Fix mdb_replication role +5852300fb1d: pg_replication_slot_advance fix +c52bd070b56: Fix compilation errors + +/* squashed to */ +52ea09c2d90: Pack of MDB-related patches: +/* */ + +0db90bbcbdd: Demonstrate and fix lock of all SQL queries by pg_stat_statements +530019f966d: MDB-21297: forbit usage of COPY TO PROGRAMM and COPY FROM PROGRAMM to non-superuser + +3a89cc36c74: Implement mdb-locales patch +dc7d503498b: Add mdb locales patch, restore COPY from/to files, enable regress. +96c30d707a7: Role mdb_superuser: feature and regress testsing +2d5f40ce3c9: Refactor optional setlocale, fix minor issues +41f04495a89: Update dependencies: bump libmdblocales, add mdb-locales +adc0b21d39f: Allow mdb_superuser to have power of pg_database_owner +ac90e1819fa: MDB-23247: startup param for auth passthrough under unpriviledged user +2bf6f042542: Add tap-test for mdb service role auth 👍👌😉 +9750b4efc44: Use fadvise to prefetch WAL in xlogrecovery +25f12802528: Fix tests after rebasecontrib tests 💅️️💅️️💅️️ now works +746dd65f557: MDB-23247: debug ouput for testing purposes lowered to DEBUG5 elog level + From a0d76c798868af495f36a792ae8895cf2ea1ed66 Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Tue, 26 Dec 2023 14:43:49 +0300 Subject: [PATCH 14/54] Untrust all contrib --- contrib/bool_plperl/bool_plperl.control | 2 +- contrib/btree_gin/btree_gin.control | 2 +- contrib/btree_gist/btree_gist.control | 2 +- contrib/citext/citext.control | 2 +- contrib/cube/cube.control | 2 +- contrib/dict_int/dict_int.control | 2 +- contrib/fuzzystrmatch/fuzzystrmatch.control | 2 +- contrib/hstore/hstore.control | 2 +- contrib/intarray/intarray.control | 2 +- contrib/isn/isn.control | 2 +- contrib/jsonb_plperl/jsonb_plperl.control | 2 +- contrib/lo/lo.control | 2 +- contrib/ltree/ltree.control | 2 +- contrib/pg_trgm/pg_trgm.control | 2 +- contrib/pgcrypto/pgcrypto.control | 2 +- contrib/seg/seg.control | 2 +- contrib/tablefunc/tablefunc.control | 2 +- contrib/tcn/tcn.control | 2 +- contrib/tsm_system_rows/tsm_system_rows.control | 2 +- contrib/tsm_system_time/tsm_system_time.control | 2 +- contrib/unaccent/unaccent.control | 2 +- contrib/uuid-ossp/uuid-ossp.control | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/contrib/bool_plperl/bool_plperl.control b/contrib/bool_plperl/bool_plperl.control index af3e6b1966f..65f9755adad 100644 --- a/contrib/bool_plperl/bool_plperl.control +++ b/contrib/bool_plperl/bool_plperl.control @@ -3,5 +3,5 @@ comment = 'transform between bool and plperl' default_version = '1.0' module_pathname = '$libdir/bool_plperl' relocatable = true -trusted = true +trusted = false requires = 'plperl' diff --git a/contrib/btree_gin/btree_gin.control b/contrib/btree_gin/btree_gin.control index 67d0c997d8d..5fc20d2cd65 100644 --- a/contrib/btree_gin/btree_gin.control +++ b/contrib/btree_gin/btree_gin.control @@ -3,4 +3,4 @@ comment = 'support for indexing common datatypes in GIN' default_version = '1.3' module_pathname = '$libdir/btree_gin' relocatable = true -trusted = true +trusted = false diff --git a/contrib/btree_gist/btree_gist.control b/contrib/btree_gist/btree_gist.control index fa9171a80a2..b3467f6a071 100644 --- a/contrib/btree_gist/btree_gist.control +++ b/contrib/btree_gist/btree_gist.control @@ -3,4 +3,4 @@ comment = 'support for indexing common datatypes in GiST' default_version = '1.7' module_pathname = '$libdir/btree_gist' relocatable = true -trusted = true +trusted = false diff --git a/contrib/citext/citext.control b/contrib/citext/citext.control index ccf445475d0..0934dec1098 100644 --- a/contrib/citext/citext.control +++ b/contrib/citext/citext.control @@ -3,4 +3,4 @@ comment = 'data type for case-insensitive character strings' default_version = '1.6' module_pathname = '$libdir/citext' relocatable = true -trusted = true +trusted = false diff --git a/contrib/cube/cube.control b/contrib/cube/cube.control index 50427ec1170..cf21f9e5fd9 100644 --- a/contrib/cube/cube.control +++ b/contrib/cube/cube.control @@ -3,4 +3,4 @@ comment = 'data type for multidimensional cubes' default_version = '1.5' module_pathname = '$libdir/cube' relocatable = true -trusted = true +trusted = false diff --git a/contrib/dict_int/dict_int.control b/contrib/dict_int/dict_int.control index ec04ccea91a..1ca11400d41 100644 --- a/contrib/dict_int/dict_int.control +++ b/contrib/dict_int/dict_int.control @@ -3,4 +3,4 @@ comment = 'text search dictionary template for integers' default_version = '1.0' module_pathname = '$libdir/dict_int' relocatable = true -trusted = true +trusted = false diff --git a/contrib/fuzzystrmatch/fuzzystrmatch.control b/contrib/fuzzystrmatch/fuzzystrmatch.control index 8b6e9fd9935..e0ffadcbddb 100644 --- a/contrib/fuzzystrmatch/fuzzystrmatch.control +++ b/contrib/fuzzystrmatch/fuzzystrmatch.control @@ -3,4 +3,4 @@ comment = 'determine similarities and distance between strings' default_version = '1.2' module_pathname = '$libdir/fuzzystrmatch' relocatable = true -trusted = true +trusted = false diff --git a/contrib/hstore/hstore.control b/contrib/hstore/hstore.control index 89e3c746c46..248223b9558 100644 --- a/contrib/hstore/hstore.control +++ b/contrib/hstore/hstore.control @@ -3,4 +3,4 @@ comment = 'data type for storing sets of (key, value) pairs' default_version = '1.8' module_pathname = '$libdir/hstore' relocatable = true -trusted = true +trusted = false diff --git a/contrib/intarray/intarray.control b/contrib/intarray/intarray.control index c3ff753e2cf..e01110f2681 100644 --- a/contrib/intarray/intarray.control +++ b/contrib/intarray/intarray.control @@ -3,4 +3,4 @@ comment = 'functions, operators, and index support for 1-D arrays of integers' default_version = '1.5' module_pathname = '$libdir/_int' relocatable = true -trusted = true +trusted = false diff --git a/contrib/isn/isn.control b/contrib/isn/isn.control index 1cb5e2b2340..f816d9b0906 100644 --- a/contrib/isn/isn.control +++ b/contrib/isn/isn.control @@ -3,4 +3,4 @@ comment = 'data types for international product numbering standards' default_version = '1.2' module_pathname = '$libdir/isn' relocatable = true -trusted = true +trusted = false diff --git a/contrib/jsonb_plperl/jsonb_plperl.control b/contrib/jsonb_plperl/jsonb_plperl.control index 4acee93a2fc..b4adfad17fb 100644 --- a/contrib/jsonb_plperl/jsonb_plperl.control +++ b/contrib/jsonb_plperl/jsonb_plperl.control @@ -3,5 +3,5 @@ comment = 'transform between jsonb and plperl' default_version = '1.0' module_pathname = '$libdir/jsonb_plperl' relocatable = true -trusted = true +trusted = false requires = 'plperl' diff --git a/contrib/lo/lo.control b/contrib/lo/lo.control index f73f8b5fae5..7b3de9e111a 100644 --- a/contrib/lo/lo.control +++ b/contrib/lo/lo.control @@ -3,4 +3,4 @@ comment = 'Large Object maintenance' default_version = '1.1' module_pathname = '$libdir/lo' relocatable = true -trusted = true +trusted = false diff --git a/contrib/ltree/ltree.control b/contrib/ltree/ltree.control index b408d64781f..c6acac33636 100644 --- a/contrib/ltree/ltree.control +++ b/contrib/ltree/ltree.control @@ -3,4 +3,4 @@ comment = 'data type for hierarchical tree-like structures' default_version = '1.2' module_pathname = '$libdir/ltree' relocatable = true -trusted = true +trusted = false diff --git a/contrib/pg_trgm/pg_trgm.control b/contrib/pg_trgm/pg_trgm.control index 1d6a9ddf259..ba1db412f3a 100644 --- a/contrib/pg_trgm/pg_trgm.control +++ b/contrib/pg_trgm/pg_trgm.control @@ -3,4 +3,4 @@ comment = 'text similarity measurement and index searching based on trigrams' default_version = '1.6' module_pathname = '$libdir/pg_trgm' relocatable = true -trusted = true +trusted = false diff --git a/contrib/pgcrypto/pgcrypto.control b/contrib/pgcrypto/pgcrypto.control index d2151d3bc4b..74841d0e7bd 100644 --- a/contrib/pgcrypto/pgcrypto.control +++ b/contrib/pgcrypto/pgcrypto.control @@ -3,4 +3,4 @@ comment = 'cryptographic functions' default_version = '1.3' module_pathname = '$libdir/pgcrypto' relocatable = true -trusted = true +trusted = false diff --git a/contrib/seg/seg.control b/contrib/seg/seg.control index e2c6a4750fc..0fa4236c079 100644 --- a/contrib/seg/seg.control +++ b/contrib/seg/seg.control @@ -3,4 +3,4 @@ comment = 'data type for representing line segments or floating-point intervals' default_version = '1.4' module_pathname = '$libdir/seg' relocatable = true -trusted = true +trusted = false diff --git a/contrib/tablefunc/tablefunc.control b/contrib/tablefunc/tablefunc.control index 7b25d161702..64d039977de 100644 --- a/contrib/tablefunc/tablefunc.control +++ b/contrib/tablefunc/tablefunc.control @@ -3,4 +3,4 @@ comment = 'functions that manipulate whole tables, including crosstab' default_version = '1.0' module_pathname = '$libdir/tablefunc' relocatable = true -trusted = true +trusted = false diff --git a/contrib/tcn/tcn.control b/contrib/tcn/tcn.control index 6972e1102e2..d54c22c9b49 100644 --- a/contrib/tcn/tcn.control +++ b/contrib/tcn/tcn.control @@ -3,4 +3,4 @@ comment = 'Triggered change notifications' default_version = '1.0' module_pathname = '$libdir/tcn' relocatable = true -trusted = true +trusted = false diff --git a/contrib/tsm_system_rows/tsm_system_rows.control b/contrib/tsm_system_rows/tsm_system_rows.control index b495fb126c0..ccc72d89058 100644 --- a/contrib/tsm_system_rows/tsm_system_rows.control +++ b/contrib/tsm_system_rows/tsm_system_rows.control @@ -3,4 +3,4 @@ comment = 'TABLESAMPLE method which accepts number of rows as a limit' default_version = '1.0' module_pathname = '$libdir/tsm_system_rows' relocatable = true -trusted = true +trusted = false diff --git a/contrib/tsm_system_time/tsm_system_time.control b/contrib/tsm_system_time/tsm_system_time.control index b1b9789debc..e0681787e33 100644 --- a/contrib/tsm_system_time/tsm_system_time.control +++ b/contrib/tsm_system_time/tsm_system_time.control @@ -3,4 +3,4 @@ comment = 'TABLESAMPLE method which accepts time in milliseconds as a limit' default_version = '1.0' module_pathname = '$libdir/tsm_system_time' relocatable = true -trusted = true +trusted = false diff --git a/contrib/unaccent/unaccent.control b/contrib/unaccent/unaccent.control index 649cf68a6e7..9cb1a520806 100644 --- a/contrib/unaccent/unaccent.control +++ b/contrib/unaccent/unaccent.control @@ -3,4 +3,4 @@ comment = 'text search dictionary that removes accents' default_version = '1.1' module_pathname = '$libdir/unaccent' relocatable = true -trusted = true +trusted = false diff --git a/contrib/uuid-ossp/uuid-ossp.control b/contrib/uuid-ossp/uuid-ossp.control index 142a99e4a89..8c2bd525b3d 100644 --- a/contrib/uuid-ossp/uuid-ossp.control +++ b/contrib/uuid-ossp/uuid-ossp.control @@ -3,4 +3,4 @@ comment = 'generate universally unique identifiers (UUIDs)' default_version = '1.1' module_pathname = '$libdir/uuid-ossp' relocatable = true -trusted = true +trusted = false From 9f99609a7ea5d494ad14cfe2183b83c1a303d9d5 Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Sun, 30 Jul 2023 21:58:04 +0000 Subject: [PATCH 15/54] Add debian for MDB 16 branch --- debian/changelog | 27 ++ debian/clean | 11 + debian/control | 292 ++++++++++++++++++ debian/copyright | 270 ++++++++++++++++ debian/gitlab-ci.yml | 1 + debian/libecpg-compat3.install | 1 + debian/libecpg-compat3.symbols | 44 +++ debian/libecpg-dev.install | 18 ++ debian/libecpg6.install | 2 + debian/libecpg6.symbols | 31 ++ debian/libpgtypes3.install | 1 + debian/libpgtypes3.symbols | 48 +++ debian/libpq-dev.dirs | 1 + debian/libpq-dev.install | 11 + debian/libpq5.install | 2 + debian/libpq5.symbols | 189 ++++++++++++ debian/patches/50-per-version-dirs.patch | 29 ++ .../patches/51-default-sockets-in-var.patch | 20 ++ debian/patches/52-tutorial-README.patch | 16 + .../53-pg_service.conf_directory_doc.patch | 19 ++ ...bian-alternatives-for-external-tools.patch | 28 ++ debian/patches/70-history | 13 + debian/patches/autoconf2.69 | 7 + debian/patches/extension_destdir | 270 ++++++++++++++++ debian/patches/filter-debug-prefix-map | 44 +++ debian/patches/focal-arm64-outline-atomics | 23 ++ debian/patches/hurd-iovec | 26 ++ debian/patches/jit-s390x | 96 ++++++ debian/patches/libpgport-pkglibdir | 84 +++++ debian/patches/series | 13 + debian/po/POTFILES.in | 1 + debian/po/de.po | 37 +++ debian/po/es.po | 58 ++++ debian/po/fr.po | 39 +++ debian/po/it.po | 37 +++ debian/po/nl.po | 40 +++ debian/po/pt.po | 39 +++ debian/po/pt_BR.po | 37 +++ debian/po/ro.po | 50 +++ debian/po/ru.po | 39 +++ debian/postgresql-16.install | 57 ++++ debian/postgresql-16.lintian-overrides | 20 ++ debian/postgresql-16.postinst | 13 + debian/postgresql-16.postrm | 80 +++++ debian/postgresql-16.preinst | 18 ++ debian/postgresql-16.prerm | 16 + debian/postgresql-16.templates | 7 + debian/postgresql-client-16.install | 45 +++ debian/postgresql-client-16.lintian-overrides | 3 + debian/postgresql-client-16.postinst | 13 + debian/postgresql-client-16.prerm | 12 + debian/postgresql-doc-16.doc-base | 18 ++ debian/postgresql-doc-16.install | 2 + debian/postgresql-doc-16.postinst | 30 ++ debian/postgresql-doc-16.prerm | 17 + debian/postgresql-plperl-16.install | 6 + debian/postgresql-plperl-16.lintian-overrides | 1 + debian/postgresql-plpython3-16.install | 6 + .../postgresql-plpython3-16.lintian-overrides | 1 + debian/postgresql-pltcl-16.install | 3 + debian/postgresql-pltcl-16.lintian-overrides | 1 + debian/postgresql-server-dev-16.install | 2 + ...postgresql-server-dev-16.lintian-overrides | 1 + debian/rules | 5 + debian/source/format | 1 + debian/source/lintian-overrides | 4 + debian/tests/Makefile.regress | 5 + debian/tests/control | 20 ++ debian/tests/installcheck | 41 +++ debian/tests/run-testsuite | 5 + debian/watch | 4 + 71 files changed, 2471 insertions(+) create mode 100644 debian/changelog create mode 100644 debian/clean create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/gitlab-ci.yml create mode 100644 debian/libecpg-compat3.install create mode 100644 debian/libecpg-compat3.symbols create mode 100644 debian/libecpg-dev.install create mode 100644 debian/libecpg6.install create mode 100644 debian/libecpg6.symbols create mode 100644 debian/libpgtypes3.install create mode 100644 debian/libpgtypes3.symbols create mode 100644 debian/libpq-dev.dirs create mode 100644 debian/libpq-dev.install create mode 100644 debian/libpq5.install create mode 100644 debian/libpq5.symbols create mode 100644 debian/patches/50-per-version-dirs.patch create mode 100644 debian/patches/51-default-sockets-in-var.patch create mode 100644 debian/patches/52-tutorial-README.patch create mode 100644 debian/patches/53-pg_service.conf_directory_doc.patch create mode 100644 debian/patches/54-debian-alternatives-for-external-tools.patch create mode 100644 debian/patches/70-history create mode 100644 debian/patches/autoconf2.69 create mode 100644 debian/patches/extension_destdir create mode 100644 debian/patches/filter-debug-prefix-map create mode 100644 debian/patches/focal-arm64-outline-atomics create mode 100644 debian/patches/hurd-iovec create mode 100644 debian/patches/jit-s390x create mode 100644 debian/patches/libpgport-pkglibdir create mode 100644 debian/patches/series create mode 100644 debian/po/POTFILES.in create mode 100644 debian/po/de.po create mode 100644 debian/po/es.po create mode 100644 debian/po/fr.po create mode 100644 debian/po/it.po create mode 100644 debian/po/nl.po create mode 100644 debian/po/pt.po create mode 100644 debian/po/pt_BR.po create mode 100644 debian/po/ro.po create mode 100644 debian/po/ru.po create mode 100755 debian/postgresql-16.install create mode 100644 debian/postgresql-16.lintian-overrides create mode 100644 debian/postgresql-16.postinst create mode 100644 debian/postgresql-16.postrm create mode 100644 debian/postgresql-16.preinst create mode 100644 debian/postgresql-16.prerm create mode 100644 debian/postgresql-16.templates create mode 100644 debian/postgresql-client-16.install create mode 100644 debian/postgresql-client-16.lintian-overrides create mode 100644 debian/postgresql-client-16.postinst create mode 100644 debian/postgresql-client-16.prerm create mode 100644 debian/postgresql-doc-16.doc-base create mode 100644 debian/postgresql-doc-16.install create mode 100644 debian/postgresql-doc-16.postinst create mode 100644 debian/postgresql-doc-16.prerm create mode 100755 debian/postgresql-plperl-16.install create mode 120000 debian/postgresql-plperl-16.lintian-overrides create mode 100755 debian/postgresql-plpython3-16.install create mode 120000 debian/postgresql-plpython3-16.lintian-overrides create mode 100644 debian/postgresql-pltcl-16.install create mode 120000 debian/postgresql-pltcl-16.lintian-overrides create mode 100644 debian/postgresql-server-dev-16.install create mode 120000 debian/postgresql-server-dev-16.lintian-overrides create mode 100755 debian/rules create mode 100644 debian/source/format create mode 100644 debian/source/lintian-overrides create mode 100644 debian/tests/Makefile.regress create mode 100644 debian/tests/control create mode 100755 debian/tests/installcheck create mode 100755 debian/tests/run-testsuite create mode 100644 debian/watch diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 00000000000..6b6a362a834 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,27 @@ +postgresql-16 (16~beta2-1.pgdg+1) sid-pgdg; urgency=medium + + * Rebuild for sid-pgdg. + * Changes applied by generate-pgdg-source: + + Moving binary packages to component 16. + + Enabling cassert. + + -- PostgreSQL on Debian and Ubuntu Wed, 28 Jun 2023 15:41:55 +0200 + +postgresql-16 (16~beta2-1) experimental; urgency=medium + + * New beta version. + + -- Christoph Berg Wed, 28 Jun 2023 15:41:55 +0200 + +postgresql-16 (16~beta1-2) experimental; urgency=medium + + * Bump postgresql-common B-D to 250 to ignore test failures on alpha et al. + * Define IOV_MAX on hurd-i386. + + -- Christoph Berg Wed, 24 May 2023 11:11:53 +0200 + +postgresql-16 (16~beta1-1) experimental; urgency=medium + + * New major upstream version 16; packaging based on postgresql-15. + + -- Christoph Berg Tue, 23 May 2023 14:05:19 +0200 diff --git a/debian/clean b/debian/clean new file mode 100644 index 00000000000..27de16e1fcf --- /dev/null +++ b/debian/clean @@ -0,0 +1,11 @@ +# force rebuild of errcodes.h and symlinks +src/backend/utils/errcodes.h +src/backend/utils/fmgr-stamp +src/backend/utils/fmgroids.h +src/backend/utils/fmgrprotos.h + +src/backend/snowball/snowball_create.sql + +# regression tests debris +src/test/regress/regress.o +src/test/regress/regress.so diff --git a/debian/control b/debian/control new file mode 100644 index 00000000000..f27bac537a3 --- /dev/null +++ b/debian/control @@ -0,0 +1,292 @@ +Source: postgresql-16 +Section: 16/database +Priority: optional +Maintainer: Debian PostgreSQL Maintainers +Uploaders: + Martin Pitt , + Peter Eisentraut , + Christoph Berg , +Standards-Version: 4.5.0 +Rules-Requires-Root: no +Build-Depends: + autoconf, + bison, + clang [!alpha !hppa !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32], + debhelper-compat (= 13), + dh-exec (>= 0.13~), + docbook-xml, + docbook-xsl (>= 1.77), + dpkg-dev (>= 1.16.1~), + flex, + gdb , + gettext, + libicu-dev, + libio-pty-perl , + libipc-run-perl , + libkrb5-dev, + libldap2-dev, + liblz4-dev, + libpam0g-dev | libpam-dev, + libperl-dev, + libreadline-dev, + libselinux1-dev [linux-any], + libssl-dev, + libsystemd-dev [linux-any], + libxml2-dev, + libxml2-utils, + libxslt1-dev, + libzstd-dev (>= 1.4.0) , + llvm-dev [!alpha !hppa !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32], + lz4 | liblz4-tool, + mawk, + perl (>= 5.8), + pkg-config, + postgresql-common (>= 250~), + python3-dev, + systemtap-sdt-dev, + tcl-dev, + uuid-dev, + xsltproc, + zlib1g-dev | libz-dev, + zstd (>= 1.4.0) , +Homepage: http://www.postgresql.org/ +Vcs-Browser: https://salsa.debian.org/postgresql/postgresql +Vcs-Git: https://salsa.debian.org/postgresql/postgresql.git -b 16 + +Package: libpq-dev +Architecture: any +Section: 16/libdevel +Depends: + libpq5 (= ${binary:Version}), + libssl-dev, + ${misc:Depends}, + ${shlibs:Depends}, +Suggests: + postgresql-doc-16, +Description: header files for libpq5 (PostgreSQL library) + Header files and static library for compiling C programs to link + with the libpq library in order to communicate with a PostgreSQL + database backend. + . + PostgreSQL is an object-relational SQL database management system. + +Package: libpq5 +Architecture: any +Section: 16/libs +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Pre-Depends: + ${misc:Pre-Depends}, +Multi-Arch: same +Description: PostgreSQL C client library + libpq is a C library that enables user programs to communicate with + the PostgreSQL database server. The server can be on another machine + and accessed through TCP/IP. This version of libpq is compatible + with servers from PostgreSQL 8.2 or later. + . + This package contains the run-time library, needed by packages using + libpq. + . + PostgreSQL is an object-relational SQL database management system. + +Package: libecpg6 +Architecture: any +Section: 16/libs +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Pre-Depends: + ${misc:Pre-Depends}, +Multi-Arch: same +Description: run-time library for ECPG programs + The libecpg shared library is used by programs built with ECPG + (Embedded PostgreSQL for C). + . + PostgreSQL is an object-relational SQL database management system. + +Package: libecpg-dev +Architecture: any +Section: 16/libdevel +Depends: + libecpg-compat3 (= ${binary:Version}), + libecpg6 (= ${binary:Version}), + libpgtypes3 (= ${binary:Version}), + libpq-dev, + ${misc:Depends}, + ${shlibs:Depends}, +Description: development files for ECPG (Embedded PostgreSQL for C) + This package contains the necessary files to build ECPG (Embedded + PostgreSQL for C) programs. It includes the development libraries + and the preprocessor program ecpg. + . + PostgreSQL is an object-relational SQL database management system. + . + Install this package if you want to write C programs with SQL statements + embedded in them (rather than run by an external process). + +Package: libecpg-compat3 +Architecture: any +Section: 16/libs +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Pre-Depends: + ${misc:Pre-Depends}, +Multi-Arch: same +Description: older version of run-time library for ECPG programs + The libecpg_compat shared library is used by programs built with ecpg. + (Embedded PostgreSQL for C). + . + PostgreSQL is an object-relational SQL database management system. + +Package: libpgtypes3 +Architecture: any +Section: 16/libs +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Pre-Depends: + ${misc:Pre-Depends}, +Multi-Arch: same +Description: shared library libpgtypes for PostgreSQL 16 + The libpgtypes shared library is used by programs built with ecpg. + (Embedded PostgreSQL for C). + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-16 +Architecture: any +Depends: + locales | locales-all, + postgresql-client-16, + postgresql-common (>= 248~), + ssl-cert, + tzdata, + ${misc:Depends}, + ${shlibs:Depends}, +Provides: + postgresql-contrib-16, + postgresql-16-jit-llvm (= ${llvm:Version}) [!alpha !hppa !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32], +Recommends: + sysstat, +Breaks: + dbconfig-common (<< 2.0.22~), +Description: The World's Most Advanced Open Source Relational Database + PostgreSQL, also known as Postgres, is a free and open-source relational + database management system (RDBMS) emphasizing extensibility and SQL + compliance. It features transactions with Atomicity, Consistency, Isolation, + Durability (ACID) properties, automatically updatable views, materialized + views, triggers, foreign keys, and stored procedures. It is designed to handle + a range of workloads, from single machines to data warehouses or Web services + with many concurrent users. + . + This package provides the database server for PostgreSQL 16.${cassert} +XB-Postgresql-Catversion: ${postgresql:Catversion} + +Package: postgresql-client-16 +Architecture: any +Multi-Arch: foreign +Depends: + libpq5 (>= ${source:Upstream-Version}), + postgresql-client-common (>= 182~), + sensible-utils, + ${misc:Depends}, + ${shlibs:Depends}, +Suggests: + postgresql-16, + postgresql-doc-16, +Provides: + postgresql-client, +Description: front-end programs for PostgreSQL 16 + This package contains client and administrative programs for + PostgreSQL: these are the interactive terminal client psql and + programs for creating and removing users and databases. + . + This is the client package for PostgreSQL 16. If you install + PostgreSQL 16 on a standalone machine, you need the server package + postgresql-16, too. On a network, you can install this package on + many client machines, while the server package may be installed on + only one machine. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-server-dev-16 +Architecture: any +Section: 16/libdevel +Depends: + clang-${llvm:Version} [!alpha !hppa !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32], + libpq-dev (>= 16~~), + llvm-${llvm:Version}-dev [!alpha !hppa !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32], + postgresql-client-16, + postgresql-common (>= 142~), + ${misc:Depends}, + ${shlibs:Depends}, +Description: development files for PostgreSQL 16 server-side programming + Header files for compiling SSI code to link into PostgreSQL's backend; for + example, for C functions to be called from SQL. + . + This package also contains the Makefiles necessary for building add-on + modules of PostgreSQL, which would otherwise have to be built in the + PostgreSQL source-code tree. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-doc-16 +Architecture: all +Build-Profiles: +Multi-Arch: foreign +Section: 16/doc +Depends: + ${misc:Depends}, +Description: documentation for the PostgreSQL database management system + This package contains all README files, user manual, and examples for + PostgreSQL 16. The manual is in HTML format. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-plperl-16 +Architecture: any +Depends: + perl, + postgresql-16 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends}, +Provides: + postgresql-plperl, +Description: PL/Perl procedural language for PostgreSQL 16 + PL/Perl enables an SQL developer to write procedural language functions + for PostgreSQL 16 in Perl. You need this package if you have any + PostgreSQL 16 functions that use the languages plperl or plperlu. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-plpython3-16 +Architecture: any +Depends: + postgresql-16 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends}, +Provides: + postgresql-plpython3, +Description: PL/Python 3 procedural language for PostgreSQL 16 + PL/Python 3 enables an SQL developer to write procedural language functions + for PostgreSQL 16 in Python 3. You need this package if you have any + PostgreSQL 16 functions that use the languages plpython3 or plpython3u. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-pltcl-16 +Architecture: any +Depends: + postgresql-16 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends}, +Provides: + postgresql-pltcl, +Description: PL/Tcl procedural language for PostgreSQL 16 + PL/Tcl enables an SQL developer to write procedural language functions + for PostgreSQL 16 in Tcl. You need this package if you have any + PostgreSQL 16 functions that use the languages pltcl or pltclu. + . + PostgreSQL is an object-relational SQL database management system. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 00000000000..df27d1ff61e --- /dev/null +++ b/debian/copyright @@ -0,0 +1,270 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: PostgreSQL +Source: https://www.postgresql.org/ftp/source/ + +### PostgreSQL copyrights ### + +Files: * +Copyright: Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + Portions Copyright (c) 1994, The Regents of the University of California +License: PostgreSQL + +Files: src/backend/regex/* +Copyright: + Copyright (c) 2013-2022, PostgreSQL Global Development Group + Copyright (c) 1998, 1999 Henry Spencer. All rights reserved. + Copyright (c) 1998 Sun Microsystems, Inc. + Copyright (c) 1998, 1999 by Scriptics Corporation. +License: PostgreSQL and Custom-regex and Tcl + +Files: src/bin/pg_dump/* +Copyright: Portions Copyright (c) 2000, Philip Warner +License: Custom-pg_dump + Rights are granted to use this software in any way so long as this notice is + not removed. The author is not responsible for loss or damages that may + result from its use. + +Files: src/backend/snowball/libstemmer/* src/include/snowball/libstemmer/* +Copyright: Copyright (c) 2001, Dr Martin Porter, + Copyright (c) 2002, Richard Boulton. +License: BSD-3-Clause +Comment: + This module uses the word stemming code developed by the Snowball project, + http://snowballstem.org which is released by them under a BSD-style license. + Postgres' files under src/backend/snowball/libstemmer/ and + src/include/snowball/libstemmer/ are taken directly from the Snowball files, + with only some minor adjustments of file inclusions. + See src/backend/snowball/README for details. + +Files: src/backend/utils/mb/Unicode/*.txt +Copyright: Copyright (C) 2001 earthian@tama.or.jp, All Rights Reserved. + Copyright (C) 2001 I'O, All Rights Reserved. + Copyright (C) 2006 Project X0213, All Rights Reserved. +License: Custom-Unicode + You can use, modify, distribute this table freely. + +Files: + src/common/sha2.c + src/include/common/sha2.h +Copyright: Copyright (c) 2000-2001, Aaron D. Gifford +License: BSD-3-Clause + +### contrib copyrights ### + +Files: contrib/fuzzystrmatch/dmetaphone.c +Copyright: Copyright 2000, Maurice Aubrey + Copyright 2003, North Carolina State Highway Patrol +License: double-metaphone + This module is free software; you may redistribute it and/or + modify it under the same terms as Perl itself. + . + All rights reserved. + . + Permission to use, copy, modify, and distribute this software and its + documentation for any purpose, without fee, and without a written agreement + is hereby granted, provided that the above copyright notice and this + paragraph and the following two paragraphs appear in all copies. + . + IN NO EVENT SHALL THE NORTH CAROLINA STATE HIGHWAY PATROL BE LIABLE TO ANY + PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, + INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS + DOCUMENTATION, EVEN IF THE NORTH CAROLINA STATE HIGHWAY PATROL HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + . + THE NORTH CAROLINA STATE HIGHWAY PATROL SPECIFICALLY DISCLAIMS ANY + WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED + HEREUNDER IS ON AN "AS IS" BASIS, AND THE NORTH CAROLINA STATE HIGHWAY PATROL + HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR + MODIFICATIONS. + . + The license of Perl is: + . + This program is free software; you can redistribute it and/or modify + it under the terms of either: + . + a) the GNU General Public License as published by the Free Software + Foundation; either version 1, or (at your option) any later + version, or + . + b) the "Artistic License" which comes with Perl. + . + On Debian GNU/Linux systems, the complete text of the GNU General + Public License version 1 can be found in + `/usr/share/common-licenses/GPL-1' and the Artistic Licence in + `/usr/share/common-licenses/Artistic'. + +Files: + contrib/pageinspect/btreefuncs.c + contrib/pgrowlocks/* + contrib/pgstattuple/* +Copyright: Copyright (c) 2006 Satoshi Nagayasu + Copyright (c) 2001, 2002, 2005-2006 Tatsuo Ishii +License: nagaysau-ishii + Permission to use, copy, modify, and distribute this software and + its documentation for any purpose, without fee, and without a + written agreement is hereby granted, provided that the above + copyright notice and this paragraph and the following two + paragraphs appear in all copies. + . + IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, + INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS + DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED + OF THE POSSIBILITY OF SUCH DAMAGE. + . + THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS + IS" BASIS, AND THE AUTHOR HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, + SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +Files: + contrib/pgcrypto/crypt-des.c +Copyright: Copyright (c) 1994 David Burren + Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. +License: BSD-3-clause + +Files: + contrib/pgcrypto/mbuf.* + contrib/pgcrypto/openssl.c + contrib/pgcrypto/pgcrypto.* + contrib/pgcrypto/pgp* + contrib/pgcrypto/px* +Copyright: Copyright (c) 2001, 2005 Marko Kreen +License: BSD-2-clause + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + . + THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +### licenses ### + +License: PostgreSQL + Permission to use, copy, modify, and distribute this software and its + documentation for any purpose, without fee, and without a written agreement + is hereby granted, provided that the above copyright notice and this + paragraph and the following two paragraphs appear in all copies. + . + IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR + DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS + DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + . + THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS + ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO + PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +License: BSD-3-Clause + All rights reserved. + . + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + . + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + . + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + . + 3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + . + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License: Custom-regex + Development of this software was funded, in part, by Cray Research Inc., + UUNET Communications Services Inc., Sun Microsystems Inc., and Scriptics + Corporation, none of whom are responsible for the results. The author + thanks all of them. + . + Redistribution and use in source and binary forms -- with or without + modification -- are permitted for any purpose, provided that + redistributions in source form retain this entire copyright notice and + indicate the origin and nature of any modifications. + . + I'd appreciate being given credit for this package in the documentation + of software which uses it, but that is not a requirement. + . + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + HENRY SPENCER BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License: Tcl + This software is copyrighted by the Regents of the University of + California, Sun Microsystems, Inc., Scriptics Corporation, ActiveState + Corporation and other parties. The following terms apply to all files + associated with the software unless explicitly disclaimed in + individual files. + . + The authors hereby grant permission to use, copy, modify, distribute, + and license this software and its documentation for any purpose, provided + that existing copyright notices are retained in all copies and that this + notice is included verbatim in any distributions. No written agreement, + license, or royalty fee is required for any of the authorized uses. + Modifications to this software may be copyrighted by their authors + and need not follow the licensing terms described here, provided that + the new terms are clearly indicated on the first page of each file where + they apply. + . + IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY + FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES + ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY + DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + . + THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. THIS SOFTWARE + IS PROVIDED ON AN "AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE + NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR + MODIFICATIONS. + . + GOVERNMENT USE: If you are acquiring this software on behalf of the + U.S. government, the Government shall have only "Restricted Rights" + in the software and related documentation as defined in the Federal + Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you + are acquiring the software on behalf of the Department of Defense, the + software shall be classified as "Commercial Computer Software" and the + Government shall have only "Restricted Rights" as defined in Clause + 252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the + authors grant the U.S. Government and others acting in its behalf + permission to use and distribute the software in accordance with the + terms specified in this license. diff --git a/debian/gitlab-ci.yml b/debian/gitlab-ci.yml new file mode 100644 index 00000000000..67e4816bd45 --- /dev/null +++ b/debian/gitlab-ci.yml @@ -0,0 +1 @@ +include: https://salsa.debian.org/postgresql/postgresql-common/raw/master/gitlab/gitlab-ci.yml diff --git a/debian/libecpg-compat3.install b/debian/libecpg-compat3.install new file mode 100644 index 00000000000..c69d50d8467 --- /dev/null +++ b/debian/libecpg-compat3.install @@ -0,0 +1 @@ +usr/lib/*/libecpg_compat.so.3* diff --git a/debian/libecpg-compat3.symbols b/debian/libecpg-compat3.symbols new file mode 100644 index 00000000000..3bd41e03e15 --- /dev/null +++ b/debian/libecpg-compat3.symbols @@ -0,0 +1,44 @@ +libecpg_compat.so.3 libecpg-compat3 #MINVER# +* Build-Depends-Package: libecpg-dev + ECPG_informix_get_var@Base 0 + ECPG_informix_reset_sqlca@Base 9.0~ + ECPG_informix_set_var@Base 0 + byleng@Base 0 + decadd@Base 0 + deccmp@Base 0 + deccopy@Base 0 + deccvasc@Base 0 + deccvdbl@Base 0 + deccvint@Base 0 + deccvlong@Base 0 + decdiv@Base 0 + decmul@Base 0 + decsub@Base 0 + dectoasc@Base 0 + dectodbl@Base 0 + dectoint@Base 0 + dectolong@Base 0 + dtcurrent@Base 0 + dtcvasc@Base 0 + dtcvfmtasc@Base 0 + dtsub@Base 0 + dttoasc@Base 0 + dttofmtasc@Base 0 + intoasc@Base 0 + ldchar@Base 0 + rdatestr@Base 0 + rdayofweek@Base 0 + rdefmtdate@Base 0 + rfmtdate@Base 0 + rfmtlong@Base 0 + rgetmsg@Base 0 + risnull@Base 0 + rjulmdy@Base 0 + rmdyjul@Base 0 + rsetnull@Base 0 + rstrdate@Base 0 + rtoday@Base 0 + rtypalign@Base 0 + rtypmsize@Base 0 + rtypwidth@Base 0 + rupshift@Base 0 diff --git a/debian/libecpg-dev.install b/debian/libecpg-dev.install new file mode 100644 index 00000000000..1935a0d885a --- /dev/null +++ b/debian/libecpg-dev.install @@ -0,0 +1,18 @@ +usr/include/postgresql/ecpg*.h +usr/include/postgresql/informix/* +usr/include/postgresql/pgtypes*.h +usr/include/postgresql/sql3types.h +usr/include/postgresql/sqlca.h +usr/include/postgresql/sqlda*.h +usr/lib/*/libecpg.a +usr/lib/*/libecpg.so +usr/lib/*/libecpg_compat.a +usr/lib/*/libecpg_compat.so +usr/lib/*/libpgtypes.a +usr/lib/*/libpgtypes.so +usr/lib/*/pkgconfig/libecpg.pc +usr/lib/*/pkgconfig/libecpg_compat.pc +usr/lib/*/pkgconfig/libpgtypes.pc +usr/lib/postgresql/*/bin/ecpg usr/bin +usr/share/locale/*/*/ecpg-*.mo +usr/share/postgresql/*/man/man1/ecpg.1* /usr/share/man/man1 diff --git a/debian/libecpg6.install b/debian/libecpg6.install new file mode 100644 index 00000000000..8ec1cee0f03 --- /dev/null +++ b/debian/libecpg6.install @@ -0,0 +1,2 @@ +usr/lib/*/libecpg.so.6* +usr/share/locale/*/*/ecpglib*.mo diff --git a/debian/libecpg6.symbols b/debian/libecpg6.symbols new file mode 100644 index 00000000000..d20881adac9 --- /dev/null +++ b/debian/libecpg6.symbols @@ -0,0 +1,31 @@ +libecpg.so.6 libecpg6 #MINVER# +* Build-Depends-Package: libecpg-dev + ECPGallocate_desc@Base 0 + ECPGconnect@Base 0 + ECPGdeallocate@Base 0 + ECPGdeallocate_all@Base 0 + ECPGdeallocate_desc@Base 0 + ECPGdebug@Base 0 + ECPGdescribe@Base 0 + ECPGdisconnect@Base 0 + ECPGdo@Base 0 + ECPGdo_descriptor@Base 0 + ECPGfree_auto_mem@Base 0 + ECPGget_PGconn@Base 8.3.1-2~ + ECPGget_desc@Base 0 + ECPGget_desc_header@Base 0 + ECPGget_sqlca@Base 0 + ECPGget_var@Base 9.0~ + ECPGis_noind_null@Base 0 + ECPGprepare@Base 0 + ECPGprepared_statement@Base 0 + ECPGset_desc@Base 0 + ECPGset_desc_header@Base 0 + ECPGset_noind_null@Base 0 + ECPGset_var@Base 9.0~ + ECPGsetcommit@Base 0 + ECPGsetconn@Base 0 + ECPGstatus@Base 0 + ECPGtrans@Base 0 + ECPGtransactionStatus@Base 9.0~ + sqlprint@Base 0 diff --git a/debian/libpgtypes3.install b/debian/libpgtypes3.install new file mode 100644 index 00000000000..decf90ae6bc --- /dev/null +++ b/debian/libpgtypes3.install @@ -0,0 +1 @@ +usr/lib/*/libpgtypes.so.3* diff --git a/debian/libpgtypes3.symbols b/debian/libpgtypes3.symbols new file mode 100644 index 00000000000..a398be3399a --- /dev/null +++ b/debian/libpgtypes3.symbols @@ -0,0 +1,48 @@ +libpgtypes.so.3 libpgtypes3 #MINVER# +* Build-Depends-Package: libecpg-dev + PGTYPESchar_free@Base 11~beta2 + PGTYPESdate_dayofweek@Base 0 + PGTYPESdate_defmt_asc@Base 0 + PGTYPESdate_fmt_asc@Base 0 + PGTYPESdate_free@Base 0 + PGTYPESdate_from_asc@Base 0 + PGTYPESdate_from_timestamp@Base 0 + PGTYPESdate_julmdy@Base 0 + PGTYPESdate_mdyjul@Base 0 + PGTYPESdate_new@Base 0 + PGTYPESdate_to_asc@Base 0 + PGTYPESdate_today@Base 0 + PGTYPESdecimal_free@Base 0 + PGTYPESdecimal_new@Base 0 + PGTYPESinterval_copy@Base 0 + PGTYPESinterval_free@Base 0 + PGTYPESinterval_from_asc@Base 0 + PGTYPESinterval_new@Base 0 + PGTYPESinterval_to_asc@Base 0 + PGTYPESnumeric_add@Base 0 + PGTYPESnumeric_cmp@Base 0 + PGTYPESnumeric_copy@Base 0 + PGTYPESnumeric_div@Base 0 + PGTYPESnumeric_free@Base 0 + PGTYPESnumeric_from_asc@Base 0 + PGTYPESnumeric_from_decimal@Base 0 + PGTYPESnumeric_from_double@Base 0 + PGTYPESnumeric_from_int@Base 0 + PGTYPESnumeric_from_long@Base 0 + PGTYPESnumeric_mul@Base 0 + PGTYPESnumeric_new@Base 0 + PGTYPESnumeric_sub@Base 0 + PGTYPESnumeric_to_asc@Base 0 + PGTYPESnumeric_to_decimal@Base 0 + PGTYPESnumeric_to_double@Base 0 + PGTYPESnumeric_to_int@Base 0 + PGTYPESnumeric_to_long@Base 0 + PGTYPEStimestamp_add_interval@Base 0 + PGTYPEStimestamp_current@Base 0 + PGTYPEStimestamp_defmt_asc@Base 0 + PGTYPEStimestamp_defmt_scan@Base 0 + PGTYPEStimestamp_fmt_asc@Base 0 + PGTYPEStimestamp_from_asc@Base 0 + PGTYPEStimestamp_sub@Base 0 + PGTYPEStimestamp_sub_interval@Base 0 + PGTYPEStimestamp_to_asc@Base 0 diff --git a/debian/libpq-dev.dirs b/debian/libpq-dev.dirs new file mode 100644 index 00000000000..e7724817552 --- /dev/null +++ b/debian/libpq-dev.dirs @@ -0,0 +1 @@ +usr/bin diff --git a/debian/libpq-dev.install b/debian/libpq-dev.install new file mode 100644 index 00000000000..734cb6f3189 --- /dev/null +++ b/debian/libpq-dev.install @@ -0,0 +1,11 @@ +usr/include/postgresql/internal/* +usr/include/postgresql/libpq-events.h +usr/include/postgresql/libpq-fe.h +usr/include/postgresql/libpq/libpq-fs.h +usr/include/postgresql/pg_config*.h +usr/include/postgresql/postgres_ext.h +usr/lib/*/libpq.a +usr/lib/*/libpq.so +usr/lib/*/pkgconfig/libpq.pc +# pg_config manpage for both the perl and C versions +usr/share/postgresql/*/man/man1/pg_config.1* /usr/share/man/man1 diff --git a/debian/libpq5.install b/debian/libpq5.install new file mode 100644 index 00000000000..c9811d7d9f1 --- /dev/null +++ b/debian/libpq5.install @@ -0,0 +1,2 @@ +usr/lib/*/libpq.so.5* +usr/share/locale/*/LC_MESSAGES/libpq*.mo diff --git a/debian/libpq5.symbols b/debian/libpq5.symbols new file mode 100644 index 00000000000..0490fcb4bbe --- /dev/null +++ b/debian/libpq5.symbols @@ -0,0 +1,189 @@ +libpq.so.5 libpq5 #MINVER# +* Build-Depends-Package: libpq-dev + PQbackendPID@Base 0 + PQbinaryTuples@Base 0 + PQcancel@Base 0 + PQclear@Base 0 + PQclientEncoding@Base 0 + PQcmdStatus@Base 0 + PQcmdTuples@Base 0 + PQconndefaults@Base 0 + PQconnectPoll@Base 0 + PQconnectStart@Base 0 + PQconnectStartParams@Base 9.0~ + PQconnectdb@Base 0 + PQconnectdbParams@Base 9.0~ + PQconnectionNeedsPassword@Base 8.3~rc1-1~ + PQconnectionUsedGSSAPI@Base 16~~ + PQconnectionUsedPassword@Base 8.3~ + PQconninfo@Base 9.3~ + PQconninfoFree@Base 0 + PQconninfoParse@Base 8.4~ + PQconsumeInput@Base 0 + PQcopyResult@Base 8.4~ + PQdb@Base 0 + PQdefaultSSLKeyPassHook_OpenSSL@Base 13~~ + PQdescribePortal@Base 0 + PQdescribePrepared@Base 0 + PQdisplayTuples@Base 0 + PQdsplen@Base 0 + PQencryptPassword@Base 0 + PQencryptPasswordConn@Base 10~~ + PQendcopy@Base 0 + PQenterPipelineMode@Base 14~~ + PQenv2encoding@Base 0 + PQerrorMessage@Base 0 + PQescapeBytea@Base 0 + PQescapeByteaConn@Base 0 + PQescapeIdentifier@Base 9.0~ + PQescapeLiteral@Base 9.0~ + PQescapeString@Base 0 + PQescapeStringConn@Base 0 + PQexec@Base 0 + PQexecParams@Base 0 + PQexecPrepared@Base 0 + PQexitPipelineMode@Base 14~~ + PQfformat@Base 0 + PQfinish@Base 0 + PQfireResultCreateEvents@Base 8.4~ + PQflush@Base 0 + PQfmod@Base 0 + PQfn@Base 0 + PQfname@Base 0 + PQfnumber@Base 0 + PQfreeCancel@Base 0 + PQfreeNotify@Base 0 + PQfreemem@Base 0 + PQfsize@Base 0 + PQftable@Base 0 + PQftablecol@Base 0 + PQftype@Base 0 + PQgetCancel@Base 0 + PQgetCopyData@Base 0 + PQgetResult@Base 0 + PQgetSSLKeyPassHook_OpenSSL@Base 13~~ + PQgetgssctx@Base 12~~ + PQgetisnull@Base 0 + PQgetlength@Base 0 + PQgetline@Base 0 + PQgetlineAsync@Base 0 + PQgetssl@Base 0 + PQgetvalue@Base 0 + PQgssEncInUse@Base 12~~ + PQhost@Base 0 + PQhostaddr@Base 12~~ + PQinitOpenSSL@Base 8.4~ + PQinitSSL@Base 0 + PQinstanceData@Base 8.4~ + PQisBusy@Base 0 + PQisnonblocking@Base 0 + PQisthreadsafe@Base 0 + PQlibVersion@Base 9.1~ + PQmakeEmptyPGresult@Base 0 + PQmblen@Base 0 + PQmblenBounded@Base 14~beta2 + PQnfields@Base 0 + PQnotifies@Base 0 + PQnparams@Base 0 + PQntuples@Base 0 + PQoidStatus@Base 0 + PQoidValue@Base 0 + PQoptions@Base 0 + PQparameterStatus@Base 0 + PQparamtype@Base 0 + PQpass@Base 0 + PQping@Base 9.1~ + PQpingParams@Base 9.1~ + PQpipelineStatus@Base 14~~ + PQpipelineSync@Base 14~~ + PQport@Base 0 + PQprepare@Base 0 + PQprint@Base 0 + PQprintTuples@Base 0 + PQprotocolVersion@Base 0 + PQputCopyData@Base 0 + PQputCopyEnd@Base 0 + PQputline@Base 0 + PQputnbytes@Base 0 + PQregisterEventProc@Base 8.4~ + PQregisterThreadLock@Base 0 + PQrequestCancel@Base 0 + PQresStatus@Base 0 + PQreset@Base 0 + PQresetPoll@Base 0 + PQresetStart@Base 0 + PQresultAlloc@Base 8.4~ + PQresultErrorField@Base 0 + PQresultErrorMessage@Base 0 + PQresultInstanceData@Base 8.4~ + PQresultMemorySize@Base 12~~ + PQresultSetInstanceData@Base 8.4~ + PQresultStatus@Base 0 + PQresultVerboseErrorMessage@Base 9.6~~ + PQsendDescribePortal@Base 0 + PQsendDescribePrepared@Base 0 + PQsendFlushRequest@Base 15~~ + PQsendPrepare@Base 0 + PQsendQuery@Base 0 + PQsendQueryParams@Base 0 + PQsendQueryPrepared@Base 0 + PQserverVersion@Base 0 + PQsetClientEncoding@Base 0 + PQsetErrorContextVisibility@Base 9.6~~ + PQsetErrorVerbosity@Base 0 + PQsetInstanceData@Base 8.4~ + PQsetNoticeProcessor@Base 0 + PQsetNoticeReceiver@Base 0 + PQsetResultAttrs@Base 8.4~ + PQsetSSLKeyPassHook_OpenSSL@Base 13~~ + PQsetSingleRowMode@Base 9.2~beta3 + PQsetTraceFlags@Base 14~beta2 + PQsetdbLogin@Base 0 + PQsetnonblocking@Base 0 + PQsetvalue@Base 8.4~ + PQsocket@Base 0 + PQsslAttribute@Base 9.5~~ + PQsslAttributeNames@Base 9.5~~ + PQsslInUse@Base 9.5~~ + PQsslStruct@Base 9.5~~ + PQstatus@Base 0 + PQtrace@Base 0 + PQtransactionStatus@Base 0 + PQtty@Base 0 + PQunescapeBytea@Base 0 + PQuntrace@Base 0 + PQuser@Base 0 + appendBinaryPQExpBuffer@Base 0 + appendPQExpBuffer@Base 0 + appendPQExpBufferChar@Base 0 + appendPQExpBufferStr@Base 0 + createPQExpBuffer@Base 0 + destroyPQExpBuffer@Base 0 + enlargePQExpBuffer@Base 0 + initPQExpBuffer@Base 0 + lo_close@Base 0 + lo_creat@Base 0 + lo_create@Base 0 + lo_export@Base 0 + lo_import@Base 0 + lo_import_with_oid@Base 8.4~ + lo_lseek64@Base 9.3~ + lo_lseek@Base 0 + lo_open@Base 0 + lo_read@Base 0 + lo_tell64@Base 9.3~ + lo_tell@Base 0 + lo_truncate64@Base 9.3~ + lo_truncate@Base 8.3~ + lo_unlink@Base 0 + lo_write@Base 0 + pg_char_to_encoding@Base 0 + pg_encoding_to_char@Base 0 + pg_utf_mblen@Base 0 + pg_valid_server_encoding@Base 0 + pg_valid_server_encoding_id@Base 8.3~beta1-2~ + pgresStatus@Base 0 + pqsignal@Base 0 + printfPQExpBuffer@Base 0 + resetPQExpBuffer@Base 0 + termPQExpBuffer@Base 0 diff --git a/debian/patches/50-per-version-dirs.patch b/debian/patches/50-per-version-dirs.patch new file mode 100644 index 00000000000..8277cdb127b --- /dev/null +++ b/debian/patches/50-per-version-dirs.patch @@ -0,0 +1,29 @@ +Author: Martin Pitt +Description: Use version specific installation directories so that several major versions can be installed in parallel. +Forwarded: No, Debian specific packaging with postgresql-common + + * Install lib files into /usr/lib/postgresql//lib/ + * Install server related header files into /usr/include/postgresql//server/ + +Bug-Debian: http://bugs.debian.org/462037 + +--- a/src/Makefile.global.in ++++ b/src/Makefile.global.in +@@ -119,7 +119,7 @@ libdir := @libdir@ + pkglibdir = $(libdir) + ifeq "$(findstring pgsql, $(pkglibdir))" "" + ifeq "$(findstring postgres, $(pkglibdir))" "" +-override pkglibdir := $(pkglibdir)/postgresql ++override pkglibdir := /usr/lib/postgresql/@PG_MAJORVERSION@/lib + endif + endif + +@@ -167,7 +167,7 @@ endif # PGXS + + # These derived path variables aren't separately configurable. + +-includedir_server = $(pkgincludedir)/server ++includedir_server = $(pkgincludedir)/@PG_MAJORVERSION@/server + includedir_internal = $(pkgincludedir)/internal + pgxsdir = $(pkglibdir)/pgxs + bitcodedir = $(pkglibdir)/bitcode diff --git a/debian/patches/51-default-sockets-in-var.patch b/debian/patches/51-default-sockets-in-var.patch new file mode 100644 index 00000000000..9da49b833a1 --- /dev/null +++ b/debian/patches/51-default-sockets-in-var.patch @@ -0,0 +1,20 @@ +Author: Martin Pitt +Description: Put server Unix sockets into /var/run/postgresql/ by default +Forwarded: No, Debian specific configuration with postgresql-common + +Using /tmp for sockets allows everyone to spoof a PostgreSQL server. Thus use +/var/run/postgresql/ for "system" clusters which run as 'postgres' (user +clusters will still use /tmp). Since system cluster are by far the common case, +set it as default. + +--- a/src/include/pg_config_manual.h ++++ b/src/include/pg_config_manual.h +@@ -206,7 +206,7 @@ + * support them yet. + */ + #ifndef WIN32 +-#define DEFAULT_PGSOCKET_DIR "/tmp" ++#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql" + #else + #define DEFAULT_PGSOCKET_DIR "" + #endif diff --git a/debian/patches/52-tutorial-README.patch b/debian/patches/52-tutorial-README.patch new file mode 100644 index 00000000000..9eb3263c2b3 --- /dev/null +++ b/debian/patches/52-tutorial-README.patch @@ -0,0 +1,16 @@ +Author: Martin Pitt +Description: Update tutorial README for required build dependencies. +Forwarded: No, Debian specific + +--- a/src/tutorial/README ++++ b/src/tutorial/README +@@ -6,8 +6,7 @@ tutorial + This directory contains SQL tutorial scripts. To look at them, first do a + % make + to compile all the scripts and C files for the user-defined functions +-and types. (make needs to be GNU make --- it may be named something +-different on your system, often 'gmake') ++and types. This requires a postgresql-server-dev-* package to be installed. + + Then, run psql with the -s (single-step) flag: + % psql -s diff --git a/debian/patches/53-pg_service.conf_directory_doc.patch b/debian/patches/53-pg_service.conf_directory_doc.patch new file mode 100644 index 00000000000..584b41c01f8 --- /dev/null +++ b/debian/patches/53-pg_service.conf_directory_doc.patch @@ -0,0 +1,19 @@ +Author: Martin Pitt +Description: Update pg_service.conf example to tell the Debian specific file location. +Forwarded: No, Debian specific + +Index: postgresql-9.2-9.2~beta1/src/interfaces/libpq/pg_service.conf.sample +=================================================================== +--- postgresql-9.2-9.2~beta1.orig/src/interfaces/libpq/pg_service.conf.sample 2011-04-27 23:17:22.000000000 +0200 ++++ postgresql-9.2-9.2~beta1/src/interfaces/libpq/pg_service.conf.sample 2011-05-10 11:25:42.151949794 +0200 +@@ -8,8 +8,8 @@ + # to look up such parameters. A sample configuration for postgres is + # included in this file. Lines beginning with '#' are comments. + # +-# Copy this to your sysconf directory (typically /usr/local/pgsql/etc) and +-# rename it pg_service.conf. ++# Copy this to /etc/postgresql-common/ (or select its location with the ++# PGSYSCONFDIR environment variable) and rename it pg_service.conf. + # + # + #[postgres] diff --git a/debian/patches/54-debian-alternatives-for-external-tools.patch b/debian/patches/54-debian-alternatives-for-external-tools.patch new file mode 100644 index 00000000000..0031989718d --- /dev/null +++ b/debian/patches/54-debian-alternatives-for-external-tools.patch @@ -0,0 +1,28 @@ +Author: Martin Pitt +Description: Use Debian alternatives for external tools instead of hardcoded programs +Forwarded: No, Debian specific + +--- a/src/bin/psql/settings.h ++++ b/src/bin/psql/settings.h +@@ -19,8 +19,8 @@ + #define DEFAULT_EDITOR "notepad.exe" + /* no DEFAULT_EDITOR_LINENUMBER_ARG for Notepad */ + #else +-#define DEFAULT_EDITOR "vi" +-#define DEFAULT_EDITOR_LINENUMBER_ARG "+" ++#define DEFAULT_EDITOR "sensible-editor" ++/*#define DEFAULT_EDITOR_LINENUMBER_ARG "+"*/ + #endif + + #define DEFAULT_PROMPT1 "%/%R%x%# " +--- a/src/include/fe_utils/print.h ++++ b/src/include/fe_utils/print.h +@@ -20,7 +20,7 @@ + + /* This is not a particularly great place for this ... */ + #ifndef __CYGWIN__ +-#define DEFAULT_PAGER "more" ++#define DEFAULT_PAGER "pager" + #else + #define DEFAULT_PAGER "less" + #endif diff --git a/debian/patches/70-history b/debian/patches/70-history new file mode 100644 index 00000000000..34c868357f9 --- /dev/null +++ b/debian/patches/70-history @@ -0,0 +1,13 @@ +Author: Christoph Berg +Description: Document Debian location of release notes files. +Forwarded: No, Debian specific + +--- a/HISTORY ++++ b/HISTORY +@@ -3,3 +3,6 @@ + + Distribution file sets include release notes for their version and preceding + versions. Visit the file doc/src/sgml/html/release.html in an HTML browser. ++ ++On Debian systems, the release notes are contained in the postgresql-doc-* ++packages, located in /usr/share/doc/postgresql-doc-*/html/release.html. diff --git a/debian/patches/autoconf2.69 b/debian/patches/autoconf2.69 new file mode 100644 index 00000000000..429044e698b --- /dev/null +++ b/debian/patches/autoconf2.69 @@ -0,0 +1,7 @@ +--- a/configure.ac ++++ b/configure.ac +@@ -22,4 +21,0 @@ AC_INIT([PostgreSQL], [15devel], [pgsql- +-m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.69 is required. +-Untested combinations of 'autoconf' and PostgreSQL versions are not +-recommended. You can remove the check from 'configure.ac' but it is then +-your responsibility whether the result works or not.])]) diff --git a/debian/patches/extension_destdir b/debian/patches/extension_destdir new file mode 100644 index 00000000000..eedc81bd4ba --- /dev/null +++ b/debian/patches/extension_destdir @@ -0,0 +1,270 @@ +--- a/src/backend/commands/extension.c ++++ b/src/backend/commands/extension.c +@@ -132,6 +132,8 @@ static void ApplyExtensionUpdates(Oid ex + bool cascade, + bool is_create); + static char *read_whole_file(const char *filename, int *length); ++static bool file_exists(const char *name); ++static bool directory_exists(const char *dir); + + + /* +@@ -392,6 +394,16 @@ get_extension_control_filename(const cha + + get_share_path(my_exec_path, sharepath); + result = (char *) palloc(MAXPGPATH); ++ /* ++ * If extension_destdir is set, try to find the file there first ++ */ ++ if (*extension_destdir != '\0') ++ { ++ snprintf(result, MAXPGPATH, "%s%s/extension/%s.control", ++ extension_destdir, sharepath, extname); ++ if (file_exists(result)) ++ return result; ++ } + snprintf(result, MAXPGPATH, "%s/extension/%s.control", + sharepath, extname); + +@@ -431,6 +443,16 @@ get_extension_aux_control_filename(Exten + scriptdir = get_extension_script_directory(control); + + result = (char *) palloc(MAXPGPATH); ++ /* ++ * If extension_destdir is set, try to find the file there first ++ */ ++ if (*extension_destdir != '\0') ++ { ++ snprintf(result, MAXPGPATH, "%s%s/%s--%s.control", ++ extension_destdir, scriptdir, control->name, version); ++ if (file_exists(result)) ++ return result; ++ } + snprintf(result, MAXPGPATH, "%s/%s--%s.control", + scriptdir, control->name, version); + +@@ -449,6 +471,23 @@ get_extension_script_filename(ExtensionC + scriptdir = get_extension_script_directory(control); + + result = (char *) palloc(MAXPGPATH); ++ /* ++ * If extension_destdir is set, try to find the file there first ++ */ ++ if (*extension_destdir != '\0') ++ { ++ if (from_version) ++ snprintf(result, MAXPGPATH, "%s%s/%s--%s--%s.sql", ++ extension_destdir, scriptdir, control->name, from_version, version); ++ else ++ snprintf(result, MAXPGPATH, "%s%s/%s--%s.sql", ++ extension_destdir, scriptdir, control->name, version); ++ if (file_exists(result)) ++ { ++ pfree(scriptdir); ++ return result; ++ } ++ } + if (from_version) + snprintf(result, MAXPGPATH, "%s/%s--%s--%s.sql", + scriptdir, control->name, from_version, version); +@@ -1186,6 +1225,59 @@ get_ext_ver_list(ExtensionControlFile *c + DIR *dir; + struct dirent *de; + ++ /* ++ * If extension_destdir is set, try to find the files there first ++ */ ++ if (*extension_destdir != '\0') ++ { ++ char location[MAXPGPATH]; ++ ++ snprintf(location, MAXPGPATH, "%s%s", extension_destdir, ++ get_extension_script_directory(control)); ++ dir = AllocateDir(location); ++ while ((de = ReadDir(dir, location)) != NULL) ++ { ++ char *vername; ++ char *vername2; ++ ExtensionVersionInfo *evi; ++ ExtensionVersionInfo *evi2; ++ ++ /* must be a .sql file ... */ ++ if (!is_extension_script_filename(de->d_name)) ++ continue; ++ ++ /* ... matching extension name followed by separator */ ++ if (strncmp(de->d_name, control->name, extnamelen) != 0 || ++ de->d_name[extnamelen] != '-' || ++ de->d_name[extnamelen + 1] != '-') ++ continue; ++ ++ /* extract version name(s) from 'extname--something.sql' filename */ ++ vername = pstrdup(de->d_name + extnamelen + 2); ++ *strrchr(vername, '.') = '\0'; ++ vername2 = strstr(vername, "--"); ++ if (!vername2) ++ { ++ /* It's an install, not update, script; record its version name */ ++ evi = get_ext_ver_info(vername, &evi_list); ++ evi->installable = true; ++ continue; ++ } ++ *vername2 = '\0'; /* terminate first version */ ++ vername2 += 2; /* and point to second */ ++ ++ /* if there's a third --, it's bogus, ignore it */ ++ if (strstr(vername2, "--")) ++ continue; ++ ++ /* Create ExtensionVersionInfos and link them together */ ++ evi = get_ext_ver_info(vername, &evi_list); ++ evi2 = get_ext_ver_info(vername2, &evi_list); ++ evi->reachable = lappend(evi->reachable, evi2); ++ } ++ FreeDir(dir); ++ } ++ + location = get_extension_script_directory(control); + dir = AllocateDir(location); + while ((de = ReadDir(dir, location)) != NULL) +@@ -3470,3 +3562,32 @@ read_whole_file(const char *filename, in + buf[*length] = '\0'; + return buf; + } ++ ++static bool ++file_exists(const char *name) ++{ ++ struct stat st; ++ ++ Assert(name != NULL); ++ ++ if (stat(name, &st) == 0) ++ return S_ISDIR(st.st_mode) ? false : true; ++ else if (!(errno == ENOENT || errno == ENOTDIR || errno == EACCES)) ++ ereport(ERROR, ++ (errcode_for_file_access(), ++ errmsg("could not access file \"%s\": %m", name))); ++ ++ return false; ++} ++ ++static bool ++directory_exists(const char *dir) ++{ ++ struct stat st; ++ ++ if (stat(dir, &st) != 0) ++ return false; ++ if (S_ISDIR(st.st_mode)) ++ return true; ++ return false; ++} +--- a/src/include/utils/guc.h ++++ b/src/include/utils/guc.h +@@ -274,6 +274,7 @@ extern PGDLLIMPORT char *ConfigFileName; + extern PGDLLIMPORT char *HbaFileName; + extern PGDLLIMPORT char *IdentFileName; + extern PGDLLIMPORT char *external_pid_file; ++extern PGDLLIMPORT char *extension_destdir; + + extern PGDLLIMPORT char *application_name; + +--- a/src/backend/utils/fmgr/dfmgr.c ++++ b/src/backend/utils/fmgr/dfmgr.c +@@ -34,6 +34,7 @@ + #include "lib/stringinfo.h" + #include "miscadmin.h" + #include "storage/shmem.h" ++#include "utils/guc.h" + #include "utils/hsearch.h" + + +@@ -432,7 +433,7 @@ expand_dynamic_library_name(const char * + { + bool have_slash; + char *new; +- char *full; ++ char *full, *full2; + + Assert(name); + +@@ -447,6 +448,19 @@ expand_dynamic_library_name(const char * + else + { + full = substitute_libpath_macro(name); ++ /* ++ * If extension_destdir is set, try to find the file there first ++ */ ++ if (*extension_destdir != '\0') ++ { ++ full2 = psprintf("%s%s", extension_destdir, full); ++ if (file_exists(full2)) ++ { ++ pfree(full); ++ return full2; ++ } ++ pfree(full2); ++ } + if (file_exists(full)) + return full; + pfree(full); +@@ -465,6 +479,19 @@ expand_dynamic_library_name(const char * + { + full = substitute_libpath_macro(new); + pfree(new); ++ /* ++ * If extension_destdir is set, try to find the file there first ++ */ ++ if (*extension_destdir != '\0') ++ { ++ full2 = psprintf("%s%s", extension_destdir, full); ++ if (file_exists(full2)) ++ { ++ pfree(full); ++ return full2; ++ } ++ pfree(full2); ++ } + if (file_exists(full)) + return full; + pfree(full); +--- a/src/backend/utils/misc/postgresql.conf.sample ++++ b/src/backend/utils/misc/postgresql.conf.sample +@@ -750,6 +750,8 @@ + # - Other Defaults - + + #dynamic_library_path = '$libdir' ++#extension_destdir = '' # prepend path when loading extensions ++ # and shared objects (added by Debian) + #gin_fuzzy_search_limit = 0 + + +--- a/src/backend/utils/misc/guc_tables.c ++++ b/src/backend/utils/misc/guc_tables.c +@@ -528,6 +528,7 @@ char *ConfigFileName; + char *HbaFileName; + char *IdentFileName; + char *external_pid_file; ++char *extension_destdir; + + char *application_name; + +@@ -4346,6 +4347,17 @@ struct config_string ConfigureNamesStrin + }, + + { ++ {"extension_destdir", PGC_SUSET, FILE_LOCATIONS, ++ gettext_noop("Path to prepend for extension loading."), ++ gettext_noop("This directory is prepended to paths when loading extensions (control and SQL files), and to the '$libdir' directive when loading modules that back functions. The location is made configurable to allow build-time testing of extensions that do not have been installed to their proper location yet."), ++ GUC_SUPERUSER_ONLY ++ }, ++ &extension_destdir, ++ "", ++ NULL, NULL, NULL ++ }, ++ ++ { + {"ssl_library", PGC_INTERNAL, PRESET_OPTIONS, + gettext_noop("Shows the name of the SSL library."), + NULL, diff --git a/debian/patches/filter-debug-prefix-map b/debian/patches/filter-debug-prefix-map new file mode 100644 index 00000000000..dcfa9d89f2e --- /dev/null +++ b/debian/patches/filter-debug-prefix-map @@ -0,0 +1,44 @@ +To make the PostgreSQL server packages build reproducibly, we need to remove +the build path from -fdebug-prefix-map and -ffile-prefix-map in CFLAGS. + +* The actual server build still uses the original CFLAGS so the build path is + correctly mapped in the object files. +* The information printed by the pg_config binary and the system view is + filtered in src/common/Makefile and the configure script. +* The build paths stored in Makefile.global are filtered in debian/rules. + (abs_top_builddir, abs_top_srcdir, configure_args, CFLAGS) +* To make PGXS module builds reproducible, pg_buildext copies the environment + CFLAGS to COPT where Makefile.global picks them up, using the prefix maps + from dpkg-buildflags. + +--- a/src/common/Makefile ++++ b/src/common/Makefile +@@ -33,7 +33,7 @@ STD_CPPFLAGS := $(filter-out -I$(top_src + STD_LDFLAGS := $(filter-out -L$(top_builddir)/src/common -L$(top_builddir)/src/port,$(LDFLAGS)) + override CPPFLAGS += -DVAL_CC="\"$(CC)\"" + override CPPFLAGS += -DVAL_CPPFLAGS="\"$(STD_CPPFLAGS)\"" +-override CPPFLAGS += -DVAL_CFLAGS="\"$(CFLAGS)\"" ++override CPPFLAGS += -DVAL_CFLAGS="\"$(filter-out -fdebug-prefix-map=% -ffile-prefix-map=%,$(CFLAGS))\"" + override CPPFLAGS += -DVAL_CFLAGS_SL="\"$(CFLAGS_SL)\"" + override CPPFLAGS += -DVAL_LDFLAGS="\"$(STD_LDFLAGS)\"" + override CPPFLAGS += -DVAL_LDFLAGS_EX="\"$(LDFLAGS_EX)\"" +--- a/configure.ac ++++ b/configure.ac +@@ -27,6 +27,7 @@ AC_COPYRIGHT([Copyright (c) 1996-2023, P + AC_CONFIG_SRCDIR([src/backend/access/common/heaptuple.c]) + AC_CONFIG_AUX_DIR(config) + AC_PREFIX_DEFAULT(/usr/local/pgsql) ++[ac_configure_args=$(echo "$ac_configure_args" | sed -e "s/ -f\(debug\|file\)-prefix-map=[^' ]*//g")] + AC_DEFINE_UNQUOTED(CONFIGURE_ARGS, ["$ac_configure_args"], [Saved arguments from configure]) + + [PG_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\)'`] +--- a/configure ++++ b/configure +@@ -2822,6 +2822,7 @@ ac_config_sub="$SHELL $ac_aux_dir/config + ac_configure="$SHELL $ac_aux_dir/configure" # Please don't use this var. + + ++ac_configure_args=$(echo "$ac_configure_args" | sed -e "s/ -f\(debug\|file\)-prefix-map=[^' ]*//g") + + + cat >>confdefs.h <<_ACEOF diff --git a/debian/patches/focal-arm64-outline-atomics b/debian/patches/focal-arm64-outline-atomics new file mode 100644 index 00000000000..b87d8f50eb9 --- /dev/null +++ b/debian/patches/focal-arm64-outline-atomics @@ -0,0 +1,23 @@ +Enable outline-atomics on arm64. + +The flag was added in focal's gcc, but is off by default there. It is enabled +by default on all later distributions (hirsute, impish, bullseye, bookwork, +sid). + +https://www.postgresql.org/message-id/flat/1635221042457.21654%40amazon.com + +This patch can be removed once focal is EOL. + +--- a/configure.ac ++++ b/configure.ac +@@ -576,6 +576,10 @@ if test "$GCC" = yes -a "$ICC" = no; the + if test -n "$NOT_THE_CFLAGS"; then + CFLAGS="$CFLAGS -Wno-cast-function-type-strict" + fi ++ if test x"$host_cpu" == x"aarch64"; then ++ PGAC_PROG_CC_CFLAGS_OPT([-moutline-atomics]) ++ PGAC_PROG_CXX_CFLAGS_OPT([-moutline-atomics]) ++ fi + elif test "$ICC" = yes; then + # Intel's compiler has a bug/misoptimization in checking for + # division by NAN (NaN == 0), -mp1 fixes it, so add it to the CFLAGS. diff --git a/debian/patches/hurd-iovec b/debian/patches/hurd-iovec new file mode 100644 index 00000000000..e5255f02230 --- /dev/null +++ b/debian/patches/hurd-iovec @@ -0,0 +1,26 @@ +hurd-i386 does not define IOV_MAX + +--- a/src/include/port/pg_iovec.h ++++ b/src/include/port/pg_iovec.h +@@ -20,9 +20,6 @@ + + #else + +-/* POSIX requires at least 16 as a maximum iovcnt. */ +-#define IOV_MAX 16 +- + /* Define our own POSIX-compatible iovec struct. */ + struct iovec + { +@@ -32,6 +29,11 @@ struct iovec + + #endif + ++/* POSIX requires at least 16 as a maximum iovcnt. */ ++#ifndef IOV_MAX ++#define IOV_MAX 16 ++#endif ++ + /* Define a reasonable maximum that is safe to use on the stack. */ + #define PG_IOV_MAX Min(IOV_MAX, 32) + diff --git a/debian/patches/jit-s390x b/debian/patches/jit-s390x new file mode 100644 index 00000000000..deb64e6af04 --- /dev/null +++ b/debian/patches/jit-s390x @@ -0,0 +1,96 @@ +From 0edaa982336823d4d7af8f10b91579fe0099ef3d Mon Sep 17 00:00:00 2001 +From: Tom Stellard +Date: Tue, 20 Apr 2021 20:14:21 -0700 +Subject: [PATCH] jit: Workaround potential datalayout mismatch on s390x + +LLVM's s390x target uses a different datalayout for z13 and newer processors. +If llvmjit_types.bc is compiled to target a processor older than z13, and +then the JIT runs on a z13 or newer processor, then there will be a mismatch +in datalayouts between llvmjit_types.bc and the JIT engine. This mismatch +causes the JIT to fail at runtime. +--- + src/backend/jit/llvm/llvmjit.c | 46 ++++++++++++++++++++++++++++++++-- + 1 file changed, 44 insertions(+), 2 deletions(-) + +--- a/src/backend/jit/llvm/llvmjit.c ++++ b/src/backend/jit/llvm/llvmjit.c +@@ -777,6 +777,37 @@ llvm_compile_module(LLVMJitContext *cont + } + + /* ++ * For the systemz target, LLVM uses a different datalayout for z13 and newer ++ * CPUs than it does for older CPUs. This can cause a mismatch in datalayouts ++ * in the case where the llvm_types_module is compiled with a pre-z13 CPU ++ * and the JIT is running on z13 or newer. ++ * See computeDataLayout() function in ++ * llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp for information on the ++ * datalayout differences. ++ */ ++static bool ++needs_systemz_workaround(void) ++{ ++ bool ret = false; ++#ifdef __s390x__ ++ LLVMContextRef llvm_context; ++ LLVMTypeRef vec_type; ++ LLVMTargetDataRef llvm_layoutref; ++ if (strncmp(LLVMGetTargetName(llvm_targetref), "systemz", strlen("systemz"))) ++ { ++ return false; ++ } ++ ++ llvm_context = LLVMGetModuleContext(llvm_types_module); ++ vec_type = LLVMVectorType(LLVMIntTypeInContext(llvm_context, 32), 4); ++ llvm_layoutref = LLVMCreateTargetData(llvm_layout); ++ ret = (LLVMABIAlignmentOfType(llvm_layoutref, vec_type) == 16); ++ LLVMDisposeTargetData(llvm_layoutref); ++#endif ++ return ret; ++} ++ ++/* + * Per session initialization. + */ + static void +@@ -785,6 +816,7 @@ llvm_session_initialize(void) + MemoryContext oldcontext; + char *error = NULL; + char *cpu = NULL; ++ char *host_features = NULL; + char *features = NULL; + LLVMTargetMachineRef opt0_tm; + LLVMTargetMachineRef opt3_tm; +@@ -826,10 +858,17 @@ llvm_session_initialize(void) + * features not all CPUs have (weird, huh). + */ + cpu = LLVMGetHostCPUName(); +- features = LLVMGetHostCPUFeatures(); ++ features = host_features = LLVMGetHostCPUFeatures(); + elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"", + cpu, features); + ++ if (needs_systemz_workaround()) ++ { ++ const char *no_vector =",-vector"; ++ features = malloc(sizeof(char) * (strlen(host_features) + strlen(no_vector) + 1)); ++ sprintf(features, "%s%s", host_features, no_vector); ++ } ++ + opt0_tm = + LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features, + LLVMCodeGenLevelNone, +@@ -843,8 +882,13 @@ llvm_session_initialize(void) + + LLVMDisposeMessage(cpu); + cpu = NULL; +- LLVMDisposeMessage(features); ++ if (features != host_features) ++ { ++ free(features); ++ } + features = NULL; ++ LLVMDisposeMessage(host_features); ++ host_features = NULL; + + /* force symbols in main binary to be loaded */ + LLVMLoadLibraryPermanently(NULL); diff --git a/debian/patches/libpgport-pkglibdir b/debian/patches/libpgport-pkglibdir new file mode 100644 index 00000000000..1118c2bfe55 --- /dev/null +++ b/debian/patches/libpgport-pkglibdir @@ -0,0 +1,84 @@ +Author: Christoph Berg +Description: Move libpgport/libpgcommon/libpgfeutils from libdir to pkglibdir + This allows client applications to link to version-specific libraries. + Used by pg-checksums. +Forwarded: No, (somewhat) Debian specific + +--- a/src/common/Makefile ++++ b/src/common/Makefile +@@ -125,15 +125,15 @@ distprep: kwlist_d.h + + # libpgcommon is needed by some contrib + install: all installdirs +- $(INSTALL_STLIB) libpgcommon.a '$(DESTDIR)$(libdir)/libpgcommon.a' +- $(INSTALL_STLIB) libpgcommon_shlib.a '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' ++ $(INSTALL_STLIB) libpgcommon.a '$(DESTDIR)$(pkglibdir)/libpgcommon.a' ++ $(INSTALL_STLIB) libpgcommon_shlib.a '$(DESTDIR)$(pkglibdir)/libpgcommon_shlib.a' + + installdirs: +- $(MKDIR_P) '$(DESTDIR)$(libdir)' ++ $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' + + uninstall: +- rm -f '$(DESTDIR)$(libdir)/libpgcommon.a' +- rm -f '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgcommon.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgcommon_shlib.a' + + libpgcommon.a: $(OBJS_FRONTEND) + rm -f $@ +--- a/src/fe_utils/Makefile ++++ b/src/fe_utils/Makefile +@@ -52,13 +52,13 @@ distprep: psqlscan.c + + # libpgfeutils could be useful to contrib, so install it + install: all installdirs +- $(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(libdir)/libpgfeutils.a' ++ $(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(pkglibdir)/libpgfeutils.a' + + installdirs: +- $(MKDIR_P) '$(DESTDIR)$(libdir)' ++ $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' + + uninstall: +- rm -f '$(DESTDIR)$(libdir)/libpgfeutils.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgfeutils.a' + + clean distclean: + rm -f libpgfeutils.a $(OBJS) lex.backup +--- a/src/port/Makefile ++++ b/src/port/Makefile +@@ -70,15 +70,15 @@ all: libpgport.a libpgport_shlib.a libpg + + # libpgport is needed by some contrib + install: all installdirs +- $(INSTALL_STLIB) libpgport.a '$(DESTDIR)$(libdir)/libpgport.a' +- $(INSTALL_STLIB) libpgport_shlib.a '$(DESTDIR)$(libdir)/libpgport_shlib.a' ++ $(INSTALL_STLIB) libpgport.a '$(DESTDIR)$(pkglibdir)/libpgport.a' ++ $(INSTALL_STLIB) libpgport_shlib.a '$(DESTDIR)$(pkglibdir)/libpgport_shlib.a' + + installdirs: +- $(MKDIR_P) '$(DESTDIR)$(libdir)' ++ $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' + + uninstall: +- rm -f '$(DESTDIR)$(libdir)/libpgport.a' +- rm -f '$(DESTDIR)$(libdir)/libpgport_shlib.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgport.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgport_shlib.a' + + libpgport.a: $(OBJS) + rm -f $@ +--- a/src/Makefile.global.in ++++ b/src/Makefile.global.in +@@ -595,8 +595,8 @@ libpq = -L$(libpq_builddir) -lpq + # on client link lines, since that also appears in $(LIBS). + # libpq_pgport_shlib is the same idea, but for use in client shared libraries. + ifdef PGXS +-libpq_pgport = -L$(libdir) -lpgcommon -lpgport $(libpq) +-libpq_pgport_shlib = -L$(libdir) -lpgcommon_shlib -lpgport_shlib $(libpq) ++libpq_pgport = -L$(pkglibdir) -lpgcommon -lpgport $(libpq) ++libpq_pgport_shlib = -L$(pkglibdir) -lpgcommon_shlib -lpgport_shlib $(libpq) + else + libpq_pgport = -L$(top_builddir)/src/common -lpgcommon -L$(top_builddir)/src/port -lpgport $(libpq) + libpq_pgport_shlib = -L$(top_builddir)/src/common -lpgcommon_shlib -L$(top_builddir)/src/port -lpgport_shlib $(libpq) diff --git a/debian/patches/series b/debian/patches/series new file mode 100644 index 00000000000..894b7d3a485 --- /dev/null +++ b/debian/patches/series @@ -0,0 +1,13 @@ +50-per-version-dirs.patch +51-default-sockets-in-var.patch +52-tutorial-README.patch +53-pg_service.conf_directory_doc.patch +54-debian-alternatives-for-external-tools.patch +70-history +filter-debug-prefix-map +libpgport-pkglibdir +extension_destdir +autoconf2.69 +focal-arm64-outline-atomics +jit-s390x +hurd-iovec diff --git a/debian/po/POTFILES.in b/debian/po/POTFILES.in new file mode 100644 index 00000000000..ebb2ac37643 --- /dev/null +++ b/debian/po/POTFILES.in @@ -0,0 +1 @@ +[type: gettext/rfc822deb] postgresql-16.templates diff --git a/debian/po/de.po b/debian/po/de.po new file mode 100644 index 00000000000..df2bd4d7012 --- /dev/null +++ b/debian/po/de.po @@ -0,0 +1,37 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# Copyright (C) Helge Kreutzmann , 2019. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11 11.1-2\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-19 07:33+0100\n" +"Last-Translator: Helge Kreutzmann \n" +"Language-Team: German \n" +"Language: de\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "" +"PostgreSQL-Verzeichnisse entfernen, wenn das Paket endgültig gelöscht wird?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Beim Entfernen der PostgreSQL-Server-Pakete werden existierende Datenbank-" +"Cluster intakt gelassen, d.h. ihre Konfigurations-, Daten- und " +"Log-Verzeichnisse werden nicht entfernt. Beim endgültigen Löschen des " +"Pakets können die Verzeichnisse optional entfernt werden." diff --git a/debian/po/es.po b/debian/po/es.po new file mode 100644 index 00000000000..d408e656a0d --- /dev/null +++ b/debian/po/es.po @@ -0,0 +1,58 @@ +# postgresql-13 po-debconf translation to Spanish. +# Copyright (C) 2021 Software in the Public Interest +# This file is distributed under the same license as the postgresql-13 package. +# +# Changes: +# - Initial translation +# Jonathan Bustillos , 2021. +# +# Traductores, si no conocen el formato PO, merece la pena leer la +# documentación de gettext, especialmente las secciones dedicadas a este +# formato, por ejemplo ejecutando: +# info -n '(gettext)PO Files' +# info -n '(gettext)Header Entry' +# +# Equipo de traducción al español, por favor lean antes de traducir +# los siguientes documentos: +# +# - El proyecto de traducción de Debian al español +# http://www.debian.org/intl/spanish/ +# especialmente las notas y normas de traducción en +# http://www.debian.org/intl/spanish/notas +# +# - La guía de traducción de po's de debconf: +# /usr/share/doc/po-debconf/README-trans +# o http://www.debian.org/intl/l10n/po-debconf/README-trans +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2021-03-31 18:37+0000\n" +"PO-Revision-Date: 2021-04-03 14:25-0600\n" +"Last-Translator: Jonathan Bustillos \n" +"Language-Team: Debian Spanish \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"X-Generator: Gtranslator 2.91.7\n" + +#. Type: boolean +#. Description +#: ../postgresql-13.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "¿Eliminar los directorios de PostgreSQL cuando se purga el paquete?" + +#. Type: boolean +#. Description +#: ../postgresql-13.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"La eliminación del paquete del servidor PostgreSQL dejará intactos los " +"clusters de bases de datos existentes, es decir, no se eliminarán sus " +"directorios de configuración, datos y registro. Al purgar el paquete, los " +"directorios pueden ser eliminados opcionalmente." diff --git a/debian/po/fr.po b/debian/po/fr.po new file mode 100644 index 00000000000..cc3c6ee55c6 --- /dev/null +++ b/debian/po/fr.po @@ -0,0 +1,39 @@ +# Translation of postgresql debconf templates to French +# Copyright (C) 2019 Debian French l10n team +# This file is distributed under the same license as the postgresql-11 package. +# +# Jean-Pierre Giraud , 2019. +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-14 14:25+0100\n" +"Last-Translator: Jean-Pierre Giraud \n" +"Language-Team: French \n" +"Language: fr\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" +"X-Generator: Lokalize 2.0\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "" +"Faut-il supprimer les répertoires de PostgreSQL lors de la purge du paquet ?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"La suppression du paquet du serveur PostgreSQL laissera les grappes de bases " +"de données existantes intactes, c'est-à-dire que leurs répertoires de " +"configuration, de données et de journal ne seront pas supprimés. Lors de la " +"purge du paquet, les répertoires peuvent être supprimés de façon optionnelle." diff --git a/debian/po/it.po b/debian/po/it.po new file mode 100644 index 00000000000..d8020b9fd8b --- /dev/null +++ b/debian/po/it.po @@ -0,0 +1,37 @@ +# postgresql-14 Italian translation. +# Copyright (C) 2022 postgresql-14's copyright holder +# This file is distributed under the same license as the postgresql-14 package. +# Ceppo , 2022. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-14\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2022-08-23 19:57+0000\n" +"PO-Revision-Date: 2022-08-23 00:00+0000\n" +"Last-Translator: Ceppo \n" +"Language-Team: Italian \n" +"Language: it\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. Type: boolean +#. Description +#: ../postgresql-14.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Rimuovere le directory di PostgreSQL quando viene eseguito il purge " +"del pacchetto?" + +#. Type: boolean +#. Description +#: ../postgresql-14.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"La rimozione del pacchetto server di PostgreSQL lascerà intatti i cluster " +"di database esistenti, cioè i loro dati, configurazione e directory di log " +"non saranno rimossi. Eseguendo il purge del pacchetto, le directory possono " +"opzionalmente essere rimosse." diff --git a/debian/po/nl.po b/debian/po/nl.po new file mode 100644 index 00000000000..11cda877b96 --- /dev/null +++ b/debian/po/nl.po @@ -0,0 +1,40 @@ +# Dutch translation of postgresql-11 debconf templates. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# FIRST AUTHOR , YEAR. +# Frans Spiesschaert , 2019. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11_11.1-2\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-19 10:43+0100\n" +"Last-Translator: Frans Spiesschaert \n" +"Language-Team: Debian Dutch l10n Team \n" +"Language: nl\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"X-Generator: Gtranslator 2.91.7\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "" +"De PostgreSQL-mappen verwijderen wanneer het pakket gewist (purged) wordt?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Bij het verwijderen van het serverpakket van PostgreSQL blijven de bestaande " +"databaseclusters intact. Dit wil zeggen dat hun configuratie-, gegevens- en " +"logboekmappen niet verwijderd worden. Bij het wissen (purge) van het pakket, " +"kunnen de mappen naar keuze verwijderd worden." diff --git a/debian/po/pt.po b/debian/po/pt.po new file mode 100644 index 00000000000..ee294d15a44 --- /dev/null +++ b/debian/po/pt.po @@ -0,0 +1,39 @@ +# Translation of postgresql-11's debconf messages to European Portuguese +# Copyright (C) 2019 THE postgresql-11'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# +# Américo Monteiro , 2019. +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11 11.1-2\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-15 00:34+0000\n" +"Last-Translator: Américo Monteiro \n" +"Language-Team: Portuguese <>\n" +"Language: pt\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"X-Generator: Lokalize 2.0\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Remover os directórios do PostgreSQL quando o pacote for purgado?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Remover o pacote do servidor PostgreSQL irá deixar intactos agrupamentos de " +"bases de dados existentes, isto é, a sua configuração, dados, e relatórios " +"são serão removidos. Ao purgar o pacote, estes directórios podem " +"opcionalmente ser removidos." + diff --git a/debian/po/pt_BR.po b/debian/po/pt_BR.po new file mode 100644 index 00000000000..5e746cf1ca2 --- /dev/null +++ b/debian/po/pt_BR.po @@ -0,0 +1,37 @@ +# Debconf translations for postgresql-11. +# Copyright (C) 2019 THE postgresql-11'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# Adriano Rafael Gomes , 2019. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-19 18:06-0200\n" +"Last-Translator: Adriano Rafael Gomes \n" +"Language-Team: Brazilian Portuguese \n" +"Language: pt_BR\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Remover diretórios do PostgreSQL ao expurgar o pacote?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Remover o pacote do servidor PostgreSQL deixará os \"clusters\" de bancos de " +"dados existentes intactos, ou seja, suas configurações, dados e diretórios " +"de log não serão removidos. Ao expurgar o pacote, os diretórios podem ser " +"opcionalmente removidos." diff --git a/debian/po/ro.po b/debian/po/ro.po new file mode 100644 index 00000000000..f38fc5581b6 --- /dev/null +++ b/debian/po/ro.po @@ -0,0 +1,50 @@ +# Mesajele în limba română pentru pachetul postgresql. +# translation of postgresql-xx_ro.po to Romanian +# Copyright © 2023 THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql package. +# +# Remus-Gabriel Chelu , 2023. +# +# Cronologia traducerii fișierului „postgresql”: +# Traducerea inițială, făcută de R-GC, pentru versiunea postgresql-15_15.1-1. +# Actualizare a traducerii pentru versiunea Y, făcută de X, Y(anul). +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-15 15.1-1\n" +"Report-Msgid-Bugs-To: postgresql-15@packages.debian.org\n" +"POT-Creation-Date: 2023-01-19 20:42+0000\n" +"PO-Revision-Date: 2023-02-12 17:42+0100\n" +"Last-Translator: Remus-Gabriel Chelu \n" +"Language-Team: Romanian \n" +"Language: ro\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=3; plural=(n==1 ? 0 : n==0 || (n!=1 && n%100>=1 && " +"n%100<=19) ? 1 : 2);\n" +"X-Generator: Poedit 3.2.2\n" + +#. Type: boolean +#. Description +#: ../postgresql-15.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Eliminați directoarele PostgreSQL atunci când pachetul este înlăturat?" + +# R-GC, scrie: +# la sugestia lui DȘ, am modificat traducerea +# acestui mesaj, de la: +# „... va lăsa intacte clusterele de baze de date existente, ...”, la: +# „... va lăsa intacte grupurile de servere (clusters) de baze de date existente, ...” +#. Type: boolean +#. Description +#: ../postgresql-15.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Eliminarea pachetului de server PostgreSQL va lăsa intacte grupurile de servere " +"(clusters) de baze de date existente, adică configurația, datele și " +"directoarele lor de jurnal nu vor fi eliminate. La înlăturarea pachetului, " +"directoarele pot fi eliminate opțional." diff --git a/debian/po/ru.po b/debian/po/ru.po new file mode 100644 index 00000000000..9828439c667 --- /dev/null +++ b/debian/po/ru.po @@ -0,0 +1,39 @@ +# Russian translation of debconf template for postgresql-11 +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# Lev Lamberov , 2019 +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-27 14:56+0500\n" +"Language-Team: Debian L10N Russian \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"X-Generator: Poedit 2.2.1\n" +"Last-Translator: Lev Lamberov \n" +"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n" +"%10<=4 && (n%100<12 || n%100>14) ? 1 : 2);\n" +"Language: ru\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Удалить каталоги PostgreSQL при вычищении пакета?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"При удалении серверного пакета PostgreSQL существующие кластеры баз данных " +"останутся нетронутыми. То есть, их каталоги с настройками, данными и " +"журналами не будут удалены. При вычистке пакета эти каталоги могут быть при " +"необходимости удалены." diff --git a/debian/postgresql-16.install b/debian/postgresql-16.install new file mode 100755 index 00000000000..bcfc33ec7d8 --- /dev/null +++ b/debian/postgresql-16.install @@ -0,0 +1,57 @@ +#!/usr/bin/dh-exec + +usr/lib/postgresql/*/bin/initdb +usr/lib/postgresql/*/bin/oid2name +usr/lib/postgresql/*/bin/pg_archivecleanup +usr/lib/postgresql/*/bin/pgbench +usr/lib/postgresql/*/bin/pg_checksums +usr/lib/postgresql/*/bin/pg_controldata +usr/lib/postgresql/*/bin/pg_ctl +usr/lib/postgresql/*/bin/pg_resetwal +usr/lib/postgresql/*/bin/pg_rewind +usr/lib/postgresql/*/bin/pg_test_fsync +usr/lib/postgresql/*/bin/pg_test_timing +usr/lib/postgresql/*/bin/pg_upgrade +usr/lib/postgresql/*/bin/pg_waldump +usr/lib/postgresql/*/bin/postgres +usr/lib/postgresql/*/bin/vacuumlo +[!alpha !hppa !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] usr/lib/postgresql/*/lib/bitcode +[!alpha !hppa !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] usr/lib/postgresql/*/lib/llvmjit_types.bc +usr/lib/postgresql/*/lib/*.so +usr/share/locale/*/LC_MESSAGES/initdb-*.mo +usr/share/locale/*/LC_MESSAGES/pg_archivecleanup-*.mo +usr/share/locale/*/LC_MESSAGES/pg_checksums-*.mo +usr/share/locale/*/LC_MESSAGES/pg_controldata-*.mo +usr/share/locale/*/LC_MESSAGES/pg_ctl-*.mo +usr/share/locale/*/LC_MESSAGES/pg_resetwal-*.mo +usr/share/locale/*/LC_MESSAGES/pg_rewind-*.mo +usr/share/locale/*/LC_MESSAGES/pg_test_fsync-*.mo +usr/share/locale/*/LC_MESSAGES/pg_test_timing-*.mo +usr/share/locale/*/LC_MESSAGES/pg_upgrade-*.mo +usr/share/locale/*/LC_MESSAGES/pg_waldump-*.mo +usr/share/locale/*/LC_MESSAGES/postgres-*.mo +usr/share/locale/*/LC_MESSAGES/plpgsql-*.mo +[linux-any] usr/share/postgresql/*/contrib/sepgsql.sql +usr/share/postgresql/*/errcodes.txt +usr/share/postgresql/*/extension/* +usr/share/postgresql/*/man/man1/initdb.1* +usr/share/postgresql/*/man/man1/oid2name.1* +usr/share/postgresql/*/man/man1/pg_archivecleanup.1* +usr/share/postgresql/*/man/man1/pgbench.1* +usr/share/postgresql/*/man/man1/pg_checksums.1* +usr/share/postgresql/*/man/man1/pg_controldata.1* +usr/share/postgresql/*/man/man1/pg_ctl.1* +usr/share/postgresql/*/man/man1/pg_resetwal.1* +usr/share/postgresql/*/man/man1/pg_rewind.1* +usr/share/postgresql/*/man/man1/pg_test_fsync.1* +usr/share/postgresql/*/man/man1/pg_test_timing.1* +usr/share/postgresql/*/man/man1/pg_upgrade.1* +usr/share/postgresql/*/man/man1/pg_waldump.1* +usr/share/postgresql/*/man/man1/postgres.1* +usr/share/postgresql/*/man/man1/vacuumlo.1* +usr/share/postgresql/*/timezonesets/* +usr/share/postgresql/*/tsearch_data +usr/share/postgresql/*/*.sql +usr/share/postgresql/*/*.conf.sample +usr/share/postgresql/*/postgres.bki +usr/share/postgresql/*/sql_features.txt diff --git a/debian/postgresql-16.lintian-overrides b/debian/postgresql-16.lintian-overrides new file mode 100644 index 00000000000..db5908332f9 --- /dev/null +++ b/debian/postgresql-16.lintian-overrides @@ -0,0 +1,20 @@ +# We test for /usr/bin/pg_dropcluster, but run it without path +command-with-path-in-maintainer-script + +# The World's Most Advanced Open Source Relational Database +description-synopsis-starts-with-article + +# We ship binaries and libs in subdirs of /usr/lib/postgresql +executable-in-usr-lib +repeated-path-segment lib * + +# These are PostgreSQL server plugins; some need no external libraries +hardening-no-fortify-functions [usr/lib/postgresql/*/lib/*] +library-not-linked-against-libc [usr/lib/postgresql/*/lib/*] +shared-library-lacks-prerequisites [usr/lib/postgresql/*/lib/*] + +# We use debconf in postrm only +no-debconf-config + +# We store the PostgreSQL catalog version in a custom control field +unknown-field *Postgresql-Catversion diff --git a/debian/postgresql-16.postinst b/debian/postgresql-16.postinst new file mode 100644 index 00000000000..be698435c3f --- /dev/null +++ b/debian/postgresql-16.postinst @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +if [ "$1" = configure ]; then + . /usr/share/postgresql-common/maintscripts-functions + + configure_version $VERSION "$2" +fi + +#DEBHELPER# diff --git a/debian/postgresql-16.postrm b/debian/postgresql-16.postrm new file mode 100644 index 00000000000..c9f88934327 --- /dev/null +++ b/debian/postgresql-16.postrm @@ -0,0 +1,80 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +clean_dir() { + if [ -d "$1" ] && [ ! -L "$1" ]; then + rmdir "$1" >/dev/null 2>/dev/null || true + fi +} + +drop_cluster() { + # if we still have the postgresql-common package, use it to also shutdown + # server, etc.; otherwise just remove the directories + if [ -x /usr/bin/pg_dropcluster ]; then + pg_dropcluster --stop-server $VERSION "$1" + else + # remove data directory + PGDATALINK="/etc/postgresql/$VERSION/$1/pgdata" + if [ -e "$PGDATALINK" ]; then + rm -rf $(readlink -f "$PGDATALINK") "$PGDATALINK" + else + rm -rf "/var/lib/postgresql/$VERSION/$1/" + fi + + # remove log file, including rotated ones + LOGLINK="/etc/postgresql/$VERSION/$1/log" + if [ -e "$LOGLINK" ]; then + LOG=$(readlink -f "$LOGLINK") + rm -f $LOG* "$LOGLINK" + else + rm -f /var/log/postgresql/postgresql-$VERSION-"$1".log* + fi + + # remove conffiles + for f in pg_hba.conf pg_ident.conf postgresql.conf start.conf environment pg_ctl.conf; do + rm -f /etc/postgresql/$VERSION/"$1"/$f + done + # remove empty conf.d directories + for d in /etc/postgresql/$VERSION/"$1"/*/; do + clean_dir "$d" + done + + clean_dir /etc/postgresql/$VERSION/"$1" + fi +} + +purge_package () { + # ask the user if they want to remove clusters. If debconf is not + # available, just remove everything + if [ -e /usr/share/debconf/confmodule ]; then + db_set $DPKG_MAINTSCRIPT_PACKAGE/postrm_purge_data true + db_input high $DPKG_MAINTSCRIPT_PACKAGE/postrm_purge_data || : + db_go || : + db_get $DPKG_MAINTSCRIPT_PACKAGE/postrm_purge_data || : + [ "$RET" = "false" ] && return 0 + fi + + for c in /etc/postgresql/$VERSION/*; do + [ -e "$c/postgresql.conf" ] || continue + cluster=$(basename "$c") + echo "Dropping cluster $cluster..." + drop_cluster "$cluster" + done + + clean_dir /etc/postgresql/$VERSION + clean_dir /var/lib/postgresql/$VERSION + clean_dir /var/log/postgresql/$VERSION +} + +if [ "$1" = purge ] && [ -d "/etc/postgresql/$VERSION" ] && [ "$(ls /etc/postgresql/$VERSION)" ]; then + # can't load debconf from a function + if [ -e /usr/share/debconf/confmodule ]; then + . /usr/share/debconf/confmodule + fi + purge_package +fi + +#DEBHELPER# diff --git a/debian/postgresql-16.preinst b/debian/postgresql-16.preinst new file mode 100644 index 00000000000..a1bccbc9191 --- /dev/null +++ b/debian/postgresql-16.preinst @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +MAJOR_VER="${DPKG_MAINTSCRIPT_PACKAGE#postgresql-}" +CATVERSION="@CATVERSION@" # set by override_dh_installdeb + +case $1 in + install|upgrade) + if [ "$2" ]; then + . /usr/share/postgresql-common/maintscripts-functions + preinst_check_catversion "$MAJOR_VER" "$CATVERSION" + fi ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/debian/postgresql-16.prerm b/debian/postgresql-16.prerm new file mode 100644 index 00000000000..828c6fbe9f5 --- /dev/null +++ b/debian/postgresql-16.prerm @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +#DEBHELPER# + +. /usr/share/postgresql-common/maintscripts-functions + +stop_version $VERSION + +if [ "$1" = remove ]; then + remove_version $VERSION +fi + diff --git a/debian/postgresql-16.templates b/debian/postgresql-16.templates new file mode 100644 index 00000000000..41b8e45186a --- /dev/null +++ b/debian/postgresql-16.templates @@ -0,0 +1,7 @@ +Template: postgresql-16/postrm_purge_data +Type: boolean +Default: true +_Description: Remove PostgreSQL directories when package is purged? + Removing the PostgreSQL server package will leave existing database clusters + intact, i.e. their configuration, data, and log directories will not be + removed. On purging the package, the directories can optionally be removed. diff --git a/debian/postgresql-client-16.install b/debian/postgresql-client-16.install new file mode 100644 index 00000000000..098585bd3cc --- /dev/null +++ b/debian/postgresql-client-16.install @@ -0,0 +1,45 @@ +usr/lib/postgresql/*/bin/clusterdb +usr/lib/postgresql/*/bin/createdb +usr/lib/postgresql/*/bin/createuser +usr/lib/postgresql/*/bin/dropdb +usr/lib/postgresql/*/bin/dropuser +usr/lib/postgresql/*/bin/pg_amcheck +usr/lib/postgresql/*/bin/pg_basebackup +usr/lib/postgresql/*/bin/pg_config +usr/lib/postgresql/*/bin/pg_dump +usr/lib/postgresql/*/bin/pg_dumpall +usr/lib/postgresql/*/bin/pg_isready +usr/lib/postgresql/*/bin/pg_receivewal +usr/lib/postgresql/*/bin/pg_recvlogical +usr/lib/postgresql/*/bin/pg_restore +usr/lib/postgresql/*/bin/pg_verifybackup +usr/lib/postgresql/*/bin/psql +usr/lib/postgresql/*/bin/reindexdb +usr/lib/postgresql/*/bin/vacuumdb +usr/lib/postgresql/*/lib/pgxs/* +usr/share/locale/*/LC_MESSAGES/pg_amcheck-*.mo +usr/share/locale/*/LC_MESSAGES/pg_basebackup-*.mo +usr/share/locale/*/LC_MESSAGES/pg_config-*.mo +usr/share/locale/*/LC_MESSAGES/pg_dump-*.mo +usr/share/locale/*/LC_MESSAGES/pg_verifybackup-*.mo +usr/share/locale/*/LC_MESSAGES/pgscripts-*.mo +usr/share/locale/*/LC_MESSAGES/psql-*.mo +usr/share/postgresql/*/man/man1/clusterdb.1* +usr/share/postgresql/*/man/man1/createdb.1* +usr/share/postgresql/*/man/man1/createuser.1* +usr/share/postgresql/*/man/man1/dropdb.1* +usr/share/postgresql/*/man/man1/dropuser.1* +usr/share/postgresql/*/man/man1/pg_amcheck.1* +usr/share/postgresql/*/man/man1/pg_basebackup.1* +usr/share/postgresql/*/man/man1/pg_dump.1* +usr/share/postgresql/*/man/man1/pg_dumpall.1* +usr/share/postgresql/*/man/man1/pg_isready.1* +usr/share/postgresql/*/man/man1/pg_receivewal.1* +usr/share/postgresql/*/man/man1/pg_recvlogical.1* +usr/share/postgresql/*/man/man1/pg_restore.1* +usr/share/postgresql/*/man/man1/pg_verifybackup.1* +usr/share/postgresql/*/man/man1/psql.1* +usr/share/postgresql/*/man/man1/reindexdb.1* +usr/share/postgresql/*/man/man1/vacuumdb.1* +usr/share/postgresql/*/man/man7/ +usr/share/postgresql/*/psqlrc.sample diff --git a/debian/postgresql-client-16.lintian-overrides b/debian/postgresql-client-16.lintian-overrides new file mode 100644 index 00000000000..a3cef5bfc6c --- /dev/null +++ b/debian/postgresql-client-16.lintian-overrides @@ -0,0 +1,3 @@ +# We ship binaries and libs in subdirs of /usr/lib/postgresql +executable-in-usr-lib +repeated-path-segment lib * diff --git a/debian/postgresql-client-16.postinst b/debian/postgresql-client-16.postinst new file mode 100644 index 00000000000..a5cf251f4f4 --- /dev/null +++ b/debian/postgresql-client-16.postinst @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +. /usr/share/postgresql-common/maintscripts-functions + +if [ "$1" = configure ]; then + configure_client_version $VERSION "$2" +fi + +#DEBHELPER# diff --git a/debian/postgresql-client-16.prerm b/debian/postgresql-client-16.prerm new file mode 100644 index 00000000000..14c21887943 --- /dev/null +++ b/debian/postgresql-client-16.prerm @@ -0,0 +1,12 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +#DEBHELPER# + +if [ "$1" = remove ]; then + . /usr/share/postgresql-common/maintscripts-functions + remove_client_version $VERSION +fi diff --git a/debian/postgresql-doc-16.doc-base b/debian/postgresql-doc-16.doc-base new file mode 100644 index 00000000000..42749e336cd --- /dev/null +++ b/debian/postgresql-doc-16.doc-base @@ -0,0 +1,18 @@ +Document: postgresql-16 +Title: PostgreSQL 16 Documentation +Author: The PostgreSQL Global Development Group +Abstract: The documentation for the PostgreSQL database management system, + version 16. PostgreSQL is a powerful, open source object-relational database + system. It is fully ACID compliant, has full support for foreign keys, joins, + views, triggers, and stored procedures (in multiple languages). It includes + most SQL:2008 data types, including INTEGER, NUMERIC, BOOLEAN, CHAR, VARCHAR, + DATE, INTERVAL, and TIMESTAMP. It also supports storage of binary large + objects, including pictures, sounds, or video. It has native programming + interfaces for C/C++, Java, .Net, Perl, Python, Ruby, Tcl, ODBC, among others, + and exceptional documentation. +Section: Data Management + +Format: HTML +Index: /usr/share/doc/postgresql-doc-16/html/index.html +Files: /usr/share/doc/postgresql-doc-16/html/* + diff --git a/debian/postgresql-doc-16.install b/debian/postgresql-doc-16.install new file mode 100644 index 00000000000..a41ef190d3f --- /dev/null +++ b/debian/postgresql-doc-16.install @@ -0,0 +1,2 @@ +usr/share/doc/postgresql-doc-* +usr/share/postgresql/*/man/man3/ diff --git a/debian/postgresql-doc-16.postinst b/debian/postgresql-doc-16.postinst new file mode 100644 index 00000000000..d647cf43e8d --- /dev/null +++ b/debian/postgresql-doc-16.postinst @@ -0,0 +1,30 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +# arguments: version master package [package] +__link_manpages() { + MANS=$(unset GREP_OPTIONS; dpkg -L $3 $4 $5 2>/dev/null | grep -E '/man/.*\.[1-9](\.gz)?$' | grep -v "$2") || true + [ -n "$MANS" ] || return 0 + + SLAVES=$(for i in $MANS; do TARGET=$(echo $i | sed "s/postgresql\/$1\///"); echo -n " --slave $TARGET $(basename $i) $i"; done) + + mkdir -p /usr/share/man/man3 + section=$(echo "$2" | sed -e 's/.*\.\(.*\)\..*/man\1/') + update-alternatives --install /usr/share/man/$section/$2 \ + $2 /usr/share/postgresql/$1/man/$section/$2 \ + ${1}0 $SLAVES +} + +if [ "$1" = configure ]; then + if [ -f /usr/share/postgresql-common/maintscripts-functions ]; then + . /usr/share/postgresql-common/maintscripts-functions + configure_doc_version $VERSION "$2" + else + __link_manpages $VERSION SPI_connect.3.gz "postgresql-doc-$VERSION" + fi +fi + +#DEBHELPER# diff --git a/debian/postgresql-doc-16.prerm b/debian/postgresql-doc-16.prerm new file mode 100644 index 00000000000..9b722399d3a --- /dev/null +++ b/debian/postgresql-doc-16.prerm @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +if [ "$1" = remove ]; then + if [ -f /usr/share/postgresql-common/maintscripts-functions ]; then + . /usr/share/postgresql-common/maintscripts-functions + remove_doc_version $VERSION + else + update-alternatives --remove SPI_connect.3.gz \ + /usr/share/postgresql/$VERSION/man/man3/SPI_connect.3.gz + fi +fi + +#DEBHELPER# diff --git a/debian/postgresql-plperl-16.install b/debian/postgresql-plperl-16.install new file mode 100755 index 00000000000..c4bac49a566 --- /dev/null +++ b/debian/postgresql-plperl-16.install @@ -0,0 +1,6 @@ +#!/usr/bin/dh-exec + +usr/lib/postgresql/*/lib/*plperl*.so +[!alpha !hppa !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] usr/lib/postgresql/*/lib/bitcode/*plperl* +usr/share/locale/*/*/plperl-*.mo +usr/share/postgresql/*/extension/*plperl* diff --git a/debian/postgresql-plperl-16.lintian-overrides b/debian/postgresql-plperl-16.lintian-overrides new file mode 120000 index 00000000000..7bab0e9e736 --- /dev/null +++ b/debian/postgresql-plperl-16.lintian-overrides @@ -0,0 +1 @@ +postgresql-16.lintian-overrides \ No newline at end of file diff --git a/debian/postgresql-plpython3-16.install b/debian/postgresql-plpython3-16.install new file mode 100755 index 00000000000..4632547eba2 --- /dev/null +++ b/debian/postgresql-plpython3-16.install @@ -0,0 +1,6 @@ +#!/usr/bin/dh-exec + +usr/lib/postgresql/*/lib/*plpython3*.so +[!alpha !hppa !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] usr/lib/postgresql/*/lib/bitcode/*plpython3* +usr/share/locale/*/*/plpython-*.mo +usr/share/postgresql/*/extension/*plpython3* diff --git a/debian/postgresql-plpython3-16.lintian-overrides b/debian/postgresql-plpython3-16.lintian-overrides new file mode 120000 index 00000000000..7bab0e9e736 --- /dev/null +++ b/debian/postgresql-plpython3-16.lintian-overrides @@ -0,0 +1 @@ +postgresql-16.lintian-overrides \ No newline at end of file diff --git a/debian/postgresql-pltcl-16.install b/debian/postgresql-pltcl-16.install new file mode 100644 index 00000000000..f56fc0fb4d4 --- /dev/null +++ b/debian/postgresql-pltcl-16.install @@ -0,0 +1,3 @@ +usr/lib/postgresql/*/lib/pltcl.so +usr/share/locale/*/*/pltcl-*.mo +usr/share/postgresql/*/extension/pltcl* diff --git a/debian/postgresql-pltcl-16.lintian-overrides b/debian/postgresql-pltcl-16.lintian-overrides new file mode 120000 index 00000000000..7bab0e9e736 --- /dev/null +++ b/debian/postgresql-pltcl-16.lintian-overrides @@ -0,0 +1 @@ +postgresql-16.lintian-overrides \ No newline at end of file diff --git a/debian/postgresql-server-dev-16.install b/debian/postgresql-server-dev-16.install new file mode 100644 index 00000000000..4c84b492909 --- /dev/null +++ b/debian/postgresql-server-dev-16.install @@ -0,0 +1,2 @@ +usr/include/postgresql/*/server +usr/lib/postgresql/*/lib/libpg*.a diff --git a/debian/postgresql-server-dev-16.lintian-overrides b/debian/postgresql-server-dev-16.lintian-overrides new file mode 120000 index 00000000000..7bab0e9e736 --- /dev/null +++ b/debian/postgresql-server-dev-16.lintian-overrides @@ -0,0 +1 @@ +postgresql-16.lintian-overrides \ No newline at end of file diff --git a/debian/rules b/debian/rules new file mode 100755 index 00000000000..e8d808c90e3 --- /dev/null +++ b/debian/rules @@ -0,0 +1,5 @@ +#!/usr/bin/make -f + +MAJOR_VER := 16 + +include /usr/share/postgresql-common/server/postgresql.mk diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 00000000000..163aaf8d82b --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/source/lintian-overrides b/debian/source/lintian-overrides new file mode 100644 index 00000000000..a244735a805 --- /dev/null +++ b/debian/source/lintian-overrides @@ -0,0 +1,4 @@ +# pregenerated docs contain some tables rendered on a single line +source: source-is-missing [doc/src/sgml/html/*] +# same for these, plus including some regression output files +source: very-long-line-length-in-source-file * diff --git a/debian/tests/Makefile.regress b/debian/tests/Makefile.regress new file mode 100644 index 00000000000..f01a80a52cf --- /dev/null +++ b/debian/tests/Makefile.regress @@ -0,0 +1,5 @@ +MODULE_big = regress +OBJS = regress.o +PG_CONFIG = pg_config +PGXS = $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) diff --git a/debian/tests/control b/debian/tests/control new file mode 100644 index 00000000000..c7406c21e99 --- /dev/null +++ b/debian/tests/control @@ -0,0 +1,20 @@ +Tests: run-testsuite +Depends: + build-essential, + debhelper, + fakeroot, + hunspell-en-us, + iproute2, + locales-all, + logrotate, + netcat-openbsd, + perl, + procps, + @, +Restrictions: needs-root + +Tests: installcheck +Depends: + build-essential, + @, +Restrictions: allow-stderr diff --git a/debian/tests/installcheck b/debian/tests/installcheck new file mode 100755 index 00000000000..fc390ea14a9 --- /dev/null +++ b/debian/tests/installcheck @@ -0,0 +1,41 @@ +#!/bin/sh + +set -eux + +SOURCE=$(dpkg-parsechangelog -SSource) +MAJOR=${SOURCE#*-} +top_srcdir=$PWD + +cd src/test/regress + +# compile regress.so +make -f $top_srcdir/debian/tests/Makefile.regress PG_CONFIG=/usr/lib/postgresql/$MAJOR/bin/pg_config with_llvm=no + +# tell regression files that regress.so is not installed +sed -i -e "s;set regresslib :libdir;set regresslib '$PWD';" sql/* expected/* + +# when root, execute testsuite as user postgres since it insists on wiping the tablespace directory +if [ $(id -u) = 0 ]; then + SU="su postgres" +else + SU="sh" +fi + +$SU < Date: Wed, 13 Nov 2024 11:54:20 +0000 Subject: [PATCH 16/54] Indeed remove stop version from prerm --- debian/postgresql-16.prerm | 2 -- 1 file changed, 2 deletions(-) diff --git a/debian/postgresql-16.prerm b/debian/postgresql-16.prerm index 828c6fbe9f5..f8d6ac10a20 100644 --- a/debian/postgresql-16.prerm +++ b/debian/postgresql-16.prerm @@ -8,8 +8,6 @@ VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} . /usr/share/postgresql-common/maintscripts-functions -stop_version $VERSION - if [ "$1" = remove ]; then remove_version $VERSION fi From 1cbe8b6cf60a624ae36b606dfa2f4be4ed32eac8 Mon Sep 17 00:00:00 2001 From: reshke Date: Mon, 31 Jul 2023 09:09:37 +0000 Subject: [PATCH 17/54] Git apply debian patches[16] --- HISTORY | 3 + configure.ac | 9 +- debian/patches/autoconf2.69 | 2 +- src/Makefile.global.in | 4 +- src/backend/commands/extension.c | 121 ++++++++++++++++++++ src/backend/jit/llvm/llvmjit.c | 48 +++++++- src/backend/utils/fmgr/dfmgr.c | 29 ++++- src/backend/utils/misc/guc_tables.c | 12 ++ src/bin/psql/settings.h | 4 +- src/common/Makefile | 12 +- src/fe_utils/Makefile | 6 +- src/include/fe_utils/print.h | 2 +- src/include/pg_config_manual.h | 2 +- src/include/utils/guc.h | 1 + src/interfaces/libpq/pg_service.conf.sample | 4 +- src/port/Makefile | 10 +- src/tutorial/README | 3 +- 17 files changed, 240 insertions(+), 32 deletions(-) diff --git a/HISTORY b/HISTORY index b87be55abf4..f12fd89e960 100644 --- a/HISTORY +++ b/HISTORY @@ -3,3 +3,6 @@ https://www.postgresql.org/docs/current/release.html Distribution file sets include release notes for their version and preceding versions. Visit the file doc/src/sgml/html/release.html in an HTML browser. + +On Debian systems, the release notes are contained in the postgresql-doc-* +packages, located in /usr/share/doc/postgresql-doc-*/html/release.html. diff --git a/configure.ac b/configure.ac index b3d5efcde3d..a9de7922c90 100644 --- a/configure.ac +++ b/configure.ac @@ -19,14 +19,11 @@ m4_pattern_forbid(^PGAC_)dnl to catch undefined macros AC_INIT([PostgreSQL], [16.13], [pgsql-bugs@lists.postgresql.org], [], [https://www.postgresql.org/]) -m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.69 is required. -Untested combinations of 'autoconf' and PostgreSQL versions are not -recommended. You can remove the check from 'configure.ac' but it is then -your responsibility whether the result works or not.])]) AC_COPYRIGHT([Copyright (c) 1996-2023, PostgreSQL Global Development Group]) AC_CONFIG_SRCDIR([src/backend/access/common/heaptuple.c]) AC_CONFIG_AUX_DIR(config) AC_PREFIX_DEFAULT(/usr/local/pgsql) +[ac_configure_args=$(echo "$ac_configure_args" | sed -e "s/ -f\(debug\|file\)-prefix-map=[^' ]*//g")] AC_DEFINE_UNQUOTED(CONFIGURE_ARGS, ["$ac_configure_args"], [Saved arguments from configure]) [PG_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\)'`] @@ -579,6 +576,10 @@ if test "$GCC" = yes -a "$ICC" = no; then if test -n "$NOT_THE_CFLAGS"; then CFLAGS="$CFLAGS -Wno-cast-function-type-strict" fi + if test x"$host_cpu" == x"aarch64"; then + PGAC_PROG_CC_CFLAGS_OPT([-moutline-atomics]) + PGAC_PROG_CXX_CFLAGS_OPT([-moutline-atomics]) + fi elif test "$ICC" = yes; then # Intel's compiler has a bug/misoptimization in checking for # division by NAN (NaN == 0), -mp1 fixes it, so add it to the CFLAGS. diff --git a/debian/patches/autoconf2.69 b/debian/patches/autoconf2.69 index 429044e698b..e5d91834a36 100644 --- a/debian/patches/autoconf2.69 +++ b/debian/patches/autoconf2.69 @@ -1,6 +1,6 @@ --- a/configure.ac +++ b/configure.ac -@@ -22,4 +21,0 @@ AC_INIT([PostgreSQL], [15devel], [pgsql- +@@ -21,4 +20,0 @@ AC_INIT([PostgreSQL], [16beta2], [pgsql- -m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.69 is required. -Untested combinations of 'autoconf' and PostgreSQL versions are not -recommended. You can remove the check from 'configure.ac' but it is then diff --git a/src/Makefile.global.in b/src/Makefile.global.in index ce05cc1429a..c61c8e835d8 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -119,7 +119,7 @@ libdir := @libdir@ pkglibdir = $(libdir) ifeq "$(findstring pgsql, $(pkglibdir))" "" ifeq "$(findstring postgres, $(pkglibdir))" "" -override pkglibdir := $(pkglibdir)/postgresql +override pkglibdir := /usr/lib/postgresql/@PG_MAJORVERSION@/lib endif endif @@ -167,7 +167,7 @@ endif # PGXS # These derived path variables aren't separately configurable. -includedir_server = $(pkgincludedir)/server +includedir_server = $(pkgincludedir)/@PG_MAJORVERSION@/server includedir_internal = $(pkgincludedir)/internal pgxsdir = $(pkglibdir)/pgxs bitcodedir = $(pkglibdir)/bitcode diff --git a/src/backend/commands/extension.c b/src/backend/commands/extension.c index 55c1d28307b..69279934c47 100644 --- a/src/backend/commands/extension.c +++ b/src/backend/commands/extension.c @@ -154,6 +154,8 @@ static void ApplyExtensionUpdates(Oid extensionOid, bool cascade, bool is_create); static char *read_whole_file(const char *filename, int *length); +static bool file_exists(const char *name); +static bool directory_exists(const char *dir); /* @@ -522,6 +524,16 @@ get_extension_control_filename(const char *extname) get_share_path(my_exec_path, sharepath); result = (char *) palloc(MAXPGPATH); + /* + * If extension_destdir is set, try to find the file there first + */ + if (*extension_destdir != '\0') + { + snprintf(result, MAXPGPATH, "%s%s/extension/%s.control", + extension_destdir, sharepath, extname); + if (file_exists(result)) + return result; + } snprintf(result, MAXPGPATH, "%s/extension/%s.control", sharepath, extname); @@ -561,6 +573,16 @@ get_extension_aux_control_filename(ExtensionControlFile *control, scriptdir = get_extension_script_directory(control); result = (char *) palloc(MAXPGPATH); + /* + * If extension_destdir is set, try to find the file there first + */ + if (*extension_destdir != '\0') + { + snprintf(result, MAXPGPATH, "%s%s/%s--%s.control", + extension_destdir, scriptdir, control->name, version); + if (file_exists(result)) + return result; + } snprintf(result, MAXPGPATH, "%s/%s--%s.control", scriptdir, control->name, version); @@ -579,6 +601,23 @@ get_extension_script_filename(ExtensionControlFile *control, scriptdir = get_extension_script_directory(control); result = (char *) palloc(MAXPGPATH); + /* + * If extension_destdir is set, try to find the file there first + */ + if (*extension_destdir != '\0') + { + if (from_version) + snprintf(result, MAXPGPATH, "%s%s/%s--%s--%s.sql", + extension_destdir, scriptdir, control->name, from_version, version); + else + snprintf(result, MAXPGPATH, "%s%s/%s--%s.sql", + extension_destdir, scriptdir, control->name, version); + if (file_exists(result)) + { + pfree(scriptdir); + return result; + } + } if (from_version) snprintf(result, MAXPGPATH, "%s/%s--%s--%s.sql", scriptdir, control->name, from_version, version); @@ -1343,6 +1382,59 @@ get_ext_ver_list(ExtensionControlFile *control) DIR *dir; struct dirent *de; + /* + * If extension_destdir is set, try to find the files there first + */ + if (*extension_destdir != '\0') + { + char location[MAXPGPATH]; + + snprintf(location, MAXPGPATH, "%s%s", extension_destdir, + get_extension_script_directory(control)); + dir = AllocateDir(location); + while ((de = ReadDir(dir, location)) != NULL) + { + char *vername; + char *vername2; + ExtensionVersionInfo *evi; + ExtensionVersionInfo *evi2; + + /* must be a .sql file ... */ + if (!is_extension_script_filename(de->d_name)) + continue; + + /* ... matching extension name followed by separator */ + if (strncmp(de->d_name, control->name, extnamelen) != 0 || + de->d_name[extnamelen] != '-' || + de->d_name[extnamelen + 1] != '-') + continue; + + /* extract version name(s) from 'extname--something.sql' filename */ + vername = pstrdup(de->d_name + extnamelen + 2); + *strrchr(vername, '.') = '\0'; + vername2 = strstr(vername, "--"); + if (!vername2) + { + /* It's an install, not update, script; record its version name */ + evi = get_ext_ver_info(vername, &evi_list); + evi->installable = true; + continue; + } + *vername2 = '\0'; /* terminate first version */ + vername2 += 2; /* and point to second */ + + /* if there's a third --, it's bogus, ignore it */ + if (strstr(vername2, "--")) + continue; + + /* Create ExtensionVersionInfos and link them together */ + evi = get_ext_ver_info(vername, &evi_list); + evi2 = get_ext_ver_info(vername2, &evi_list); + evi->reachable = lappend(evi->reachable, evi2); + } + FreeDir(dir); + } + location = get_extension_script_directory(control); dir = AllocateDir(location); while ((de = ReadDir(dir, location)) != NULL) @@ -3624,3 +3716,32 @@ read_whole_file(const char *filename, int *length) buf[*length] = '\0'; return buf; } + +static bool +file_exists(const char *name) +{ + struct stat st; + + Assert(name != NULL); + + if (stat(name, &st) == 0) + return S_ISDIR(st.st_mode) ? false : true; + else if (!(errno == ENOENT || errno == ENOTDIR || errno == EACCES)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not access file \"%s\": %m", name))); + + return false; +} + +static bool +directory_exists(const char *dir) +{ + struct stat st; + + if (stat(dir, &st) != 0) + return false; + if (S_ISDIR(st.st_mode)) + return true; + return false; +} diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c index c96e2d87ac6..34403044a0e 100644 --- a/src/backend/jit/llvm/llvmjit.c +++ b/src/backend/jit/llvm/llvmjit.c @@ -896,6 +896,37 @@ llvm_compile_module(LLVMJitContext *context) errhidecontext(true))); } +/* + * For the systemz target, LLVM uses a different datalayout for z13 and newer + * CPUs than it does for older CPUs. This can cause a mismatch in datalayouts + * in the case where the llvm_types_module is compiled with a pre-z13 CPU + * and the JIT is running on z13 or newer. + * See computeDataLayout() function in + * llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp for information on the + * datalayout differences. + */ +static bool +needs_systemz_workaround(void) +{ + bool ret = false; +#ifdef __s390x__ + LLVMContextRef llvm_context; + LLVMTypeRef vec_type; + LLVMTargetDataRef llvm_layoutref; + if (strncmp(LLVMGetTargetName(llvm_targetref), "systemz", strlen("systemz"))) + { + return false; + } + + llvm_context = LLVMGetModuleContext(llvm_types_module); + vec_type = LLVMVectorType(LLVMIntTypeInContext(llvm_context, 32), 4); + llvm_layoutref = LLVMCreateTargetData(llvm_layout); + ret = (LLVMABIAlignmentOfType(llvm_layoutref, vec_type) == 16); + LLVMDisposeTargetData(llvm_layoutref); +#endif + return ret; +} + /* * Per session initialization. */ @@ -905,6 +936,7 @@ llvm_session_initialize(void) MemoryContext oldcontext; char *error = NULL; char *cpu = NULL; + char *host_features = NULL; char *features = NULL; LLVMTargetMachineRef opt0_tm; LLVMTargetMachineRef opt3_tm; @@ -962,10 +994,17 @@ llvm_session_initialize(void) * features not all CPUs have (weird, huh). */ cpu = LLVMGetHostCPUName(); - features = LLVMGetHostCPUFeatures(); + features = host_features = LLVMGetHostCPUFeatures(); elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"", cpu, features); + if (needs_systemz_workaround()) + { + const char *no_vector =",-vector"; + features = malloc(sizeof(char) * (strlen(host_features) + strlen(no_vector) + 1)); + sprintf(features, "%s%s", host_features, no_vector); + } + opt0_tm = LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features, LLVMCodeGenLevelNone, @@ -979,8 +1018,13 @@ llvm_session_initialize(void) LLVMDisposeMessage(cpu); cpu = NULL; - LLVMDisposeMessage(features); + if (features != host_features) + { + free(features); + } features = NULL; + LLVMDisposeMessage(host_features); + host_features = NULL; /* force symbols in main binary to be loaded */ LLVMLoadLibraryPermanently(NULL); diff --git a/src/backend/utils/fmgr/dfmgr.c b/src/backend/utils/fmgr/dfmgr.c index b85d52c913c..758cac13445 100644 --- a/src/backend/utils/fmgr/dfmgr.c +++ b/src/backend/utils/fmgr/dfmgr.c @@ -34,6 +34,7 @@ #include "lib/stringinfo.h" #include "miscadmin.h" #include "storage/shmem.h" +#include "utils/guc.h" #include "utils/hsearch.h" @@ -432,7 +433,7 @@ expand_dynamic_library_name(const char *name) { bool have_slash; char *new; - char *full; + char *full, *full2; Assert(name); @@ -447,6 +448,19 @@ expand_dynamic_library_name(const char *name) else { full = substitute_libpath_macro(name); + /* + * If extension_destdir is set, try to find the file there first + */ + if (*extension_destdir != '\0') + { + full2 = psprintf("%s%s", extension_destdir, full); + if (file_exists(full2)) + { + pfree(full); + return full2; + } + pfree(full2); + } if (file_exists(full)) return full; pfree(full); @@ -465,6 +479,19 @@ expand_dynamic_library_name(const char *name) { full = substitute_libpath_macro(new); pfree(new); + /* + * If extension_destdir is set, try to find the file there first + */ + if (*extension_destdir != '\0') + { + full2 = psprintf("%s%s", extension_destdir, full); + if (file_exists(full2)) + { + pfree(full); + return full2; + } + pfree(full2); + } if (file_exists(full)) return full; pfree(full); diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 721188a88ba..8bcb030c3de 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -549,6 +549,7 @@ char *ConfigFileName; char *HbaFileName; char *IdentFileName; char *external_pid_file; +char *extension_destdir; char *application_name; @@ -4376,6 +4377,17 @@ struct config_string ConfigureNamesString[] = check_canonical_path, NULL, NULL }, + { + {"extension_destdir", PGC_SUSET, FILE_LOCATIONS, + gettext_noop("Path to prepend for extension loading."), + gettext_noop("This directory is prepended to paths when loading extensions (control and SQL files), and to the '$libdir' directive when loading modules that back functions. The location is made configurable to allow build-time testing of extensions that do not have been installed to their proper location yet."), + GUC_SUPERUSER_ONLY + }, + &extension_destdir, + "", + NULL, NULL, NULL + }, + { {"ssl_library", PGC_INTERNAL, PRESET_OPTIONS, gettext_noop("Shows the name of the SSL library."), diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h index 1106954236d..0892aa18cc8 100644 --- a/src/bin/psql/settings.h +++ b/src/bin/psql/settings.h @@ -19,8 +19,8 @@ #define DEFAULT_EDITOR "notepad.exe" /* no DEFAULT_EDITOR_LINENUMBER_ARG for Notepad */ #else -#define DEFAULT_EDITOR "vi" -#define DEFAULT_EDITOR_LINENUMBER_ARG "+" +#define DEFAULT_EDITOR "sensible-editor" +/*#define DEFAULT_EDITOR_LINENUMBER_ARG "+"*/ #endif #define DEFAULT_PROMPT1 "%/%R%x%# " diff --git a/src/common/Makefile b/src/common/Makefile index 113029bf7b9..2d5563b65a8 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -33,7 +33,7 @@ STD_CPPFLAGS := $(filter-out -I$(top_srcdir)/src/include -I$(top_builddir)/src/i STD_LDFLAGS := $(filter-out -L$(top_builddir)/src/common -L$(top_builddir)/src/port,$(LDFLAGS)) override CPPFLAGS += -DVAL_CC="\"$(CC)\"" override CPPFLAGS += -DVAL_CPPFLAGS="\"$(STD_CPPFLAGS)\"" -override CPPFLAGS += -DVAL_CFLAGS="\"$(CFLAGS)\"" +override CPPFLAGS += -DVAL_CFLAGS="\"$(filter-out -fdebug-prefix-map=% -ffile-prefix-map=%,$(CFLAGS))\"" override CPPFLAGS += -DVAL_CFLAGS_SL="\"$(CFLAGS_SL)\"" override CPPFLAGS += -DVAL_LDFLAGS="\"$(STD_LDFLAGS)\"" override CPPFLAGS += -DVAL_LDFLAGS_EX="\"$(LDFLAGS_EX)\"" @@ -125,15 +125,15 @@ distprep: kwlist_d.h # libpgcommon is needed by some contrib install: all installdirs - $(INSTALL_STLIB) libpgcommon.a '$(DESTDIR)$(libdir)/libpgcommon.a' - $(INSTALL_STLIB) libpgcommon_shlib.a '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' + $(INSTALL_STLIB) libpgcommon.a '$(DESTDIR)$(pkglibdir)/libpgcommon.a' + $(INSTALL_STLIB) libpgcommon_shlib.a '$(DESTDIR)$(pkglibdir)/libpgcommon_shlib.a' installdirs: - $(MKDIR_P) '$(DESTDIR)$(libdir)' + $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' uninstall: - rm -f '$(DESTDIR)$(libdir)/libpgcommon.a' - rm -f '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgcommon.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgcommon_shlib.a' libpgcommon.a: $(OBJS_FRONTEND) rm -f $@ diff --git a/src/fe_utils/Makefile b/src/fe_utils/Makefile index 456d6dd3904..7f4cab147cc 100644 --- a/src/fe_utils/Makefile +++ b/src/fe_utils/Makefile @@ -52,13 +52,13 @@ distprep: psqlscan.c # libpgfeutils could be useful to contrib, so install it install: all installdirs - $(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(libdir)/libpgfeutils.a' + $(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(pkglibdir)/libpgfeutils.a' installdirs: - $(MKDIR_P) '$(DESTDIR)$(libdir)' + $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' uninstall: - rm -f '$(DESTDIR)$(libdir)/libpgfeutils.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgfeutils.a' clean distclean: rm -f libpgfeutils.a $(OBJS) lex.backup diff --git a/src/include/fe_utils/print.h b/src/include/fe_utils/print.h index cc6652def9e..30a4e6f519f 100644 --- a/src/include/fe_utils/print.h +++ b/src/include/fe_utils/print.h @@ -20,7 +20,7 @@ /* This is not a particularly great place for this ... */ #ifndef __CYGWIN__ -#define DEFAULT_PAGER "more" +#define DEFAULT_PAGER "pager" #else #define DEFAULT_PAGER "less" #endif diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h index a1a93ad706e..fc67f6de3df 100644 --- a/src/include/pg_config_manual.h +++ b/src/include/pg_config_manual.h @@ -206,7 +206,7 @@ * support them yet. */ #ifndef WIN32 -#define DEFAULT_PGSOCKET_DIR "/tmp" +#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql" #else #define DEFAULT_PGSOCKET_DIR "" #endif diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index 3c1f78bb992..3abcdca6327 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -275,6 +275,7 @@ extern PGDLLIMPORT char *ConfigFileName; extern PGDLLIMPORT char *HbaFileName; extern PGDLLIMPORT char *IdentFileName; extern PGDLLIMPORT char *external_pid_file; +extern PGDLLIMPORT char *extension_destdir; extern PGDLLIMPORT char *application_name; diff --git a/src/interfaces/libpq/pg_service.conf.sample b/src/interfaces/libpq/pg_service.conf.sample index 5a1c083538b..7ef2ebde701 100644 --- a/src/interfaces/libpq/pg_service.conf.sample +++ b/src/interfaces/libpq/pg_service.conf.sample @@ -8,8 +8,8 @@ # to look up such parameters. A sample configuration for postgres is # included in this file. Lines beginning with '#' are comments. # -# Copy this to your sysconf directory (typically /usr/local/pgsql/etc) and -# rename it pg_service.conf. +# Copy this to /etc/postgresql-common/ (or select its location with the +# PGSYSCONFDIR environment variable) and rename it pg_service.conf. # # #[postgres] diff --git a/src/port/Makefile b/src/port/Makefile index 711f59e32bd..c52a63b2ae1 100644 --- a/src/port/Makefile +++ b/src/port/Makefile @@ -70,15 +70,15 @@ all: libpgport.a libpgport_shlib.a libpgport_srv.a # libpgport is needed by some contrib install: all installdirs - $(INSTALL_STLIB) libpgport.a '$(DESTDIR)$(libdir)/libpgport.a' - $(INSTALL_STLIB) libpgport_shlib.a '$(DESTDIR)$(libdir)/libpgport_shlib.a' + $(INSTALL_STLIB) libpgport.a '$(DESTDIR)$(pkglibdir)/libpgport.a' + $(INSTALL_STLIB) libpgport_shlib.a '$(DESTDIR)$(pkglibdir)/libpgport_shlib.a' installdirs: - $(MKDIR_P) '$(DESTDIR)$(libdir)' + $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' uninstall: - rm -f '$(DESTDIR)$(libdir)/libpgport.a' - rm -f '$(DESTDIR)$(libdir)/libpgport_shlib.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgport.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgport_shlib.a' libpgport.a: $(OBJS) rm -f $@ diff --git a/src/tutorial/README b/src/tutorial/README index b137cdfad34..be070c573db 100644 --- a/src/tutorial/README +++ b/src/tutorial/README @@ -6,8 +6,7 @@ tutorial This directory contains SQL tutorial scripts. To look at them, first do a % make to compile all the scripts and C files for the user-defined functions -and types. (make needs to be GNU make --- it may be named something -different on your system, often 'gmake') +and types. This requires a postgresql-server-dev-* package to be installed. Then, run psql with the -s (single-step) flag: % psql -s From b85c33da2a0fffc85eac28a45825c9ac1e7d76cb Mon Sep 17 00:00:00 2001 From: Dmitry Fedorov Date: Mon, 17 Jul 2023 06:59:00 +0000 Subject: [PATCH 18/54] Pull request #134: MDB-24221: added support for jammy Merge in MDB/postgres-dev from MDB-24221 to MDB_15_STABLE Squashed commit of the following: commit 16d1cd273560f2f9df6b4e7e1b21f8a8667e6c29 Author: Dmitry S. Fedorov Date: Thu Jul 13 11:14:19 2023 +0300 MDB-24221: added support for jammy commit 12f75e39ee084e7ff54297729d7f4162c80e5b22 Author: Dmitry S. Fedorov Date: Thu Jul 13 11:14:19 2023 +0300 MDB-24221: added support for jammy Add docker entrypoint and util script Add prepare-build script enable tests --- Dockerfile | 53 ++++++++++++++++++++++++++++++++++++++++++++ docker/entrypoint.sh | 20 +++++++++++++++++ docker/tzdata.sh | 9 ++++++++ prepare-build.sh | 17 ++++++++++++++ 4 files changed, 99 insertions(+) create mode 100644 Dockerfile create mode 100755 docker/entrypoint.sh create mode 100755 docker/tzdata.sh create mode 100755 prepare-build.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..76faa3a2045 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +ARG codename +FROM ubuntu:${codename:-bionic} + +ARG codename +ENV CODE_NAME=${codename:-bionic} + +ARG pgdg +ENV PGDG_VER=${pgdg:-242-2-pgdg18.04+1+yandex220} + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Europe/Moskow +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN sed -i 's/archive.ubuntu.com/mirror.yandex.ru/g' /etc/apt/sources.list &&\ + apt-get update && apt-get install -y --no-install-recommends \ + sudo build-essential \ + gcc lsb-release libssl-dev gnupg openssl \ + gdb git curl + +RUN echo "deb http://dist.yandex.ru/mdb-${CODE_NAME}-secure stable/all/" >> /etc/apt/sources.list +RUN echo "deb http://dist.yandex.ru/mdb-${CODE_NAME}-secure stable/\$(ARCH)/" >> /etc/apt/sources.list + +RUN curl -s 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xafc3ce0d00e3c45a357e9e637fcd11186050cd1a' | \ + gpg --dearmour -o /etc/apt/trusted.gpg.d/yandex.gpg + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo build-essential \ + gcc lsb-release libssl-dev gnupg openssl \ + gdb git \ + libpam0g-dev \ + debhelper debootstrap devscripts make equivs debhelper-compat \ + libz-dev flex libicu-dev libio-pty-perl libipc-run-perl libkrb5-dev \ + libldap2-dev liblz4-dev liblz4-tool zstd libperl-dev libreadline-dev libselinux1-dev llvm-dev \ + libsystemd-dev libxml2-dev libxml2-utils libxslt1-dev \ + pkg-config python3-dev systemtap-sdt-dev tcl-dev uuid-dev xsltproc zlib1g-dev \ + bison dh-exec docbook-xml docbook-xsl + +RUN apt-get install -y \ + libmdblocales1 libmdblocales-dev \ + postgresql-client-common=${PGDG_VER} \ + postgresql-common=${PGDG_VER} + +RUN groupadd -g 999 build-user && \ + useradd -r -u 999 -g build-user build-user + +COPY . /home/build-user +RUN chown build-user:build-user /home -R && usermod -aG sudo build-user + +RUN echo 'build-user ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER build-user + +ENTRYPOINT ["/home/build-user/docker/entrypoint.sh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000000..fb2d7b3a735 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +export DEBIAN_FRONTEND=noninteractive +export TZ=Europe/Moskow +sudo bash -c "echo $TZ > /etc/timezone" + +cd /home/build-user + +sudo ./docker/tzdata.sh + +cat debian/changelog +#export DEB_BUILD_OPTIONS="nocheck" + +sudo mk-build-deps --build-dep --install --tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes' debian/control + +dpkg-buildpackage -us -uc + +cd /home +rm -fr build-user + diff --git a/docker/tzdata.sh b/docker/tzdata.sh new file mode 100755 index 00000000000..004e6de4101 --- /dev/null +++ b/docker/tzdata.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive +#install tzdata package +apt-get install -y tzdata +# set your timezone +ln -fs /usr/share/zoneinfo/$TZ /etc/localtime +dpkg-reconfigure --frontend noninteractive tzdata + diff --git a/prepare-build.sh b/prepare-build.sh new file mode 100755 index 00000000000..2a758f5486e --- /dev/null +++ b/prepare-build.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +export PACKAGE_NAME=postgresql-16 +export BUILD_USER=mdb-cc +export VERSION=$(grep 'PACKAGE_VERSION=' configure | cut -d= -f2 | sed s/\'//g)-201-yandex.$(git rev-list HEAD --count).$(git rev-parse --short HEAD) +export LC_ALL=C + +cat > debian/changelog< $(date +'%a, %d %b %Y %H:%M:%S %z') +EOH + +echo "VERSION=$VERSION" > version.properties + From a1ef3bde7e01cec232b7eec199e77b891fc5bda0 Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Mon, 17 Jul 2023 08:38:50 +0000 Subject: [PATCH 19/54] Jammy-fixes for yc checker patch to compile Fixes for mdb build to compile: disable gss Build fix --- configure.ac | 12 ------------ docker/entrypoint.sh | 4 +++- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/configure.ac b/configure.ac index a9de7922c90..2a7a993289d 100644 --- a/configure.ac +++ b/configure.ac @@ -903,18 +903,6 @@ PGAC_ARG_BOOL(with, python, no, [build Python modules (PL/Python)]) AC_MSG_RESULT([$with_python]) AC_SUBST(with_python) -# -# GSSAPI -# -AC_MSG_CHECKING([whether to build with GSSAPI support]) -PGAC_ARG_BOOL(with, gssapi, no, [build with GSSAPI support], -[ - AC_DEFINE(ENABLE_GSS, 1, [Define to build with GSSAPI support. (--with-gssapi)]) - krb_srvtab="FILE:\$(sysconfdir)/krb5.keytab" -]) -AC_MSG_RESULT([$with_gssapi]) -AC_SUBST(with_gssapi) - AC_SUBST(krb_srvtab) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index fb2d7b3a735..3c27e80a4f5 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -ex export DEBIAN_FRONTEND=noninteractive export TZ=Europe/Moskow @@ -13,7 +14,8 @@ cat debian/changelog sudo mk-build-deps --build-dep --install --tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes' debian/control -dpkg-buildpackage -us -uc +dpkg-buildpackage -b -rfakeroot -us -uc +#dpkg-buildpackage -us -uc cd /home rm -fr build-user From 009f1fa172276ebbd33c4a7c0c7d54450761c6da Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Tue, 28 May 2024 14:18:25 +0300 Subject: [PATCH 20/54] Restrict DROP DATABASE to superuser only Add test for drop database MDB behaviour --- src/backend/commands/dbcommands.c | 9 +++++ src/test/mdb_admin/t/dropdb.pl | 55 +++++++++++++++++++++++++++++++ src/test/mdb_admin/t/signals.pl | 2 +- 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 src/test/mdb_admin/t/dropdb.pl diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index f39709666ea..799e927792f 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -1631,6 +1631,15 @@ dropdb(const char *dbname, bool missing_ok, bool force) aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_DATABASE, dbname); + /* -- MDB kostyl begin + * remove this when feature will be supported + */ + if (!superuser()) + aclcheck_error(ACLCHECK_NO_PRIV, OBJECT_DATABASE, + dbname); + + /* MDB kostyl end */ + /* DROP hook for the database being removed */ InvokeObjectDropHook(DatabaseRelationId, db_id, 0); diff --git a/src/test/mdb_admin/t/dropdb.pl b/src/test/mdb_admin/t/dropdb.pl new file mode 100644 index 00000000000..c8b8902a94e --- /dev/null +++ b/src/test/mdb_admin/t/dropdb.pl @@ -0,0 +1,55 @@ + +# Copyright (c) 2024-2024, MDB, Mother Russia + +# Minimal test testing drop database restrictions +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize primary node +my $node_primary = PostgreSQL::Test::Cluster->new('primary'); +$node_primary->init(); +$node_primary->start; + +# Create some content on primary and check its presence in standby nodes +$node_primary->safe_psql('postgres', + " + CREATE ROLE mdb_admin; + CREATE ROLE mdb_superuser; + CREATE ROLE mdb_reg_lh_1; + GRANT mdb_admin TO mdb_reg_lh_1; + GRANT mdb_superuser TO mdb_reg_lh_1; + ALTER ROLE mdb_reg_lh_1 CREATEDB; +"); + +# Create some content on primary +$node_primary->safe_psql('postgres', + " + SET ROLE mdb_reg_lh_1; + CREATE DATABASE regress_db1; +"); + + +my ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1) = $node_primary->psql('postgres', + " + SET ROLE mdb_reg_lh_1; + DROP DATABASE regress_db1; +"); + +# print ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1, "\n"); + +ok($res_reg_lh_1 != 0, "should fail for non-superuser"); +like($stderr_reg_lh_1, qr/ERROR: permission denied for database regress_db1/, "matches"); + +my ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2) = $node_primary->psql('postgres', + " + DROP DATABASE regress_db1; +"); + +ok($res_reg_lh_2 == 0, "should success for superuser"); + +# print ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2, "\n"); + +done_testing(); diff --git a/src/test/mdb_admin/t/signals.pl b/src/test/mdb_admin/t/signals.pl index a4b3f07fb89..d5d3adae486 100644 --- a/src/test/mdb_admin/t/signals.pl +++ b/src/test/mdb_admin/t/signals.pl @@ -1,7 +1,7 @@ # Copyright (c) 2024-2024, MDB, Mother Russia -# Minimal test testing streaming replication +# Minimal test testing mdb_admin role use strict; use warnings; use PostgreSQL::Test::Cluster; From 9afe831f45b3f54479a2121644f3e5b4f79ac552 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Mon, 8 Jul 2024 14:57:31 +0500 Subject: [PATCH 21/54] [MDB-28474] Increate readaheadchunk for XlogPageReader() We learned from the field incidents, that 128Kb is not enough. This time we are going to try to increase this value. --- src/backend/access/transam/xlogrecovery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 39d07245241..c4754ca97af 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -3392,7 +3392,7 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, /* * Prefetch next wal blocks to avoid page misses on next read iterations. */ -#define RACHUNK (128*1024) +#define RACHUNK (16*1024*1024) if (readOff % RACHUNK == 0) { pgstat_report_wait_start(WAIT_EVENT_WAL_PREFETCH); posix_fadvise(readFile, readOff + RACHUNK, RACHUNK, POSIX_FADV_WILLNEED); From 375858483f6895b89f9a8654a28784696056dc63 Mon Sep 17 00:00:00 2001 From: diphantxm Date: Mon, 9 Sep 2024 16:50:15 +0300 Subject: [PATCH 22/54] parameter max_log_size to truncate logs There is no need to log the entire query, because it may be large and take lots of space on disk. Parameter max_log_size set the maximum length for logged query. Everything beyond that length is truncated. Value 0 disables the parameter. --- src/backend/utils/error/elog.c | 8 ++++++++ src/backend/utils/misc/guc_tables.c | 11 +++++++++++ src/backend/utils/misc/postgresql.conf.sample | 2 ++ src/bin/pg_ctl/t/004_logrotate.pl | 15 +++++++++++++++ src/include/utils/elog.h | 1 + 5 files changed, 37 insertions(+) diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 4f99f701ea5..9dcd2a41360 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -114,6 +114,7 @@ int Log_destination = LOG_DESTINATION_STDERR; char *Log_destination_string = NULL; bool syslog_sequence_numbers = true; bool syslog_split_messages = true; +int max_log_size = 0; /* Processed form of backtrace_symbols GUC */ static char *backtrace_symbol_list; @@ -1696,6 +1697,13 @@ EmitErrorReport(void) CHECK_STACK_DEPTH(); oldcontext = MemoryContextSwitchTo(edata->assoc_context); + if (max_log_size != 0 && debug_query_string != NULL) + { + char* str = debug_query_string; + str[pg_mbcliplen(str, strlen(str), max_log_size)] = '\0'; + debug_query_string = str; + } + /* * Call hook before sending message to log. The hook function is allowed * to turn off edata->output_to_server, so we must recheck that afterward. diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 8bcb030c3de..0bb0f8b95cb 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3537,6 +3537,17 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"max_log_size", PGC_SIGHUP, LOGGING_WHAT, + gettext_noop("Sets max size of logged statement."), + NULL + }, + &max_log_size, + 5 * (1024 * 1024), + 0, INT_MAX, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index bcb63b681b9..9437c218ef1 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -601,6 +601,8 @@ # bind-parameter values to N bytes; # -1 means print in full, 0 disables #log_statement = 'none' # none, ddl, mod, all +#max_log_size = 0 # max size of logged statement_timeout + # 0 disables the feature #log_replication_commands = off #log_temp_files = -1 # log temporary files equal or larger # than the specified size in kilobytes; diff --git a/src/bin/pg_ctl/t/004_logrotate.pl b/src/bin/pg_ctl/t/004_logrotate.pl index 8d48e56ee9b..ce605fb0a57 100644 --- a/src/bin/pg_ctl/t/004_logrotate.pl +++ b/src/bin/pg_ctl/t/004_logrotate.pl @@ -69,6 +69,7 @@ sub check_log_pattern # these ensure stability of test results: log_rotation_age = 0 lc_messages = 'C' +max_log_size = 32 )); $node->start(); @@ -135,6 +136,20 @@ sub check_log_pattern check_log_pattern('csvlog', $new_current_logfiles, 'syntax error', $node); check_log_pattern('jsonlog', $new_current_logfiles, 'syntax error', $node); +$node->psql('postgres', 'INSERT INTO SOME_NON_EXISTANT_TABLE VALUES (TEST)'); +for (my $attempts = 0; $attempts < $max_attempts; $attempts++) +{ + eval { + $current_logfiles = slurp_file($node->data_dir . '/current_logfiles'); + }; + last unless $@; + usleep(100_000); +} +die $@ if $@; +check_log_pattern('stderr', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node); +check_log_pattern('csvlog', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node); +check_log_pattern('jsonlog', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node); + $node->stop(); done_testing(); diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index b3e95044174..9d60eb19587 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -502,6 +502,7 @@ extern PGDLLIMPORT int Log_destination; extern PGDLLIMPORT char *Log_destination_string; extern PGDLLIMPORT bool syslog_sequence_numbers; extern PGDLLIMPORT bool syslog_split_messages; +extern PGDLLIMPORT int max_log_size; /* Log destination bitmap */ #define LOG_DESTINATION_STDERR 1 From 4660aa385c6e6d619705d0cdb2f9de187b53019c Mon Sep 17 00:00:00 2001 From: diphantxm Date: Tue, 12 Nov 2024 15:45:28 +0300 Subject: [PATCH 23/54] truncate query to be logged in simple query --- src/backend/tcop/postgres.c | 10 ++++++++-- src/backend/utils/error/elog.c | 36 ++++++++++++++++++++++++++++------ src/include/utils/elog.h | 1 + 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 73d6a6194f1..95df771a41a 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -70,6 +70,7 @@ #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "tcop/utility.h" +#include "utils/elog.h" #include "utils/guc_hooks.h" #include "utils/lsyscache.h" #include "utils/memutils.h" @@ -1024,11 +1025,14 @@ exec_simple_query(const char *query_string) bool was_logged = false; bool use_implicit_block; char msec_str[32]; + char* query_log; /* * Report query to various monitoring facilities. */ debug_query_string = query_string; + bool copied = false; + query_log = build_query_log(query_string, &copied); pgstat_report_activity(STATE_RUNNING, query_string); @@ -1073,7 +1077,7 @@ exec_simple_query(const char *query_string) if (check_log_statement(parsetree_list)) { ereport(LOG, - (errmsg("statement: %s", query_string), + (errmsg("statement: %s", query_log), errhidestmt(true), errdetail_execute(parsetree_list))); was_logged = true; @@ -1372,7 +1376,7 @@ exec_simple_query(const char *query_string) case 2: ereport(LOG, (errmsg("duration: %s ms statement: %s", - msec_str, query_string), + msec_str, query_log), errhidestmt(true), errdetail_execute(parsetree_list))); break; @@ -1383,6 +1387,8 @@ exec_simple_query(const char *query_string) TRACE_POSTGRESQL_QUERY_DONE(query_string); + if (query_log && copied) + pfree(query_log); debug_query_string = NULL; } diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 9dcd2a41360..77f77c8274c 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -1697,12 +1697,9 @@ EmitErrorReport(void) CHECK_STACK_DEPTH(); oldcontext = MemoryContextSwitchTo(edata->assoc_context); - if (max_log_size != 0 && debug_query_string != NULL) - { - char* str = debug_query_string; - str[pg_mbcliplen(str, strlen(str), max_log_size)] = '\0'; - debug_query_string = str; - } + const char* old_query_string = debug_query_string; + bool copied = false; + debug_query_string = build_query_log(debug_query_string, &copied); /* * Call hook before sending message to log. The hook function is allowed @@ -1736,6 +1733,12 @@ EmitErrorReport(void) MemoryContextSwitchTo(oldcontext); recursion_depth--; + + if (debug_query_string && copied) + { + pfree(debug_query_string); + debug_query_string = old_query_string; + } } /* @@ -3837,3 +3840,24 @@ trace_recovery(int trace_level) return trace_level; } + +char* +build_query_log(const char* query, bool *copied) +{ + *copied = false; + if (!query) + return NULL; + + size_t query_len = strlen(query); + if (max_log_size == 0 || query_len < max_log_size) + { + return query; + } + + *copied = true; + size_t query_log_len = pg_mbcliplen(query, query_len, max_log_size); + char* query_log = (char*)palloc(query_log_len+1); + memcpy(query_log, query, query_log_len); + query_log[query_log_len] = '\0'; + return query_log; +} diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index 9d60eb19587..6192b22779e 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -517,6 +517,7 @@ extern void log_status_format(StringInfo buf, const char *format, extern void DebugFileOpen(void); extern char *unpack_sql_state(int sql_state); extern bool in_error_recursion_trouble(void); +char* build_query_log(const char* query, bool *copied); /* Common functions shared across destinations */ extern void reset_formatted_start_time(void); From 4cf0e42cf8c3d37133c3c4d53807d1b6a796e4c7 Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Tue, 10 Sep 2024 17:05:11 +0300 Subject: [PATCH 24/54] Never check for superuser in walsender --- src/backend/replication/slot.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index b5271921e6d..ea60f67970f 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -1231,17 +1231,11 @@ CheckSlotPermissions(void) void CheckRoleMDBReplSlotPermissions(bool role_has_rolreplication, bool is_member_of_mdb_replication) { - /* superuser can do it */ - if (superuser()) { - return; - } - /* mdb_replication can do it */ if (is_member_of_mdb_replication) { return; } - if (!role_has_rolreplication) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -1253,10 +1247,6 @@ CheckRoleMDBReplSlotPermissions(bool role_has_rolreplication, bool is_member_of_ void CheckRoleUseMDBReservedName(const char *name, bool role_has_rolreplication) { - /* superuser can do it */ - if (superuser()) { - return; - } /* ugly coding for speed (taken from IsReservedName) */ if (name[0] == 'm' && name[1] == 'd' && From 19975f387e737bbe3c6076cb60f135eb610064aa Mon Sep 17 00:00:00 2001 From: Andrei Liarskii Date: Sat, 2 Nov 2024 08:40:02 +0000 Subject: [PATCH 25/54] Pull request #199: bump llvm to 18 pg 16.4 Merge in MDB/postgres-dev from MDB-31374-bump-llvm-18-pg16_4 to MDB_16_4_no_force Squashed commit of the following: commit fac6ac32dda9ca581ed17134cf67db5755144386 Author: Andrey Lyarskiy Date: Tue Oct 29 12:59:04 2024 +0300 bump llvm to 18 --- Dockerfile | 2 +- debian/control | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 76faa3a2045..2fb32956ea4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpam0g-dev \ debhelper debootstrap devscripts make equivs debhelper-compat \ libz-dev flex libicu-dev libio-pty-perl libipc-run-perl libkrb5-dev \ - libldap2-dev liblz4-dev liblz4-tool zstd libperl-dev libreadline-dev libselinux1-dev llvm-dev \ + libldap2-dev liblz4-dev liblz4-tool zstd libperl-dev libreadline-dev libselinux1-dev llvm-18-dev \ libsystemd-dev libxml2-dev libxml2-utils libxslt1-dev \ pkg-config python3-dev systemtap-sdt-dev tcl-dev uuid-dev xsltproc zlib1g-dev \ bison dh-exec docbook-xml docbook-xsl diff --git a/debian/control b/debian/control index f27bac537a3..95b5bb4d6ac 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Rules-Requires-Root: no Build-Depends: autoconf, bison, - clang [!alpha !hppa !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32], + clang-18 [!alpha !hppa !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !m68k !powerpc !riscv64 !s390x !sh4 !sparc64 !x32], debhelper-compat (= 13), dh-exec (>= 0.13~), docbook-xml, @@ -36,7 +36,7 @@ Build-Depends: libxml2-utils, libxslt1-dev, libzstd-dev (>= 1.4.0) , - llvm-dev [!alpha !hppa !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32], + llvm-18-dev [!alpha !hppa !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32], lz4 | liblz4-tool, mawk, perl (>= 5.8), From 789af359ef1b8774a4c0b8dbab4719284de57710 Mon Sep 17 00:00:00 2001 From: diphantxm Date: Mon, 26 Aug 2024 16:17:08 +0300 Subject: [PATCH 26/54] Support FORCE option in analyze command Add new parameter to VACUUM command, FORCE, meaning VACUUM should terminate all backends that prevents the execution by holding conflicting lock --- doc/src/sgml/ref/analyze.sgml | 1 + doc/src/sgml/ref/vacuum.sgml | 11 ++++++ doc/src/sgml/ref/vacuumdb.sgml | 9 +++++ src/backend/commands/async.c | 27 ++++++++++++++ src/backend/commands/vacuum.c | 53 ++++++++++++++++++++++++++-- src/backend/storage/ipc/procsignal.c | 4 +++ src/bin/psql/tab-complete.c | 8 ++--- src/bin/scripts/t/100_vacuumdb.pl | 4 +++ src/bin/scripts/vacuumdb.c | 21 +++++++++++ src/include/commands/async.h | 2 ++ src/include/commands/vacuum.h | 1 + src/include/storage/procsignal.h | 3 ++ src/test/regress/expected/vacuum.out | 6 ++++ src/test/regress/sql/vacuum.sql | 6 ++++ 14 files changed, 150 insertions(+), 6 deletions(-) diff --git a/doc/src/sgml/ref/analyze.sgml b/doc/src/sgml/ref/analyze.sgml index 73fa3b3dff3..c832b17d7f0 100644 --- a/doc/src/sgml/ref/analyze.sgml +++ b/doc/src/sgml/ref/analyze.sgml @@ -29,6 +29,7 @@ ANALYZE [ VERBOSE ] [ table_and_columnsboolean ] SKIP_LOCKED [ boolean ] BUFFER_USAGE_LIMIT size + FORCE [ boolean ] and table_and_columns is: diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index 2b85c3d385d..09c381ce7fb 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -35,6 +35,7 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ boolean ] PROCESS_TOAST [ boolean ] + FORCE [ boolean ] TRUNCATE [ boolean ] PARALLEL integer SKIP_DATABASE_STATS [ boolean ] @@ -193,6 +194,16 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ table [ (column [,...]) ] diff --git a/src/backend/commands/async.c b/src/backend/commands/async.c index 10ea65b0a5a..7b2fce1c398 100644 --- a/src/backend/commands/async.c +++ b/src/backend/commands/async.c @@ -1864,6 +1864,33 @@ HandleNotifyInterrupt(void) SetLatch(MyLatch); } + + +/* + * HandleRvrInterrupt + * + * Signal handler portion of interrupt handling. Let the backend know + * that there's a pending notify interrupt. If we're currently reading + * from the client, this will interrupt the read and + * ProcessClientReadInterrupt() will call ProcessNotifyInterrupt(). + */ +void +HandleRvrInterrupt(void) +{ + /* + * Note: this is called by a SIGNAL HANDLER. You must be very wary what + * you do here. + */ + + /* signal that work needs to be done */ + QueryCancelPending = true; + InterruptPending = true; + ProcDiePending = true; + + /* make sure the event is processed in due course */ + SetLatch(MyLatch); +} + /* * ProcessNotifyInterrupt * diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 6c3208eeb84..ba041a5c703 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -154,6 +154,7 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) BufferAccessStrategy bstrategy = NULL; bool verbose = false; bool skip_locked = false; + bool force = false; bool analyze = false; bool freeze = false; bool full = false; @@ -214,6 +215,8 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) ring_size = result; } + else if (strcmp(opt->defname, "force") == 0) + force = defGetBoolean(opt); else if (!vacstmt->is_vacuumcmd) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), @@ -306,7 +309,8 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) (process_main ? VACOPT_PROCESS_MAIN : 0) | (process_toast ? VACOPT_PROCESS_TOAST : 0) | (skip_database_stats ? VACOPT_SKIP_DATABASE_STATS : 0) | - (only_database_stats ? VACOPT_ONLY_DATABASE_STATS : 0); + (only_database_stats ? VACOPT_ONLY_DATABASE_STATS : 0) | + (force ? VACOPT_FORCE : 0); /* sanity checks on options */ Assert(params.options & (VACOPT_VACUUM | VACOPT_ANALYZE)); @@ -516,6 +520,15 @@ vacuum(List *relations, VacuumParams *params, BufferAccessStrategy bstrategy, errmsg("%s cannot be executed from VACUUM or ANALYZE", stmttype))); + /* sanity check for FORCE */ + if ((params->options & VACOPT_FORCE) != 0) + { + if ((params->options & VACOPT_SKIP_LOCKED) != 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("VACUUM option FORCE cannot be used with SKIP_LOCKED"))); + } + /* * Build list of relation(s) to process, putting any new data in * vac_context for safekeeping. @@ -791,6 +804,42 @@ vacuum_open_relation(Oid relid, RangeVar *relation, bits32 options, rel = try_relation_open(relid, lmode); else if (ConditionalLockRelationOid(relid, lmode)) rel = try_relation_open(relid, NoLock); + else if (options & VACOPT_FORCE) + { + LOCKTAG tag; + Oid dbid; + + if (IsSharedRelation(relid)) + dbid = InvalidOid; + else + dbid = MyDatabaseId; + + SET_LOCKTAG_RELATION(tag, dbid, relid); + + while (rel == NULL) + { + VirtualTransactionId* backends = GetLockConflicts(&tag, lmode, NULL); + + /* + * Send signals to all the backends holding the conflicting locks + */ + while (VirtualTransactionIdIsValid(*backends)) + { + SignalVirtualTransaction(*backends, + PROCSIG_CONFLICT_RVR_FORCE, + false); + backends++; + } + rel = try_relation_open(relid, lmode); + if (rel == NULL) + { + ereport(NOTICE, + (errcode(ERRCODE_LOCK_NOT_AVAILABLE), + errmsg("retrying attemts of acquiring lock for \"%s\" --- lock not available", + relation->relname))); + } + } + } else { rel = NULL; @@ -914,7 +963,7 @@ expand_vacuum_rel(VacuumRelation *vrel, MemoryContext vac_context, * below, as well as find_all_inheritors's expectation that the caller * holds some lock on the starting relation. */ - rvr_opts = (options & VACOPT_SKIP_LOCKED) ? RVR_SKIP_LOCKED : 0; + rvr_opts = ((options & VACOPT_SKIP_LOCKED) ? RVR_SKIP_LOCKED : 0); relid = RangeVarGetRelidExtended(vrel->relation, AccessShareLock, rvr_opts, diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index c85cb5cc18d..5c3a983553b 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -682,6 +682,10 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN)) RecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN); + /* MDB additions */ + if (CheckProcSignal(PROCSIG_CONFLICT_RVR_FORCE)) + HandleRvrInterrupt(); + SetLatch(MyLatch); errno = save_errno; diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 585a79293d9..7c988d1b845 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -2675,8 +2675,8 @@ psql_completion(const char *text, int start, int end) * one word, so the above test is correct. */ if (ends_with(prev_wd, '(') || ends_with(prev_wd, ',')) - COMPLETE_WITH("VERBOSE", "SKIP_LOCKED", "BUFFER_USAGE_LIMIT"); - else if (TailMatches("VERBOSE|SKIP_LOCKED")) + COMPLETE_WITH("VERBOSE", "SKIP_LOCKED", "BUFFER_USAGE_LIMIT", "FORCE"); + else if (TailMatches("VERBOSE|SKIP_LOCKED|FORCE")) COMPLETE_WITH("ON", "OFF"); } else if (HeadMatches("ANALYZE") && TailMatches("(")) @@ -4632,8 +4632,8 @@ psql_completion(const char *text, int start, int end) "DISABLE_PAGE_SKIPPING", "SKIP_LOCKED", "INDEX_CLEANUP", "PROCESS_MAIN", "PROCESS_TOAST", "TRUNCATE", "PARALLEL", "SKIP_DATABASE_STATS", - "ONLY_DATABASE_STATS", "BUFFER_USAGE_LIMIT"); - else if (TailMatches("FULL|FREEZE|ANALYZE|VERBOSE|DISABLE_PAGE_SKIPPING|SKIP_LOCKED|PROCESS_MAIN|PROCESS_TOAST|TRUNCATE|SKIP_DATABASE_STATS|ONLY_DATABASE_STATS")) + "ONLY_DATABASE_STATS", "BUFFER_USAGE_LIMIT", "FORCE"); + else if (TailMatches("FULL|FREEZE|ANALYZE|VERBOSE|DISABLE_PAGE_SKIPPING|SKIP_LOCKED|PROCESS_MAIN|PROCESS_TOAST|TRUNCATE|SKIP_DATABASE_STATS|ONLY_DATABASE_STATS|FORCE")) COMPLETE_WITH("ON", "OFF"); else if (TailMatches("INDEX_CLEANUP")) COMPLETE_WITH("AUTO", "ON", "OFF"); diff --git a/src/bin/scripts/t/100_vacuumdb.pl b/src/bin/scripts/t/100_vacuumdb.pl index 925079bbedb..05ff80894b8 100644 --- a/src/bin/scripts/t/100_vacuumdb.pl +++ b/src/bin/scripts/t/100_vacuumdb.pl @@ -48,6 +48,10 @@ [ 'vacuumdb', '--skip-locked', '--analyze-only', 'postgres' ], qr/statement: ANALYZE \(SKIP_LOCKED\).*;/, 'vacuumdb --skip-locked --analyze-only'); +$node->issues_sql_like( + [ 'vacuumdb', '--force', '--analyze-only', 'postgres' ], + qr/statement: ANALYZE \(FORCE\).*;/, + 'vacuumdb --force --analyze-only'); $node->command_fails( [ 'vacuumdb', '--analyze-only', '--disable-page-skipping', 'postgres' ], '--analyze-only and --disable-page-skipping specified together'); diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c index a1ebabc0735..d73c7c931c4 100644 --- a/src/bin/scripts/vacuumdb.c +++ b/src/bin/scripts/vacuumdb.c @@ -47,6 +47,7 @@ typedef struct vacuumingOptions bool process_toast; bool skip_database_stats; char *buffer_usage_limit; + bool force; } vacuumingOptions; /* object filter options */ @@ -127,6 +128,7 @@ main(int argc, char *argv[]) {"no-process-toast", no_argument, NULL, 11}, {"no-process-main", no_argument, NULL, 12}, {"buffer-usage-limit", required_argument, NULL, 13}, + {"force", no_argument, NULL, 14}, {NULL, 0, NULL, 0} }; @@ -274,6 +276,9 @@ main(int argc, char *argv[]) case 13: vacopts.buffer_usage_limit = escape_quotes(optarg); break; + case 14: + vacopts.force = true; + break; default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -359,6 +364,7 @@ main(int argc, char *argv[]) pg_fatal("cannot use the \"%s\" option with the \"%s\" option", "buffer-usage-limit", "full"); + /* fill cparams except for dbname, which is set below */ cparams.pghost = host; cparams.pgport = port; @@ -1005,6 +1011,13 @@ prepare_vacuum_command(PQExpBuffer sql, int serverVersion, appendPQExpBuffer(sql, "%sSKIP_LOCKED", sep); sep = comma; } + if (vacopts->force) + { + ///* FORCE is supported since v12 */ + Assert(serverVersion >= 120000); + appendPQExpBuffer(sql, "%sFORCE", sep); + sep = comma; + } if (vacopts->verbose) { appendPQExpBuffer(sql, "%sVERBOSE", sep); @@ -1091,6 +1104,13 @@ prepare_vacuum_command(PQExpBuffer sql, int serverVersion, appendPQExpBuffer(sql, "%sSKIP_LOCKED", sep); sep = comma; } + if (vacopts->force) + { + ///* FORCE is supported since v12 */ + Assert(serverVersion >= 120000); + appendPQExpBuffer(sql, "%sFORCE", sep); + sep = comma; + } if (vacopts->full) { appendPQExpBuffer(sql, "%sFULL", sep); @@ -1200,6 +1220,7 @@ help(const char *progname) printf(_(" -P, --parallel=PARALLEL_WORKERS use this many background workers for vacuum, if available\n")); printf(_(" -q, --quiet don't write any messages\n")); printf(_(" --skip-locked skip relations that cannot be immediately locked\n")); + printf(_(" --force terminate backends holding conflicting lock\n")); printf(_(" -t, --table='TABLE[(COLUMNS)]' vacuum specific table(s) only\n")); printf(_(" -v, --verbose write a lot of output\n")); printf(_(" -V, --version output version information, then exit\n")); diff --git a/src/include/commands/async.h b/src/include/commands/async.h index fbf091e1226..4ce5fe31844 100644 --- a/src/include/commands/async.h +++ b/src/include/commands/async.h @@ -47,6 +47,8 @@ extern void AtPrepare_Notify(void); /* signal handler for inbound notifies (PROCSIG_NOTIFY_INTERRUPT) */ extern void HandleNotifyInterrupt(void); +extern void HandleRvrInterrupt(void); + /* process interrupts */ extern void ProcessNotifyInterrupt(bool flush); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 39fbd5f10a5..0b826b12e3a 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -191,6 +191,7 @@ typedef struct VacAttrStats #define VACOPT_DISABLE_PAGE_SKIPPING 0x100 /* don't skip any pages */ #define VACOPT_SKIP_DATABASE_STATS 0x200 /* skip vac_update_datfrozenxid() */ #define VACOPT_ONLY_DATABASE_STATS 0x400 /* only vac_update_datfrozenxid() */ +#define VACOPT_FORCE 0x800 /* terminate conflicting backend if cannot get lock */ /* * Values used by index_cleanup and truncate params. diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h index 2f52100b009..c9b5d3fe2e6 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -46,6 +46,9 @@ typedef enum PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, PROCSIG_RECOVERY_CONFLICT_STARTUP_DEADLOCK, + /* MDB additions */ + PROCSIG_CONFLICT_RVR_FORCE, + NUM_PROCSIGNALS /* Must be last! */ } ProcSignalReason; diff --git a/src/test/regress/expected/vacuum.out b/src/test/regress/expected/vacuum.out index 4aaf4f025d3..b7690f85e4e 100644 --- a/src/test/regress/expected/vacuum.out +++ b/src/test/regress/expected/vacuum.out @@ -315,6 +315,12 @@ VACUUM (SKIP_LOCKED) vactst; VACUUM (SKIP_LOCKED, FULL) vactst; ANALYZE (SKIP_LOCKED) vactst; RESET client_min_messages; +-- FORCE option +ANALYZE (FORCE) vactst; +VACUUM (ANALYZE, FORCE) vactst; +VACUUM (FORCE) vactst; +VACUUM (ANALYZE, FORCE, SKIP_LOCKED) vactst; +ERROR: VACUUM option FORCE cannot be used with SKIP_LOCKED -- ensure VACUUM and ANALYZE don't have a problem with serializable SET default_transaction_isolation = serializable; VACUUM vactst; diff --git a/src/test/regress/sql/vacuum.sql b/src/test/regress/sql/vacuum.sql index ae36b546410..f964f067112 100644 --- a/src/test/regress/sql/vacuum.sql +++ b/src/test/regress/sql/vacuum.sql @@ -251,6 +251,12 @@ VACUUM (SKIP_LOCKED, FULL) vactst; ANALYZE (SKIP_LOCKED) vactst; RESET client_min_messages; +-- FORCE option +ANALYZE (FORCE) vactst; +VACUUM (ANALYZE, FORCE) vactst; +VACUUM (FORCE) vactst; +VACUUM (ANALYZE, FORCE, SKIP_LOCKED) vactst; + -- ensure VACUUM and ANALYZE don't have a problem with serializable SET default_transaction_isolation = serializable; VACUUM vactst; From feee5574b673096d7c882e777691a6480b863e2b Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Tue, 12 Nov 2024 23:01:32 +0500 Subject: [PATCH 27/54] Use fadvise in walsender remove bogus progress reporting --- src/backend/access/transam/xlogreader.c | 9 +++++++++ src/backend/access/transam/xlogrecovery.c | 22 ++++++++++------------ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 6db8dc00e82..acccf3d2536 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -1572,6 +1572,15 @@ WALRead(XLogReaderState *state, /* Reset errno first; eases reporting non-errno-affecting errors */ errno = 0; + +#if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) + /* + * Prefetch next wal blocks to avoid page misses on next read iterations. + */ +#define RACHUNK (16*1024*1024) + if (p == 0) + posix_fadvise(state->seg.ws_file, 0, RACHUNK, POSIX_FADV_WILLNEED); +#endif readbytes = pg_pread(state->seg.ws_file, p, segbytes, (off_t) startoff); #ifndef FRONTEND diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index c4754ca97af..4fb243a30c2 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -3357,6 +3357,16 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, readOff = targetPageOff; pgstat_report_wait_start(WAIT_EVENT_WAL_READ); + +#if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) + /* + * Prefetch next wal blocks to avoid page misses on next read iterations. + */ +#define RACHUNK (16*1024*1024) + if (readOff == 0) { + posix_fadvise(readFile, 0, RACHUNK, POSIX_FADV_WILLNEED); + } +#endif r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); if (r != XLOG_BLCKSZ) { @@ -3388,18 +3398,6 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, Assert(targetPageOff == readOff); Assert(reqLen <= readLen); -#if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) - /* - * Prefetch next wal blocks to avoid page misses on next read iterations. - */ -#define RACHUNK (16*1024*1024) - if (readOff % RACHUNK == 0) { - pgstat_report_wait_start(WAIT_EVENT_WAL_PREFETCH); - posix_fadvise(readFile, readOff + RACHUNK, RACHUNK, POSIX_FADV_WILLNEED); - pgstat_report_wait_end(); - } -#endif - xlogreader->seg.ws_tli = curFileTLI; /* From 2dbd07e341a7212d8045f2730ac79633e25749af Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Wed, 13 Nov 2024 12:03:52 +0500 Subject: [PATCH 28/54] GUCify NUM_BUFFER_PARTITIONS Also truncate query to be logged in simple query Remove unused functions in buf_internals.h --- src/backend/storage/buffer/bufmgr.c | 8 ++++++++ src/backend/utils/init/globals.c | 4 ++++ src/backend/utils/misc/guc_tables.c | 11 +++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/storage/buf_internals.h | 14 +------------- src/include/storage/lwlock.h | 4 +++- src/include/utils/guc_hooks.h | 1 + src/test/modules/test_misc/t/003_check_guc.pl | 1 + 8 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index ef14791e2da..58453ff3f67 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -55,6 +55,7 @@ #include "storage/proc.h" #include "storage/smgr.h" #include "storage/standby.h" +#include "utils/guc_hooks.h" #include "utils/memdebug.h" #include "utils/ps_status.h" #include "utils/rel.h" @@ -140,6 +141,13 @@ int bgwriter_lru_maxpages = 100; double bgwriter_lru_multiplier = 2.0; bool track_io_timing = false; +/* GUC assign hook for num_buffer_partitions_log2 */ +void +assign_num_buffer_partitions_log2(int newval, void *extra) +{ + num_buffer_partitions_mask = (1 << newval) - 1; +} + /* * How many buffers PrefetchBuffer callers should try to stay ahead of their * ReadBuffer calls by. Zero means "never prefetch". This value is only used diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c index 011ec18015a..19edc3d8674 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -154,3 +154,7 @@ int64 VacuumPageDirty = 0; int VacuumCostBalance = 0; /* working state for vacuum */ bool VacuumCostActive = false; + +/* shared buffers partitions number and mask */ +int num_buffer_partitions_log2 = 7; +int num_buffer_partitions_mask = 127; diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 0bb0f8b95cb..ba6e7ef993c 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -2412,6 +2412,17 @@ struct config_int ConfigureNamesInt[] = check_max_stack_depth, assign_max_stack_depth, NULL }, + { + /* MDB prefix here is needed in case when vanilla is starting with config with this value */ + {"ycmdb.num_buffer_partitions_log2", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Sets number of partitions for shared buffers mapping hashtable."), + NULL, + }, + &num_buffer_partitions_log2, + 7, 5, 16, + NULL, assign_num_buffer_partitions_log2, NULL + }, + { {"temp_file_limit", PGC_SUSET, RESOURCES_DISK, gettext_noop("Limits the total size of all temporary files used by each process."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 9437c218ef1..c92bdfeef81 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -144,6 +144,7 @@ #autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem #logical_decoding_work_mem = 64MB # min 64kB #max_stack_depth = 2MB # min 100kB +#mdb.num_buffer_partitions_log2 = 7 # number of partitions in shared buffer mapping hashtable #shared_memory_type = mmap # the default is the first option # supported by the operating system: # mmap diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index 98cd2499098..f9ce4a22104 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -175,23 +175,11 @@ BufTagMatchesRelFileLocator(const BufferTag *tag, * hash code with BufTableHashCode(), then apply BufMappingPartitionLock(). * NB: NUM_BUFFER_PARTITIONS must be a power of 2! */ -static inline uint32 -BufTableHashPartition(uint32 hashcode) -{ - return hashcode % NUM_BUFFER_PARTITIONS; -} - static inline LWLock * BufMappingPartitionLock(uint32 hashcode) { return &MainLWLockArray[BUFFER_MAPPING_LWLOCK_OFFSET + - BufTableHashPartition(hashcode)].lock; -} - -static inline LWLock * -BufMappingPartitionLockByIndex(uint32 index) -{ - return &MainLWLockArray[BUFFER_MAPPING_LWLOCK_OFFSET + index].lock; + (hashcode & num_buffer_partitions_mask)].lock; } /* diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index cab38447d48..7535ab8b022 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -92,7 +92,9 @@ extern PGDLLIMPORT int NamedLWLockTrancheRequests; */ /* Number of partitions of the shared buffer mapping hashtable */ -#define NUM_BUFFER_PARTITIONS 128 +extern int num_buffer_partitions_log2; +extern int num_buffer_partitions_mask; +#define NUM_BUFFER_PARTITIONS (1 << num_buffer_partitions_log2) /* Number of partitions the shared lock tables are divided into */ #define LOG2_NUM_LOCK_PARTITIONS 4 diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 0ea33fede9c..499d379d7d0 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -89,6 +89,7 @@ extern bool check_max_worker_processes(int *newval, void **extra, GucSource source); extern bool check_max_stack_depth(int *newval, void **extra, GucSource source); extern void assign_max_stack_depth(int newval, void *extra); +extern void assign_num_buffer_partitions_log2(int newval, void *extra); extern bool check_primary_slot_name(char **newval, void **extra, GucSource source); extern bool check_random_seed(double *newval, void **extra, GucSource source); diff --git a/src/test/modules/test_misc/t/003_check_guc.pl b/src/test/modules/test_misc/t/003_check_guc.pl index 5feadc558ed..50bdd48a0e8 100644 --- a/src/test/modules/test_misc/t/003_check_guc.pl +++ b/src/test/modules/test_misc/t/003_check_guc.pl @@ -70,6 +70,7 @@ } push @gucs_in_file, "ycmdb.yc_grant_checker"; +push @gucs_in_file, "ycmdb.num_buffer_partitions_log2"; close $contents; From 01caaaca43b03310a7dae508ac972b8ba40e6f9e Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Wed, 11 Dec 2024 20:14:55 +0300 Subject: [PATCH 29/54] Change storage class for mdb_replication utilities --- src/include/replication/slot.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index 0f20a4596de..bbce7a31ca2 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -265,11 +265,11 @@ extern void CheckRoleMDBReplSlotPermissions(bool role_has_rolreplication, bool i */ extern void CheckRoleUseMDBReservedName(const char* name, bool role_has_rolreplication); -inline void CheckMDBReservedName(const char* name) { +static inline void CheckMDBReservedName(const char* name) { CheckRoleUseMDBReservedName(name, has_rolreplication(GetUserId())); } -inline void CheckMDBReplSlotPermissions(void) { +static inline void CheckMDBReplSlotPermissions(void) { Oid role; role = get_role_oid("mdb_replication", /* missing ok*/ true); return CheckRoleMDBReplSlotPermissions(has_rolreplication(GetUserId()), is_member_of_role(GetUserId(), role)); From f8523bbe1fcf4134c44f5514678663b8b32005df Mon Sep 17 00:00:00 2001 From: reshke Date: Thu, 16 Jan 2025 19:19:02 +0300 Subject: [PATCH 30/54] Add CI Do not run mdb locales tests in OS Add llvm dir tmp --- .github/workflows/regress.yml | 20 ++++++++++++++++ docker/regress/Dockerfile | 44 +++++++++++++++++++++++++++++++++++ docker/regress/run_tests.sh | 12 ++++++++++ 3 files changed, 76 insertions(+) create mode 100644 .github/workflows/regress.yml create mode 100644 docker/regress/Dockerfile create mode 100755 docker/regress/run_tests.sh diff --git a/.github/workflows/regress.yml b/.github/workflows/regress.yml new file mode 100644 index 00000000000..0395de114cb --- /dev/null +++ b/.github/workflows/regress.yml @@ -0,0 +1,20 @@ +name: Docker Image CI + +on: + push: + branches: [ "MDB_*" ] + pull_request: + branches: [ "MDB_*" ] + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Build the Docker image + run: docker build . --file docker/regress/Dockerfile --tag regress_test:1234 && docker run regress_test:1234 + + diff --git a/docker/regress/Dockerfile b/docker/regress/Dockerfile new file mode 100644 index 00000000000..b35c855a15a --- /dev/null +++ b/docker/regress/Dockerfile @@ -0,0 +1,44 @@ +ARG codename +FROM ubuntu:${codename:-bionic} + +ARG codename +ENV CODE_NAME=${codename:-bionic} + +ARG pgdg +ENV PGDG_VER=${pgdg:-242-2-pgdg18.04+1+yandex220} + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Europe/Moskow +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo build-essential \ + gcc lsb-release libssl-dev gnupg openssl \ + gdb git curl + +RUN curl -s 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xafc3ce0d00e3c45a357e9e637fcd11186050cd1a' | \ + gpg --dearmour -o /etc/apt/trusted.gpg.d/yandex.gpg + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo build-essential \ + gcc lsb-release libssl-dev gnupg openssl \ + gdb git \ + libpam0g-dev \ + debhelper debootstrap devscripts make equivs debhelper-compat \ + libz-dev flex libicu-dev libio-pty-perl libipc-run-perl libkrb5-dev \ + libldap2-dev liblz4-dev liblz4-tool zstd libperl-dev libreadline-dev libselinux1-dev llvm-dev \ + libsystemd-dev libxml2-dev libxml2-utils libxslt1-dev \ + pkg-config python3-dev systemtap-sdt-dev tcl-dev uuid-dev xsltproc zlib1g-dev \ + bison dh-exec docbook-xml docbook-xsl + +RUN groupadd -g 999 build-user && \ + useradd -r -u 999 -g build-user build-user + +COPY . /home/build-user +RUN chown build-user:build-user /home -R && usermod -aG sudo build-user + +RUN echo 'build-user ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER build-user + +ENTRYPOINT ["/home/build-user/docker/regress/run_tests.sh"] diff --git a/docker/regress/run_tests.sh b/docker/regress/run_tests.sh new file mode 100755 index 00000000000..64d2a32536b --- /dev/null +++ b/docker/regress/run_tests.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -ex + +cd /home/build-user + +CFLAGS="" ./configure --prefix=/home/build-user/pgbin --without-mdblocales --enable-depend --enable-cassert --enable-debug --enable-tap-tests + +make -j8 +sed -i '/mdb-related/,$d' src/test/regress/expected/misc.out src/test/regress/sql/misc.sql + +make check From 11c1c0635b3030b69ba201aaf9d562a6abed8f89 Mon Sep 17 00:00:00 2001 From: reshke Date: Fri, 17 Jan 2025 14:21:45 +0500 Subject: [PATCH 31/54] MDB-32132: fix grantor selection for mdb_superuser (#5) --- src/backend/utils/adt/acl.c | 2 +- src/test/regress/expected/mdb_superuser.out | 21 +++++++++++++++++ src/test/regress/sql/mdb_superuser.sql | 26 ++++++++++++++++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 12e40119bfa..a9bffa5277a 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5437,7 +5437,7 @@ select_best_grantor(Oid roleId, AclMode privileges, if (is_member_of_role(GetUserId(), mdb_superuser_roleoid) && has_privs_of_role(GetUserId(), ownerId)) { - *grantorId = mdb_superuser_roleoid; + *grantorId = ownerId; AclMode mdb_superuser_allowed_privs = needed_goptions; *grantOptions = mdb_superuser_allowed_privs; return; diff --git a/src/test/regress/expected/mdb_superuser.out b/src/test/regress/expected/mdb_superuser.out index e8dbe2fe1cc..8d989661434 100644 --- a/src/test/regress/expected/mdb_superuser.out +++ b/src/test/regress/expected/mdb_superuser.out @@ -1,11 +1,15 @@ CREATE ROLE regress_mdb_superuser_user1; CREATE ROLE regress_mdb_superuser_user2; CREATE ROLE regress_mdb_superuser_user3; +CREATE ROLE regress_mdb_su_role_o1; +CREATE ROLE regress_mdb_su_role_o2; +GRANT mdb_superuser TO regress_mdb_su_role_o1; GRANT mdb_admin TO mdb_superuser; CREATE ROLE regress_superuser WITH SUPERUSER; GRANT mdb_superuser TO regress_mdb_superuser_user1; GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user3; +GRANT CREATE ON DATABASE regression TO regress_mdb_su_role_o2; SET ROLE regress_mdb_superuser_user2; CREATE FUNCTION regress_mdb_superuser_add(integer, integer) RETURNS integer AS 'SELECT $1 + $2;' @@ -90,6 +94,20 @@ ERROR: permission denied for schema regtest SET ROLE regress_mdb_superuser_user1; GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; +-- Check grantor +SET ROLE regress_mdb_su_role_o2; +CREATE TABLE public.role_o2_t(); +SET ROLE mdb_superuser; +GRANT SELECT ON public.role_o2_t TO regress_mdb_su_role_o1; +SELECT + grantor +from information_schema.role_table_grants +where grantee='regress_mdb_su_role_o1' AND table_name = 'role_o2_t'; + grantor +------------------------ + regress_mdb_su_role_o2 +(1 row) + \c regression DROP DATABASE regress_check_owner; -- end tests @@ -97,6 +115,9 @@ RESET SESSION AUTHORIZATION; -- REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_su_role_o2; +DROP ROLE regress_mdb_su_role_o1; +DROP ROLE regress_mdb_su_role_o2; DROP VIEW regress_mdb_superuser_view; DROP FUNCTION regress_mdb_superuser_add; DROP TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table; diff --git a/src/test/regress/sql/mdb_superuser.sql b/src/test/regress/sql/mdb_superuser.sql index 77dc489f715..b0e702c630c 100644 --- a/src/test/regress/sql/mdb_superuser.sql +++ b/src/test/regress/sql/mdb_superuser.sql @@ -2,6 +2,11 @@ CREATE ROLE regress_mdb_superuser_user1; CREATE ROLE regress_mdb_superuser_user2; CREATE ROLE regress_mdb_superuser_user3; +CREATE ROLE regress_mdb_su_role_o1; +CREATE ROLE regress_mdb_su_role_o2; + +GRANT mdb_superuser TO regress_mdb_su_role_o1; + GRANT mdb_admin TO mdb_superuser; CREATE ROLE regress_superuser WITH SUPERUSER; @@ -10,7 +15,7 @@ GRANT mdb_superuser TO regress_mdb_superuser_user1; GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user3; - +GRANT CREATE ON DATABASE regression TO regress_mdb_su_role_o2; SET ROLE regress_mdb_superuser_user2; @@ -118,6 +123,21 @@ SET ROLE regress_mdb_superuser_user1; GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; +-- Check grantor + +SET ROLE regress_mdb_su_role_o2; + +CREATE TABLE public.role_o2_t(); + +SET ROLE mdb_superuser; + +GRANT SELECT ON public.role_o2_t TO regress_mdb_su_role_o1; + +SELECT + grantor +from information_schema.role_table_grants +where grantee='regress_mdb_su_role_o1' AND table_name = 'role_o2_t'; + \c regression DROP DATABASE regress_check_owner; @@ -127,6 +147,10 @@ RESET SESSION AUTHORIZATION; -- REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_su_role_o2; + +DROP ROLE regress_mdb_su_role_o1; +DROP ROLE regress_mdb_su_role_o2; DROP VIEW regress_mdb_superuser_view; DROP FUNCTION regress_mdb_superuser_add; From 52bc8091131974635b91cc3086d98856f8d2380d Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 21 Jan 2025 16:15:57 +0500 Subject: [PATCH 32/54] Do not use schema public in mdb_superuser regression tests (#8) --- src/test/regress/expected/mdb_superuser.out | 5 +++-- src/test/regress/sql/mdb_superuser.sql | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/mdb_superuser.out b/src/test/regress/expected/mdb_superuser.out index 8d989661434..7849311cd1d 100644 --- a/src/test/regress/expected/mdb_superuser.out +++ b/src/test/regress/expected/mdb_superuser.out @@ -85,6 +85,7 @@ CREATE DATABASE regress_check_owner OWNER regress_mdb_superuser_user2; SET ROLE regress_mdb_superuser_user2; CREATE SCHEMA regtest; CREATE TABLE regtest.regtest(); +GRANT CREATE ON SCHEMA regtest TO regress_mdb_su_role_o2; -- this should fail SET ROLE regress_mdb_superuser_user3; GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user3; @@ -96,9 +97,9 @@ GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; -- Check grantor SET ROLE regress_mdb_su_role_o2; -CREATE TABLE public.role_o2_t(); +CREATE TABLE regtest.role_o2_t(); SET ROLE mdb_superuser; -GRANT SELECT ON public.role_o2_t TO regress_mdb_su_role_o1; +GRANT SELECT ON regtest.role_o2_t TO regress_mdb_su_role_o1; SELECT grantor from information_schema.role_table_grants diff --git a/src/test/regress/sql/mdb_superuser.sql b/src/test/regress/sql/mdb_superuser.sql index b0e702c630c..0877d9258f6 100644 --- a/src/test/regress/sql/mdb_superuser.sql +++ b/src/test/regress/sql/mdb_superuser.sql @@ -113,6 +113,8 @@ SET ROLE regress_mdb_superuser_user2; CREATE SCHEMA regtest; CREATE TABLE regtest.regtest(); +GRANT CREATE ON SCHEMA regtest TO regress_mdb_su_role_o2; + -- this should fail SET ROLE regress_mdb_superuser_user3; @@ -127,11 +129,11 @@ ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; SET ROLE regress_mdb_su_role_o2; -CREATE TABLE public.role_o2_t(); +CREATE TABLE regtest.role_o2_t(); SET ROLE mdb_superuser; -GRANT SELECT ON public.role_o2_t TO regress_mdb_su_role_o1; +GRANT SELECT ON regtest.role_o2_t TO regress_mdb_su_role_o1; SELECT grantor From 6026436ee5b11baf917a540389d8da11b4a9583a Mon Sep 17 00:00:00 2001 From: Yury Frolov Date: Fri, 24 Jan 2025 11:08:03 +0500 Subject: [PATCH 33/54] Rework docker image logic --- .github/workflows/regress.yml | 2 +- docker/regress/Dockerfile | 8 ++++---- docker/regress/run_tests.sh | 6 ------ 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/regress.yml b/.github/workflows/regress.yml index 0395de114cb..10f428ce0d4 100644 --- a/.github/workflows/regress.yml +++ b/.github/workflows/regress.yml @@ -15,6 +15,6 @@ jobs: steps: - uses: actions/checkout@v4 - name: Build the Docker image - run: docker build . --file docker/regress/Dockerfile --tag regress_test:1234 && docker run regress_test:1234 + run: docker build . --file docker/regress/Dockerfile --tag regress_test:1234 && docker run --entrypoint /home/build-user/docker/base/run_tests.sh regress_test:1234 diff --git a/docker/regress/Dockerfile b/docker/regress/Dockerfile index b35c855a15a..0546ec5d727 100644 --- a/docker/regress/Dockerfile +++ b/docker/regress/Dockerfile @@ -4,9 +4,6 @@ FROM ubuntu:${codename:-bionic} ARG codename ENV CODE_NAME=${codename:-bionic} -ARG pgdg -ENV PGDG_VER=${pgdg:-242-2-pgdg18.04+1+yandex220} - ENV DEBIAN_FRONTEND=noninteractive ENV TZ=Europe/Moskow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone @@ -40,5 +37,8 @@ RUN chown build-user:build-user /home -R && usermod -aG sudo build-user RUN echo 'build-user ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER build-user +WORKDIR /home/build-user -ENTRYPOINT ["/home/build-user/docker/regress/run_tests.sh"] +RUN CFLAGS="" ./configure --prefix=/home/build-user/pgbin --without-mdblocales --enable-depend --enable-cassert --enable-debug --enable-tap-tests && \ + make -j8 && \ + sudo make install diff --git a/docker/regress/run_tests.sh b/docker/regress/run_tests.sh index 64d2a32536b..f415fe5dab2 100755 --- a/docker/regress/run_tests.sh +++ b/docker/regress/run_tests.sh @@ -1,12 +1,6 @@ #!/bin/bash - set -ex -cd /home/build-user - -CFLAGS="" ./configure --prefix=/home/build-user/pgbin --without-mdblocales --enable-depend --enable-cassert --enable-debug --enable-tap-tests - -make -j8 sed -i '/mdb-related/,$d' src/test/regress/expected/misc.out src/test/regress/sql/misc.sql make check From 3451cbf103518d56e2423102c2d1cff7e5e1259a Mon Sep 17 00:00:00 2001 From: Yury Frolov Date: Fri, 24 Jan 2025 11:13:00 +0500 Subject: [PATCH 34/54] Fix CI --- .github/workflows/regress.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regress.yml b/.github/workflows/regress.yml index 10f428ce0d4..7652df14da7 100644 --- a/.github/workflows/regress.yml +++ b/.github/workflows/regress.yml @@ -15,6 +15,6 @@ jobs: steps: - uses: actions/checkout@v4 - name: Build the Docker image - run: docker build . --file docker/regress/Dockerfile --tag regress_test:1234 && docker run --entrypoint /home/build-user/docker/base/run_tests.sh regress_test:1234 + run: docker build . --file docker/regress/Dockerfile --tag regress_test:1234 && docker run --entrypoint /home/build-user/docker/regress/run_tests.sh regress_test:1234 From 4dfe562a30d44b4619f2a53e69b21723a3d3a9c0 Mon Sep 17 00:00:00 2001 From: Yury Frolov <57130330+EinKrebs@users.noreply.github.com> Date: Fri, 24 Jan 2025 18:15:58 +0500 Subject: [PATCH 35/54] check-world in worklows in PG16 (#17) * check-world in worlfows * Run both check & check-world --- docker/regress/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/regress/run_tests.sh b/docker/regress/run_tests.sh index f415fe5dab2..b48b1472953 100755 --- a/docker/regress/run_tests.sh +++ b/docker/regress/run_tests.sh @@ -3,4 +3,4 @@ set -ex sed -i '/mdb-related/,$d' src/test/regress/expected/misc.out src/test/regress/sql/misc.sql -make check +make check check-world From d1c0f6242f8d54061b062731a8fad4992963e5e1 Mon Sep 17 00:00:00 2001 From: reshke Date: Fri, 7 Feb 2025 13:07:37 +0000 Subject: [PATCH 36/54] Introduce mdb_read_all_data/mdb_write_all_data --- src/backend/catalog/aclchk.c | 28 ++++++ .../regress/expected/mdb_read_write_roles.out | 82 +++++++++++++++++ src/test/regress/expected/mdb_superuser.out | 4 +- src/test/regress/expected/test_setup.out | 3 + src/test/regress/parallel_schedule | 2 + src/test/regress/sql/mdb_read_write_roles.sql | 89 +++++++++++++++++++ src/test/regress/sql/mdb_superuser.sql | 5 +- src/test/regress/sql/test_setup.sql | 3 + 8 files changed, 211 insertions(+), 5 deletions(-) create mode 100644 src/test/regress/expected/mdb_read_write_roles.out create mode 100644 src/test/regress/sql/mdb_read_write_roles.sql diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index 2d80a04ce9f..1133ee0dcdc 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -3288,6 +3288,8 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, bool isNull; Acl *acl; Oid ownerId; + Oid mdb_read_all_data_oid; + Oid mdb_write_all_data_oid; /* * Must get the relation's tuple from pg_class @@ -3338,6 +3340,9 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, */ ownerId = classForm->relowner; + mdb_read_all_data_oid = get_role_oid("mdb_read_all_data", true); + mdb_write_all_data_oid = get_role_oid("mdb_write_all_data", true); + aclDatum = SysCacheGetAttr(RELOID, tuple, Anum_pg_class_relacl, &isNull); if (isNull) @@ -3377,6 +3382,17 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, has_privs_of_role(roleid, ROLE_PG_READ_ALL_DATA)) result |= ACL_SELECT; + + /* + * Check if ACL_SELECT is being checked and, if so, and not set already as + * part of the result, then check if the user is a member of the + * mdb_read_all_data role, and this is not some dangerous relation to grant SELECT to + */ + if (mask & ACL_SELECT && !(result & ACL_SELECT) && + has_privs_of_role(roleid, mdb_read_all_data_oid) && + !has_privs_of_unwanted_system_role(ownerId)) + result |= ACL_SELECT; + /* * Check if ACL_INSERT, ACL_UPDATE, or ACL_DELETE is being checked and, if * so, and not set already as part of the result, then check if the user @@ -3389,6 +3405,18 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, has_privs_of_role(roleid, ROLE_PG_WRITE_ALL_DATA)) result |= (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)); + /* + * Check if ACL_INSERT, ACL_UPDATE, or ACL_DELETE is being checked and, if + * so, and not set already as part of the result, then check if the user + * is a member of the mdb_write_all_data role, and this is not some + * dangerous relation to grant write access. + */ + if (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE) && + !(result & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)) && + has_privs_of_role(roleid, mdb_write_all_data_oid) && + !has_privs_of_unwanted_system_role(ownerId)) + result |= (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)); + return result; } diff --git a/src/test/regress/expected/mdb_read_write_roles.out b/src/test/regress/expected/mdb_read_write_roles.out new file mode 100644 index 00000000000..080e95cafb9 --- /dev/null +++ b/src/test/regress/expected/mdb_read_write_roles.out @@ -0,0 +1,82 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; +CREATE ROLE regress_superuser WITH SUPERUSER; +GRANT mdb_superuser TO regress_mdb_superuser_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; +SET ROLE regress_superuser; +CREATE TABLE regress_superuser_table(); +SET ROLE pg_read_server_files; +CREATE TABLE regress_pgrsf_table(); +SET ROLE pg_write_server_files; +CREATE TABLE regress_pgwsf_table(); +SET ROLE pg_execute_server_program; +CREATE TABLE regress_pgxsp_table(); +SET ROLE pg_read_all_data; +CREATE TABLE regress_pgrad_table(); +SET ROLE pg_write_all_data; +CREATE TABLE regress_pgrwd_table(); +SET ROLE regress_mdb_superuser_user1; +CREATE TABLE regress_mdbsu_table(); +SET ROLE regress_mdb_superuser_user2; +CREATE TABLE regress_mdbsu_table2(); +SET ROLE mdb_read_all_data; +-- cannot read all data (fail) +TABLE pg_authid; +ERROR: permission denied for table pg_authid +TABLE regress_superuser_table; +ERROR: permission denied for table regress_superuser_table +TABLE regress_pgrsf_table; +ERROR: permission denied for table regress_pgrsf_table +TABLE regress_pgwsf_table; +ERROR: permission denied for table regress_pgwsf_table +TABLE regress_pgxsp_table; +ERROR: permission denied for table regress_pgxsp_table +TABLE regress_pgrad_table; +ERROR: permission denied for table regress_pgrad_table +TABLE regress_pgwsf_table; +ERROR: permission denied for table regress_pgwsf_table +-- is allow to read all other data +TABLE regress_mdbsu_table; +-- +(0 rows) + +TABLE regress_mdbsu_table2; +-- +(0 rows) + +SET ROLE mdb_write_all_data; +CREATE TABLE regress_tt_dat(); +-- cannot read all data (fail) +INSERT INTO regress_superuser_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_superuser_table +INSERT INTO regress_pgrsf_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgrsf_table +INSERT INTO regress_pgwsf_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgwsf_table +INSERT INTO regress_pgxsp_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgxsp_table +INSERT INTO regress_pgrad_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgrad_table +INSERT INTO regress_pgwsf_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgwsf_table +-- is allow to read all other data +INSERT INTO regress_mdbsu_table TABLE regress_tt_dat; +INSERT INTO regress_mdbsu_table2 TABLE regress_tt_dat; +-- end tests +RESET SESSION AUTHORIZATION; +-- +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; +DROP TABLE regress_mdbsu_table; +DROP TABLE regress_mdbsu_table2; +DROP TABLE regress_superuser_table; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/expected/mdb_superuser.out b/src/test/regress/expected/mdb_superuser.out index 7849311cd1d..82de2e63461 100644 --- a/src/test/regress/expected/mdb_superuser.out +++ b/src/test/regress/expected/mdb_superuser.out @@ -4,7 +4,6 @@ CREATE ROLE regress_mdb_superuser_user3; CREATE ROLE regress_mdb_su_role_o1; CREATE ROLE regress_mdb_su_role_o2; GRANT mdb_superuser TO regress_mdb_su_role_o1; -GRANT mdb_admin TO mdb_superuser; CREATE ROLE regress_superuser WITH SUPERUSER; GRANT mdb_superuser TO regress_mdb_superuser_user1; GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; @@ -59,7 +58,7 @@ DROP TABLE regress_pgwsf_table; DROP TABLE regress_pgxsp_table; DROP TABLE regress_pgrad_table; DROP TABLE regress_pgrwd_table; --- does allowed to creare database, role or extension +-- does NOT allowed to create database, role or extension -- or grant such priviledge CREATE DATABASE regress_db_fail; ERROR: permission denied to create database @@ -128,3 +127,4 @@ DROP SCHEMA regress_mdb_superuser_schema; DROP ROLE regress_mdb_superuser_user1; DROP ROLE regress_mdb_superuser_user2; DROP ROLE regress_mdb_superuser_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/expected/test_setup.out b/src/test/regress/expected/test_setup.out index ffd15883d10..049df6b67cd 100644 --- a/src/test/regress/expected/test_setup.out +++ b/src/test/regress/expected/test_setup.out @@ -245,5 +245,8 @@ create function fipshash(text) return substr(encode(sha256($1::bytea), 'hex'), 1, 32); CREATE ROLE mdb_admin; CREATE ROLE mdb_superuser; +CREATE ROLE mdb_read_all_data; +CREATE ROLE mdb_write_all_data; CREATE ROLE mdb_replication; +GRANT mdb_admin TO mdb_superuser; GRANT pg_create_subscription TO mdb_admin; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 23b862c0b3b..aa1e46ec076 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -21,6 +21,8 @@ test: mdb_replication test: mdb_copy +test: mdb_read_write_roles + # ---------- # The first group of parallel tests # ---------- diff --git a/src/test/regress/sql/mdb_read_write_roles.sql b/src/test/regress/sql/mdb_read_write_roles.sql new file mode 100644 index 00000000000..291f63297a1 --- /dev/null +++ b/src/test/regress/sql/mdb_read_write_roles.sql @@ -0,0 +1,89 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; + +CREATE ROLE regress_superuser WITH SUPERUSER; + +GRANT mdb_superuser TO regress_mdb_superuser_user1; + +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; + +SET ROLE regress_superuser; +CREATE TABLE regress_superuser_table(); + +SET ROLE pg_read_server_files; +CREATE TABLE regress_pgrsf_table(); + +SET ROLE pg_write_server_files; +CREATE TABLE regress_pgwsf_table(); + +SET ROLE pg_execute_server_program; +CREATE TABLE regress_pgxsp_table(); + +SET ROLE pg_read_all_data; +CREATE TABLE regress_pgrad_table(); + +SET ROLE pg_write_all_data; +CREATE TABLE regress_pgrwd_table(); + +SET ROLE regress_mdb_superuser_user1; +CREATE TABLE regress_mdbsu_table(); + +SET ROLE regress_mdb_superuser_user2; +CREATE TABLE regress_mdbsu_table2(); + +SET ROLE mdb_read_all_data; +-- cannot read all data (fail) +TABLE pg_authid; +TABLE regress_superuser_table; +TABLE regress_pgrsf_table; +TABLE regress_pgwsf_table; +TABLE regress_pgxsp_table; +TABLE regress_pgrad_table; +TABLE regress_pgwsf_table; + + +-- is allow to read all other data + +TABLE regress_mdbsu_table; +TABLE regress_mdbsu_table2; + + +SET ROLE mdb_write_all_data; +CREATE TABLE regress_tt_dat(); +-- cannot read all data (fail) +INSERT INTO regress_superuser_table TABLE regress_tt_dat; +INSERT INTO regress_pgrsf_table TABLE regress_tt_dat; +INSERT INTO regress_pgwsf_table TABLE regress_tt_dat; +INSERT INTO regress_pgxsp_table TABLE regress_tt_dat; +INSERT INTO regress_pgrad_table TABLE regress_tt_dat; +INSERT INTO regress_pgwsf_table TABLE regress_tt_dat; + + +-- is allow to read all other data + +INSERT INTO regress_mdbsu_table TABLE regress_tt_dat; +INSERT INTO regress_mdbsu_table2 TABLE regress_tt_dat; + +-- end tests + +RESET SESSION AUTHORIZATION; +-- + +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; + +DROP TABLE regress_mdbsu_table; +DROP TABLE regress_mdbsu_table2; +DROP TABLE regress_superuser_table; + +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; + +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/sql/mdb_superuser.sql b/src/test/regress/sql/mdb_superuser.sql index 0877d9258f6..cc5caf531b3 100644 --- a/src/test/regress/sql/mdb_superuser.sql +++ b/src/test/regress/sql/mdb_superuser.sql @@ -7,8 +7,6 @@ CREATE ROLE regress_mdb_su_role_o2; GRANT mdb_superuser TO regress_mdb_su_role_o1; -GRANT mdb_admin TO mdb_superuser; - CREATE ROLE regress_superuser WITH SUPERUSER; GRANT mdb_superuser TO regress_mdb_superuser_user1; @@ -90,7 +88,7 @@ DROP TABLE regress_pgrad_table; DROP TABLE regress_pgrwd_table; --- does allowed to creare database, role or extension +-- does NOT allowed to create database, role or extension -- or grant such priviledge CREATE DATABASE regress_db_fail; @@ -162,3 +160,4 @@ DROP SCHEMA regress_mdb_superuser_schema; DROP ROLE regress_mdb_superuser_user1; DROP ROLE regress_mdb_superuser_user2; DROP ROLE regress_mdb_superuser_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/sql/test_setup.sql b/src/test/regress/sql/test_setup.sql index 1e2e50f9009..eae213fc25e 100644 --- a/src/test/regress/sql/test_setup.sql +++ b/src/test/regress/sql/test_setup.sql @@ -302,6 +302,9 @@ create function fipshash(text) CREATE ROLE mdb_admin; CREATE ROLE mdb_superuser; +CREATE ROLE mdb_read_all_data; +CREATE ROLE mdb_write_all_data; CREATE ROLE mdb_replication; +GRANT mdb_admin TO mdb_superuser; GRANT pg_create_subscription TO mdb_admin; From 353f194e02f29954941a07ee395ada929a2a7cf7 Mon Sep 17 00:00:00 2001 From: reshke Date: Sat, 26 Apr 2025 08:16:33 +0000 Subject: [PATCH 37/54] Add check for mdb_service_auth role --- src/backend/catalog/aclchk.c | 4 ++-- src/backend/utils/adt/acl.c | 20 +++++++++++++++----- src/include/utils/acl.h | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index 1133ee0dcdc..7d6687c3225 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -3390,7 +3390,7 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, */ if (mask & ACL_SELECT && !(result & ACL_SELECT) && has_privs_of_role(roleid, mdb_read_all_data_oid) && - !has_privs_of_unwanted_system_role(ownerId)) + !has_privs_of_unwanted_system_role(ownerId, true)) result |= ACL_SELECT; /* @@ -3414,7 +3414,7 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, if (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE) && !(result & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)) && has_privs_of_role(roleid, mdb_write_all_data_oid) && - !has_privs_of_unwanted_system_role(ownerId)) + !has_privs_of_unwanted_system_role(ownerId, true)) result |= (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)); return result; diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index a9bffa5277a..e6bd0bfa7fd 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5030,7 +5030,9 @@ has_privs_of_role_strict_no_cache(Oid member, Oid role) */ bool -has_privs_of_unwanted_system_role(Oid role) { +has_privs_of_unwanted_system_role(Oid role, bool check_mdb_service_auth) { + Oid mdb_service_authoid; + if (has_privs_of_role_strict(role, ROLE_PG_READ_SERVER_FILES)) { return true; } @@ -5047,6 +5049,14 @@ has_privs_of_unwanted_system_role(Oid role) { return true; } + if (check_mdb_service_auth) { + mdb_service_authoid = get_role_oid("mdb_service_auth", true); + + if (has_privs_of_role_strict(role, mdb_service_authoid)) { + return true; + } + } + return false; } @@ -5093,7 +5103,7 @@ has_privs_of_role(Oid member, Oid role) * if target role is neither superuser nor * some dangerous system role */ - if (!has_privs_of_unwanted_system_role(role)) { + if (!has_privs_of_unwanted_system_role(role, true)) { return true; } } @@ -5148,7 +5158,7 @@ mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) */ /* All checks passed, hope will not be hacked here (again) */ - return !has_privs_of_unwanted_system_role(ownerId); + return !has_privs_of_unwanted_system_role(ownerId, true); } // -- non-upstream patch end @@ -5227,7 +5237,7 @@ check_mdb_admin_is_member_of_role(Oid member, Oid role) GetUserNameFromId(role, false)))); } - if (has_privs_of_unwanted_system_role(role)) { + if (has_privs_of_unwanted_system_role(role, true)) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("forbidden to transfer ownership to this system role in Cloud"))); @@ -5261,7 +5271,7 @@ bool mdb_admin_is_member_of_role(Oid member, Oid role) { return false; } - return !has_privs_of_unwanted_system_role(role); + return !has_privs_of_unwanted_system_role(role, true); } // -- mdb admin patch diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 0cd8dc4d7a9..3dd3e9b4f31 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -230,7 +230,7 @@ extern bool mdb_admin_is_member_of_role(Oid member, Oid role); extern Oid select_best_admin(Oid member, Oid role); extern Oid get_role_oid(const char *rolname, bool missing_ok); extern Oid get_role_oid_or_public(const char *rolname); -extern bool has_privs_of_unwanted_system_role(Oid role); +extern bool has_privs_of_unwanted_system_role(Oid role, bool check_mdb_service_auth); extern Oid get_rolespec_oid(const RoleSpec *role, bool missing_ok); extern void check_rolespec_name(const RoleSpec *role, const char *detail_msg); extern HeapTuple get_rolespec_tuple(const RoleSpec *role); From a06e7ac6fae79f96733caa8e6c88964265be74d7 Mon Sep 17 00:00:00 2001 From: reshke Date: Sat, 26 Apr 2025 11:35:01 +0500 Subject: [PATCH 38/54] Fix CI && fast CI circuit (#24) --- .github/workflows/regress.yml | 11 ++++++++++- docker/regress/run_tests.sh | 4 ++-- docker/regress/run_tests_f.sh | 6 ++++++ 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100755 docker/regress/run_tests_f.sh diff --git a/.github/workflows/regress.yml b/.github/workflows/regress.yml index 7652df14da7..c8b48bcfb3c 100644 --- a/.github/workflows/regress.yml +++ b/.github/workflows/regress.yml @@ -8,7 +8,16 @@ on: jobs: - build: + check: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Build the Docker image + run: docker build . --file docker/regress/Dockerfile --tag regress_test:1234 && docker run --entrypoint /home/build-user/docker/regress/run_tests_f.sh regress_test:1234 + + check-world: runs-on: ubuntu-latest diff --git a/docker/regress/run_tests.sh b/docker/regress/run_tests.sh index b48b1472953..5b72eaea60d 100755 --- a/docker/regress/run_tests.sh +++ b/docker/regress/run_tests.sh @@ -1,6 +1,6 @@ #!/bin/bash set -ex -sed -i '/mdb-related/,$d' src/test/regress/expected/misc.out src/test/regress/sql/misc.sql +sed -i '/mdb-related/,$d' src/test/regress/*/misc.* -make check check-world +make check-world diff --git a/docker/regress/run_tests_f.sh b/docker/regress/run_tests_f.sh new file mode 100755 index 00000000000..a7ff7f136aa --- /dev/null +++ b/docker/regress/run_tests_f.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -ex + +sed -i '/mdb-related/,$d' src/test/regress/*/misc.* + +make check From 2c1ab98426e529aff950b3d37e7fbe4d4f75227f Mon Sep 17 00:00:00 2001 From: reshke Date: Sat, 26 Apr 2025 12:05:23 +0500 Subject: [PATCH 39/54] Allow usage on schema for mdb_read_all_data (#23) --- src/backend/catalog/aclchk.c | 10 ++++++++++ .../regress/expected/mdb_read_write_roles.out | 15 +++++++++++++++ src/test/regress/sql/mdb_read_write_roles.sql | 12 ++++++++++++ 3 files changed, 37 insertions(+) diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index 7d6687c3225..72b9f53f804 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -3630,6 +3630,7 @@ pg_namespace_aclmask(Oid nsp_oid, Oid roleid, bool isNull; Acl *acl; Oid ownerId; + Oid mdb_read_all_data_oid; /* Superusers bypass all permission checking. */ if (superuser_arg(roleid)) @@ -3674,6 +3675,8 @@ pg_namespace_aclmask(Oid nsp_oid, Oid roleid, ownerId = ((Form_pg_namespace) GETSTRUCT(tuple))->nspowner; + mdb_read_all_data_oid = get_role_oid("mdb_read_all_data", true); + aclDatum = SysCacheGetAttr(NAMESPACEOID, tuple, Anum_pg_namespace_nspacl, &isNull); if (isNull) @@ -3706,6 +3709,13 @@ pg_namespace_aclmask(Oid nsp_oid, Oid roleid, (has_privs_of_role(roleid, ROLE_PG_READ_ALL_DATA) || has_privs_of_role(roleid, ROLE_PG_WRITE_ALL_DATA))) result |= ACL_USAGE; + + + if (mask & ACL_USAGE && !(result & ACL_USAGE) && + has_privs_of_role(roleid, mdb_read_all_data_oid) && + !has_privs_of_unwanted_system_role(ownerId, true)) + result |= ACL_USAGE; + return result; } diff --git a/src/test/regress/expected/mdb_read_write_roles.out b/src/test/regress/expected/mdb_read_write_roles.out index 080e95cafb9..4c8b04eec89 100644 --- a/src/test/regress/expected/mdb_read_write_roles.out +++ b/src/test/regress/expected/mdb_read_write_roles.out @@ -18,8 +18,10 @@ SET ROLE pg_write_all_data; CREATE TABLE regress_pgrwd_table(); SET ROLE regress_mdb_superuser_user1; CREATE TABLE regress_mdbsu_table(); +CREATE SCHEMA regress_schema CREATE TABLE regress_mdbsu_table(); SET ROLE regress_mdb_superuser_user2; CREATE TABLE regress_mdbsu_table2(); +CREATE SCHEMA regress_schema2 CREATE TABLE regress_mdbsu_table2(); SET ROLE mdb_read_all_data; -- cannot read all data (fail) TABLE pg_authid; @@ -45,6 +47,15 @@ TABLE regress_mdbsu_table2; -- (0 rows) +-- check USAGE of schema +TABLE regress_schema.regress_mdbsu_table; +-- +(0 rows) + +TABLE regress_schema2.regress_mdbsu_table2; +-- +(0 rows) + SET ROLE mdb_write_all_data; CREATE TABLE regress_tt_dat(); -- cannot read all data (fail) @@ -73,6 +84,10 @@ DROP TABLE regress_pgrad_table; DROP TABLE regress_pgrwd_table; DROP TABLE regress_mdbsu_table; DROP TABLE regress_mdbsu_table2; +DROP TABLE regress_schema.regress_mdbsu_table; +DROP TABLE regress_schema2.regress_mdbsu_table2; +DROP SCHEMA regress_schema; +DROP SCHEMA regress_schema2; DROP TABLE regress_superuser_table; REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; diff --git a/src/test/regress/sql/mdb_read_write_roles.sql b/src/test/regress/sql/mdb_read_write_roles.sql index 291f63297a1..0522e45a631 100644 --- a/src/test/regress/sql/mdb_read_write_roles.sql +++ b/src/test/regress/sql/mdb_read_write_roles.sql @@ -28,9 +28,11 @@ CREATE TABLE regress_pgrwd_table(); SET ROLE regress_mdb_superuser_user1; CREATE TABLE regress_mdbsu_table(); +CREATE SCHEMA regress_schema CREATE TABLE regress_mdbsu_table(); SET ROLE regress_mdb_superuser_user2; CREATE TABLE regress_mdbsu_table2(); +CREATE SCHEMA regress_schema2 CREATE TABLE regress_mdbsu_table2(); SET ROLE mdb_read_all_data; -- cannot read all data (fail) @@ -48,6 +50,9 @@ TABLE regress_pgwsf_table; TABLE regress_mdbsu_table; TABLE regress_mdbsu_table2; +-- check USAGE of schema +TABLE regress_schema.regress_mdbsu_table; +TABLE regress_schema2.regress_mdbsu_table2; SET ROLE mdb_write_all_data; CREATE TABLE regress_tt_dat(); @@ -78,6 +83,13 @@ DROP TABLE regress_pgrwd_table; DROP TABLE regress_mdbsu_table; DROP TABLE regress_mdbsu_table2; + +DROP TABLE regress_schema.regress_mdbsu_table; +DROP TABLE regress_schema2.regress_mdbsu_table2; + +DROP SCHEMA regress_schema; +DROP SCHEMA regress_schema2; + DROP TABLE regress_superuser_table; REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; From 3d458c9e08ba6a3cca35bde50daf98327a445a62 Mon Sep 17 00:00:00 2001 From: Yury Frolov <57130330+EinKrebs@users.noreply.github.com> Date: Fri, 30 May 2025 17:00:10 +0500 Subject: [PATCH 40/54] Use mirror apt repo (#39) --- docker/regress/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/regress/Dockerfile b/docker/regress/Dockerfile index 0546ec5d727..b812d09225c 100644 --- a/docker/regress/Dockerfile +++ b/docker/regress/Dockerfile @@ -8,10 +8,11 @@ ENV DEBIAN_FRONTEND=noninteractive ENV TZ=Europe/Moskow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN sed -i 's/archive.ubuntu.com/mirror.yandex.ru/g' /etc/apt/sources.list +RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install -y --no-install-recommends --allow-unauthenticated \ sudo build-essential \ gcc lsb-release libssl-dev gnupg openssl \ - gdb git curl + gdb git curl ca-certificates RUN curl -s 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xafc3ce0d00e3c45a357e9e637fcd11186050cd1a' | \ gpg --dearmour -o /etc/apt/trusted.gpg.d/yandex.gpg From 313083bdcd8594a3e2dc581a474d124c4589730c Mon Sep 17 00:00:00 2001 From: Yury Frolov Date: Mon, 2 Jun 2025 14:51:20 +0500 Subject: [PATCH 41/54] Refactor regress Dockerfile --- docker/regress/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/regress/Dockerfile b/docker/regress/Dockerfile index b812d09225c..67ee2d63644 100644 --- a/docker/regress/Dockerfile +++ b/docker/regress/Dockerfile @@ -8,8 +8,9 @@ ENV DEBIAN_FRONTEND=noninteractive ENV TZ=Europe/Moskow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN apt update && apt install -y ca-certificates RUN sed -i 's/archive.ubuntu.com/mirror.yandex.ru/g' /etc/apt/sources.list -RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install -y --no-install-recommends --allow-unauthenticated \ +RUN apt-get update && apt-get install -y --no-install-recommends \ sudo build-essential \ gcc lsb-release libssl-dev gnupg openssl \ gdb git curl ca-certificates From 396d433881f262ded12193be8c0267efde2ff4cb Mon Sep 17 00:00:00 2001 From: reshke Date: Sun, 17 Aug 2025 22:51:22 +0500 Subject: [PATCH 42/54] Update Dockerfile: try --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 2fb32956ea4..db298dfe507 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,8 @@ RUN echo "deb http://dist.yandex.ru/mdb-${CODE_NAME}-secure stable/\$(ARCH)/" >> RUN curl -s 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xafc3ce0d00e3c45a357e9e637fcd11186050cd1a' | \ gpg --dearmour -o /etc/apt/trusted.gpg.d/yandex.gpg +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys FF5F4D0E27393420 + RUN apt-get update && apt-get install -y --no-install-recommends \ sudo build-essential \ gcc lsb-release libssl-dev gnupg openssl \ From d044e2f4991b170604f2d8e3d2021bcfab7773a2 Mon Sep 17 00:00:00 2001 From: rkhapov Date: Mon, 12 May 2025 15:29:17 +0500 Subject: [PATCH 43/54] pg_stat_statements.c: cancelable qtext_load_file In the case of a large PGSS_TEXT_FILE, the work time of the qtext_load_file function will be quite long, and the query to the pg_stat_statements table will not be cancellable, as there is no CHECK_FOR_INTERRUPT in the function. Also, the amount of bytes read can reach 1 GB, which leads to a slow read system call that does not allow cancellation of the query. Testing the speed of sequential read using fio with different block sizes shows that there is no significant difference between 16 MB blocks and 1 GB blocks. Therefore, this patch changes the maximum read value from 1 GB to 16 MB and adds INTERRUPTS_PENDING_CONDITION() check in the read loop of qtext_load_file to make it cancellable. For now, only statement execution is cancellable (fail_on_interrupt is true only for calls from pg_stat_statements_internal) Signed-off-by: rkhapov Reviewed-by: reshke --- .../pg_stat_statements/pg_stat_statements.c | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 92282d2b586..201959339c5 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -353,7 +353,7 @@ static pgssEntry *entry_alloc(pgssHashKey *key, Size query_offset, int query_len static void entry_dealloc(void); static bool qtext_store(const char *query, int query_len, Size *query_offset, int *gc_count); -static char *qtext_load_file(Size *buffer_size); +static char *qtext_load_file(Size *buffer_size, bool fail_on_interrupts); static char *qtext_fetch(Size query_offset, int query_len, char *buffer, Size buffer_size); static bool need_gc_qtexts(void); @@ -753,7 +753,7 @@ pgss_shmem_shutdown(int code, Datum arg) if (fwrite(&num_entries, sizeof(int32), 1, file) != 1) goto error; - qbuffer = qtext_load_file(&qbuffer_size); + qbuffer = qtext_load_file(&qbuffer_size, false /* fail_on_interrupts */ ); if (qbuffer == NULL) goto error; @@ -1650,7 +1650,7 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, /* No point in loading file now if there are active writers */ if (n_writers == 0) - qbuffer = qtext_load_file(&qbuffer_size); + qbuffer = qtext_load_file(&qbuffer_size, true /* fail_on_interrupts */ ); } /* @@ -1684,7 +1684,7 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, { if (qbuffer) pfree(qbuffer); - qbuffer = qtext_load_file(&qbuffer_size); + qbuffer = qtext_load_file(&qbuffer_size, true /* fail_on_interrupts */ ); } } @@ -1701,6 +1701,12 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); + /* Can't process interrupts here - pgss-lock is acquired */ + if (INTERRUPTS_PENDING_CONDITION()) + { + break; + } + values[i++] = ObjectIdGetDatum(entry->key.userid); values[i++] = ObjectIdGetDatum(entry->key.dbid); if (api_version >= PGSS_V1_9) @@ -1869,6 +1875,7 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, if (qbuffer) pfree(qbuffer); + CHECK_FOR_INTERRUPTS(); } /* Number of output arguments (columns) for pg_stat_statements_info */ @@ -2184,7 +2191,7 @@ qtext_store(const char *query, int query_len, * the caller is responsible for verifying that the result is sane. */ static char * -qtext_load_file(Size *buffer_size) +qtext_load_file(Size *buffer_size, bool fail_on_interrupts) { char *buf; int fd; @@ -2237,7 +2244,14 @@ qtext_load_file(Size *buffer_size) nread = 0; while (nread < stat.st_size) { - int toread = Min(1024 * 1024 * 1024, stat.st_size - nread); + int toread = Min(32 * 1024 * 1024, stat.st_size - nread); + + if (fail_on_interrupts && INTERRUPTS_PENDING_CONDITION()) + { + free(buf); + CloseTransientFile(fd); + return NULL; + } /* * If we get a short read and errno doesn't get set, the reason is @@ -2378,7 +2392,7 @@ gc_qtexts(void) * file is only going to get bigger; hoping for a future non-OOM result is * risky and can easily lead to complete denial of service. */ - qbuffer = qtext_load_file(&qbuffer_size); + qbuffer = qtext_load_file(&qbuffer_size, false /* fail_on_interrupts */ ); if (qbuffer == NULL) goto gc_fail; From 6fb2c2bbb6b65959f8b1e24e9c687fc648ed4063 Mon Sep 17 00:00:00 2001 From: Victor Popov Date: Mon, 1 Dec 2025 17:24:19 +0300 Subject: [PATCH 44/54] MDB-40410: Allow to kill backends which have application_name starting with "MDB" instead of exactly matching --- src/backend/storage/ipc/signalfuncs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index e8b40f17cf4..6aa6de0bdcf 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -106,7 +106,7 @@ pg_signal_backend(int pid, int sig) /* mdb admin allowed to kill proc with application name 'MDB' or autovacuum */ if (local_beentry->backendStatus.st_backendType == B_AUTOVAC_WORKER) { // ok - } else if (appname != NULL && strcmp(appname, "MDB") == 0) { + } else if (appname != NULL && strncmp(appname, "MDB", 3) == 0) { // ok } else { return SIGNAL_BACKEND_NOSUPERUSER; From a41be221e2f48bfd59a9c76a7ef7faedff0ed1a5 Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Mon, 25 Nov 2024 16:36:37 -0600 Subject: [PATCH 45/54] pg_dump: Add dumpSchema and dumpData derivative flags. Various parts of pg_dump consult the --schema-only and --data-only options to determine whether to run a section of code. While this is simple enough for two mutually-exclusive options, it will become progressively more complicated as more options are added. In anticipation of that, this commit introduces new internal flags called dumpSchema and dumpData, which are derivatives of --schema-only and --data-only. This commit also removes the schemaOnly and dataOnly members from the dump/restore options structs to prevent their use elsewhere. Note that this change neither adds new user-facing command-line options nor changes the existing --schema-only and --data-only options. Author: Corey Huinker Reviewed-by: Jeff Davis Discussion: https://postgr.es/m/CADkLM%3DcQgghMJOS8EcAVBwRO4s1dUVtxGZv5gLPfZkQ1nL1gzA%40mail.gmail.com --- src/bin/pg_dump/pg_backup.h | 10 +- src/bin/pg_dump/pg_backup_archiver.c | 34 +++-- src/bin/pg_dump/pg_dump.c | 202 ++++++++++++++------------- src/bin/pg_dump/pg_restore.c | 14 +- 4 files changed, 139 insertions(+), 121 deletions(-) diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 558a8f00abf..30dbc3b2df8 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -114,8 +114,6 @@ typedef struct _restoreOptions int strict_names; const char *filename; - int dataOnly; - int schemaOnly; int dumpSections; int verbose; int aclsSkip; @@ -156,6 +154,9 @@ typedef struct _restoreOptions int binary_upgrade; char *restrict_key; + /* flags derived from the user-settable flags */ + bool dumpSchema; + bool dumpData; } RestoreOptions; typedef struct _dumpOptions @@ -165,8 +166,6 @@ typedef struct _dumpOptions int binary_upgrade; /* various user-settable parameters */ - bool schemaOnly; - bool dataOnly; int dumpSections; /* bitmask of chosen sections */ bool aclsSkip; const char *lockWaitTimeout; @@ -204,6 +203,9 @@ typedef struct _dumpOptions int do_nothing; char *restrict_key; + /* flags derived from the user-settable flags */ + bool dumpSchema; + bool dumpData; } DumpOptions; /* diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 74e3c564972..8e87d5d0f8a 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -163,6 +163,8 @@ InitDumpOptions(DumpOptions *opts) opts->include_everything = true; opts->cparams.promptPassword = TRI_DEFAULT; opts->dumpSections = DUMP_UNSECTIONED; + opts->dumpSchema = true; + opts->dumpData = true; } /* @@ -181,8 +183,8 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt) dopt->cparams.username = ropt->cparams.username ? pg_strdup(ropt->cparams.username) : NULL; dopt->cparams.promptPassword = ropt->cparams.promptPassword; dopt->outputClean = ropt->dropSchema; - dopt->dataOnly = ropt->dataOnly; - dopt->schemaOnly = ropt->schemaOnly; + dopt->dumpData = ropt->dumpData; + dopt->dumpSchema = ropt->dumpSchema; dopt->if_exists = ropt->if_exists; dopt->column_inserts = ropt->column_inserts; dopt->dumpSections = ropt->dumpSections; @@ -434,12 +436,12 @@ RestoreArchive(Archive *AHX) * Work out if we have an implied data-only restore. This can happen if * the dump was data only or if the user has used a toc list to exclude * all of the schema data. All we do is look for schema entries - if none - * are found then we set the dataOnly flag. + * are found then we unset the dumpSchema flag. * * We could scan for wanted TABLE entries, but that is not the same as - * dataOnly. At this stage, it seems unnecessary (6-Mar-2001). + * data-only. At this stage, it seems unnecessary (6-Mar-2001). */ - if (!ropt->dataOnly) + if (ropt->dumpSchema) { int impliedDataOnly = 1; @@ -453,7 +455,7 @@ RestoreArchive(Archive *AHX) } if (impliedDataOnly) { - ropt->dataOnly = impliedDataOnly; + ropt->dumpSchema = false; pg_log_info("implied data-only restore"); } } @@ -793,7 +795,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel) /* Dump any relevant dump warnings to stderr */ if (!ropt->suppressDumpWarnings && strcmp(te->desc, "WARNING") == 0) { - if (!ropt->dataOnly && te->defn != NULL && strlen(te->defn) != 0) + if (ropt->dumpSchema && te->defn != NULL && strlen(te->defn) != 0) pg_log_warning("warning from original dump file: %s", te->defn); else if (te->copyStmt != NULL && strlen(te->copyStmt) != 0) pg_log_warning("warning from original dump file: %s", te->copyStmt); @@ -1011,6 +1013,8 @@ NewRestoreOptions(void) opts->dumpSections = DUMP_UNSECTIONED; opts->compression_spec.algorithm = PG_COMPRESSION_NONE; opts->compression_spec.level = 0; + opts->dumpSchema = true; + opts->dumpData = true; return opts; } @@ -1021,7 +1025,7 @@ _disableTriggersIfNecessary(ArchiveHandle *AH, TocEntry *te) RestoreOptions *ropt = AH->public.ropt; /* This hack is only needed in a data-only restore */ - if (!ropt->dataOnly || !ropt->disable_triggers) + if (ropt->dumpSchema || !ropt->disable_triggers) return; pg_log_info("disabling triggers for %s", te->tag); @@ -1047,7 +1051,7 @@ _enableTriggersIfNecessary(ArchiveHandle *AH, TocEntry *te) RestoreOptions *ropt = AH->public.ropt; /* This hack is only needed in a data-only restore */ - if (!ropt->dataOnly || !ropt->disable_triggers) + if (ropt->dumpSchema || !ropt->disable_triggers) return; pg_log_info("enabling triggers for %s", te->tag); @@ -3090,13 +3094,13 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) if ((strcmp(te->desc, "") == 0) && (strcmp(te->tag, "Max OID") == 0)) return 0; - /* Mask it if we only want schema */ - if (ropt->schemaOnly) + /* Mask it if we don't want data */ + if (!ropt->dumpData) { /* - * The sequence_data option overrides schemaOnly for SEQUENCE SET. + * The sequence_data option overrides dumpData for SEQUENCE SET. * - * In binary-upgrade mode, even with schemaOnly set, we do not mask + * In binary-upgrade mode, even with dumpData unset, we do not mask * out large objects. (Only large object definitions, comments and * other metadata should be generated in binary-upgrade mode, not the * actual data, but that need not concern us here.) @@ -3113,8 +3117,8 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) res = res & REQ_SCHEMA; } - /* Mask it if we only want data */ - if (ropt->dataOnly) + /* Mask it if we don't want schema */ + if (!ropt->dumpSchema) res = res & REQ_DATA; return res; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 597c828db81..09943d457c0 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -358,6 +358,8 @@ main(int argc, char **argv) char *compression_algorithm_str = "none"; char *error_detail = NULL; bool user_compression_defined = false; + bool data_only = false; + bool schema_only = false; static DumpOptions dopt; @@ -471,7 +473,7 @@ main(int argc, char **argv) switch (c) { case 'a': /* Dump data only */ - dopt.dataOnly = true; + data_only = true; break; case 'b': /* Dump LOs */ @@ -544,7 +546,7 @@ main(int argc, char **argv) break; case 's': /* dump schema only */ - dopt.schemaOnly = true; + schema_only = true; break; case 'S': /* Username for superuser in plain text output */ @@ -698,21 +700,25 @@ main(int argc, char **argv) if (dopt.binary_upgrade) dopt.sequence_data = 1; - if (dopt.dataOnly && dopt.schemaOnly) + if (data_only && schema_only) pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together"); - if (dopt.schemaOnly && foreign_servers_include_patterns.head != NULL) + if (schema_only && foreign_servers_include_patterns.head != NULL) pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together"); if (numWorkers > 1 && foreign_servers_include_patterns.head != NULL) pg_fatal("option --include-foreign-data is not supported with parallel backup"); - if (dopt.dataOnly && dopt.outputClean) + if (data_only && dopt.outputClean) pg_fatal("options -c/--clean and -a/--data-only cannot be used together"); if (dopt.if_exists && !dopt.outputClean) pg_fatal("option --if-exists requires option -c/--clean"); + /* set derivative flags */ + dopt.dumpSchema = (!data_only); + dopt.dumpData = (!schema_only); + /* * --inserts are already implied above if --column-inserts or * --rows-per-insert were specified. @@ -905,7 +911,7 @@ main(int argc, char **argv) * -s means "schema only" and LOs are data, not schema, so we never * include LOs when -s is used. */ - if (dopt.include_everything && !dopt.schemaOnly && !dopt.dontOutputLOs) + if (dopt.include_everything && dopt.dumpData && !dopt.dontOutputLOs) dopt.outputLOs = true; /* @@ -919,15 +925,15 @@ main(int argc, char **argv) */ tblinfo = getSchemaData(fout, &numTables); - if (!dopt.schemaOnly) + if (dopt.dumpData) { getTableData(&dopt, tblinfo, numTables, 0); buildMatViewRefreshDependencies(fout); - if (dopt.dataOnly) + if (!dopt.dumpSchema) getTableDataFKConstraints(); } - if (dopt.schemaOnly && dopt.sequence_data) + if (!dopt.dumpData && dopt.sequence_data) getTableData(&dopt, tblinfo, numTables, RELKIND_SEQUENCE); /* @@ -1012,8 +1018,8 @@ main(int argc, char **argv) ropt->cparams.username = dopt.cparams.username ? pg_strdup(dopt.cparams.username) : NULL; ropt->cparams.promptPassword = dopt.cparams.promptPassword; ropt->dropSchema = dopt.outputClean; - ropt->dataOnly = dopt.dataOnly; - ropt->schemaOnly = dopt.schemaOnly; + ropt->dumpData = dopt.dumpData; + ropt->dumpSchema = dopt.dumpSchema; ropt->if_exists = dopt.if_exists; ropt->column_inserts = dopt.column_inserts; ropt->dumpSections = dopt.dumpSections; @@ -1905,7 +1911,7 @@ selectDumpableType(TypeInfo *tyinfo, Archive *fout) * Mark a default ACL as to be dumped or not * * For per-schema default ACLs, dump if the schema is to be dumped. - * Otherwise dump if we are dumping "everything". Note that dataOnly + * Otherwise dump if we are dumping "everything". Note that dumpSchema * and aclsSkip are checked separately. */ static void @@ -4004,8 +4010,8 @@ dumpPolicy(Archive *fout, const PolicyInfo *polinfo) const char *cmd; char *tag; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* @@ -4223,8 +4229,8 @@ dumpPublication(Archive *fout, const PublicationInfo *pubinfo) char *qpubname; bool first = true; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; delq = createPQExpBuffer(); @@ -4538,8 +4544,8 @@ dumpPublicationNamespace(Archive *fout, const PublicationSchemaInfo *pubsinfo) PQExpBuffer query; char *tag; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; tag = psprintf("%s %s", pubinfo->dobj.name, schemainfo->dobj.name); @@ -4581,8 +4587,8 @@ dumpPublicationTable(Archive *fout, const PublicationRelInfo *pubrinfo) PQExpBuffer query; char *tag; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; tag = psprintf("%s %s", pubinfo->dobj.name, tbinfo->dobj.name); @@ -4849,8 +4855,8 @@ dumpSubscription(Archive *fout, const SubscriptionInfo *subinfo) int i; char two_phase_disabled[] = {LOGICALREP_TWOPHASE_STATE_DISABLED, '\0'}; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; delq = createPQExpBuffer(); @@ -7025,8 +7031,8 @@ getPartitioningInfo(Archive *fout) /* hash partitioning didn't exist before v11 */ if (fout->remoteVersion < 110000) return; - /* needn't bother if schema-only dump */ - if (fout->dopt->schemaOnly) + /* needn't bother if not dumping data */ + if (!fout->dopt->dumpData) return; query = createPQExpBuffer(); @@ -8770,7 +8776,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) * Now get info about column defaults. This is skipped for a data-only * dump, as it is only needed for table schemas. */ - if (!dopt->dataOnly && tbloids->len > 1) + if (dopt->dumpSchema && tbloids->len > 1) { AttrDefInfo *attrdefs; int numDefaults; @@ -8900,7 +8906,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) * Get info about table CHECK constraints. This is skipped for a * data-only dump, as it is only needed for table schemas. */ - if (!dopt->dataOnly && checkoids->len > 2) + if (dopt->dumpSchema && checkoids->len > 2) { ConstraintInfo *constrs; int numConstrs; @@ -9852,13 +9858,13 @@ dumpCommentExtended(Archive *fout, const char *type, /* Comments are schema not data ... except LO comments are data */ if (strcmp(type, "LARGE OBJECT") != 0) { - if (dopt->dataOnly) + if (!dopt->dumpSchema) return; } else { /* We do dump LO comments in binary-upgrade mode */ - if (dopt->schemaOnly && !dopt->binary_upgrade) + if (!dopt->dumpData && !dopt->binary_upgrade) return; } @@ -9965,7 +9971,7 @@ dumpTableComment(Archive *fout, const TableInfo *tbinfo, return; /* Comments are SCHEMA not data */ - if (dopt->dataOnly) + if (!dopt->dumpSchema) return; /* Search for comments associated with relation, using table */ @@ -10402,8 +10408,8 @@ dumpNamespace(Archive *fout, const NamespaceInfo *nspinfo) PQExpBuffer delq; char *qnspname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -10479,8 +10485,8 @@ dumpExtension(Archive *fout, const ExtensionInfo *extinfo) PQExpBuffer delq; char *qextname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -10604,8 +10610,8 @@ dumpType(Archive *fout, const TypeInfo *tyinfo) { DumpOptions *dopt = fout->dopt; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* Dump out in proper style */ @@ -11720,8 +11726,8 @@ dumpShellType(Archive *fout, const ShellTypeInfo *stinfo) DumpOptions *dopt = fout->dopt; PQExpBuffer q; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -11772,8 +11778,8 @@ dumpProcLang(Archive *fout, const ProcLangInfo *plang) FuncInfo *inlineInfo = NULL; FuncInfo *validatorInfo = NULL; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* @@ -11980,8 +11986,8 @@ dumpFunc(Archive *fout, const FuncInfo *finfo) int nconfigitems = 0; const char *keyword; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -12372,8 +12378,8 @@ dumpCast(Archive *fout, const CastInfo *cast) const char *sourceType; const char *targetType; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* Cannot dump if we don't have the cast function's info */ @@ -12478,8 +12484,8 @@ dumpTransform(Archive *fout, const TransformInfo *transform) char *lanname; const char *transformType; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* Cannot dump if we don't have the transform functions' info */ @@ -12627,8 +12633,8 @@ dumpOpr(Archive *fout, const OprInfo *oprinfo) char *oprregproc; char *oprref; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* @@ -12914,8 +12920,8 @@ dumpAccessMethod(Archive *fout, const AccessMethodInfo *aminfo) PQExpBuffer delq; char *qamname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -13017,8 +13023,8 @@ dumpOpclass(Archive *fout, const OpclassInfo *opcinfo) bool needComma; int i; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -13288,8 +13294,8 @@ dumpOpfamily(Archive *fout, const OpfamilyInfo *opfinfo) bool needComma; int i; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -13495,8 +13501,8 @@ dumpCollation(Archive *fout, const CollInfo *collinfo) const char *colliculocale; const char *collicurules; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -13735,8 +13741,8 @@ dumpConversion(Archive *fout, const ConvInfo *convinfo) const char *conproc; bool condefault; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -13883,8 +13889,8 @@ dumpAgg(Archive *fout, const AggInfo *agginfo) const char *proparallel; char defaultfinalmodify; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -14213,8 +14219,8 @@ dumpTSParser(Archive *fout, const TSParserInfo *prsinfo) PQExpBuffer delq; char *qprsname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -14281,8 +14287,8 @@ dumpTSDictionary(Archive *fout, const TSDictInfo *dictinfo) char *nspname; char *tmplname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -14357,8 +14363,8 @@ dumpTSTemplate(Archive *fout, const TSTemplateInfo *tmplinfo) PQExpBuffer delq; char *qtmplname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -14423,8 +14429,8 @@ dumpTSConfig(Archive *fout, const TSConfigInfo *cfginfo) int i_tokenname; int i_dictname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -14535,8 +14541,8 @@ dumpForeignDataWrapper(Archive *fout, const FdwInfo *fdwinfo) PQExpBuffer delq; char *qfdwname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -14608,8 +14614,8 @@ dumpForeignServer(Archive *fout, const ForeignServerInfo *srvinfo) char *qsrvname; char *fdwname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -14799,8 +14805,8 @@ dumpDefaultACL(Archive *fout, const DefaultACLInfo *daclinfo) PQExpBuffer tag; const char *type; - /* Do nothing in data-only dump, or if we're skipping ACLs */ - if (dopt->dataOnly || dopt->aclsSkip) + /* Do nothing if not dumping schema, or if we're skipping ACLs */ + if (!dopt->dumpSchema || dopt->aclsSkip) return; q = createPQExpBuffer(); @@ -14898,7 +14904,7 @@ dumpACL(Archive *fout, DumpId objDumpId, DumpId altDumpId, return InvalidDumpId; /* --data-only skips ACLs *except* large object ACLs */ - if (dopt->dataOnly && strcmp(type, "LARGE OBJECT") != 0) + if (!dopt->dumpSchema && strcmp(type, "LARGE OBJECT") != 0) return InvalidDumpId; sql = createPQExpBuffer(); @@ -15025,13 +15031,13 @@ dumpSecLabel(Archive *fout, const char *type, const char *name, */ if (strcmp(type, "LARGE OBJECT") != 0) { - if (dopt->dataOnly) + if (!dopt->dumpSchema) return; } else { /* We do dump large object security labels in binary-upgrade mode */ - if (dopt->schemaOnly && !dopt->binary_upgrade) + if (!dopt->dumpData && !dopt->binary_upgrade) return; } @@ -15099,7 +15105,7 @@ dumpTableSecLabel(Archive *fout, const TableInfo *tbinfo, const char *reltypenam return; /* SecLabel are SCHEMA not data */ - if (dopt->dataOnly) + if (!dopt->dumpSchema) return; /* Search for comments associated with relation, using table */ @@ -15338,8 +15344,8 @@ dumpTable(Archive *fout, const TableInfo *tbinfo) DumpId tableAclDumpId = InvalidDumpId; char *namecopy; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; if (tbinfo->dobj.dump & DUMP_COMPONENT_DEFINITION) @@ -16366,8 +16372,8 @@ dumpTableAttach(Archive *fout, const TableAttachInfo *attachinfo) PGresult *res; char *partbound; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -16438,8 +16444,8 @@ dumpAttrDef(Archive *fout, const AttrDefInfo *adinfo) char *tag; char *foreign; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* Skip if not "separate"; it was dumped in the table's definition */ @@ -16527,8 +16533,8 @@ dumpIndex(Archive *fout, const IndxInfo *indxinfo) char *qindxname; char *qqindxname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -16670,8 +16676,8 @@ dumpIndex(Archive *fout, const IndxInfo *indxinfo) static void dumpIndexAttach(Archive *fout, const IndexAttachInfo *attachinfo) { - /* Do nothing in data-only dump */ - if (fout->dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!fout->dopt->dumpSchema) return; if (attachinfo->partitionIdx->dobj.dump & DUMP_COMPONENT_DEFINITION) @@ -16721,8 +16727,8 @@ dumpStatisticsExt(Archive *fout, const StatsExtInfo *statsextinfo) PGresult *res; char *stxdef; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -16798,8 +16804,8 @@ dumpConstraint(Archive *fout, const ConstraintInfo *coninfo) char *tag = NULL; char *foreign; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -17451,8 +17457,8 @@ dumpTrigger(Archive *fout, const TriggerInfo *tginfo) int findx; char *tag; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -17685,8 +17691,8 @@ dumpEventTrigger(Archive *fout, const EventTriggerInfo *evtinfo) PQExpBuffer delqry; char *qevtname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -17781,8 +17787,8 @@ dumpRule(Archive *fout, const RuleInfo *rinfo) PGresult *res; char *tag; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* @@ -18048,7 +18054,7 @@ processExtensionTables(Archive *fout, ExtensionInfo extinfo[], * objects for them, ensuring their data will be dumped even though the * tables themselves won't be. * - * Note that we create TableDataInfo objects even in schemaOnly mode, ie, + * Note that we create TableDataInfo objects even in schema-only mode, ie, * user data in a configuration table is treated like schema data. This * seems appropriate since system data in a config table would get * reloaded by CREATE EXTENSION. If the extension is not listed in the diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index f05c24510ac..a79a8f031c2 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -74,6 +74,8 @@ main(int argc, char **argv) static int no_security_labels = 0; static int no_subscriptions = 0; static int strict_names = 0; + bool data_only = false; + bool schema_only = false; struct option cmdopts[] = { {"clean", 0, NULL, 'c'}, @@ -158,7 +160,7 @@ main(int argc, char **argv) switch (c) { case 'a': /* Dump data only */ - opts->dataOnly = 1; + data_only = true; break; case 'c': /* clean (i.e., drop) schema prior to create */ opts->dropSchema = 1; @@ -234,7 +236,7 @@ main(int argc, char **argv) simple_string_list_append(&opts->triggerNames, optarg); break; case 's': /* dump schema only */ - opts->schemaOnly = 1; + schema_only = true; break; case 'S': /* Superuser username */ if (strlen(optarg) != 0) @@ -345,10 +347,10 @@ main(int argc, char **argv) pg_fatal("invalid restrict key"); } - if (opts->dataOnly && opts->schemaOnly) + if (data_only && schema_only) pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together"); - if (opts->dataOnly && opts->dropSchema) + if (data_only && opts->dropSchema) pg_fatal("options -c/--clean and -a/--data-only cannot be used together"); /* @@ -362,6 +364,10 @@ main(int argc, char **argv) if (opts->single_txn && numWorkers > 1) pg_fatal("cannot specify both --single-transaction and multiple jobs"); + /* set derivative flags */ + opts->dumpSchema = (!data_only); + opts->dumpData = (!schema_only); + opts->disable_triggers = disable_triggers; opts->enable_row_security = enable_row_security; opts->noDataForFailedTables = no_data_for_failed_tables; From ff830e7cdac117fef771997e0f299fdef10d0732 Mon Sep 17 00:00:00 2001 From: reshke Date: Mon, 8 Dec 2025 11:22:37 +0000 Subject: [PATCH 46/54] Add TAP test for MDB-kill feature --- src/test/mdb_admin/t/mdb_app_name.pl | 77 ++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/test/mdb_admin/t/mdb_app_name.pl diff --git a/src/test/mdb_admin/t/mdb_app_name.pl b/src/test/mdb_admin/t/mdb_app_name.pl new file mode 100644 index 00000000000..38fe6b029da --- /dev/null +++ b/src/test/mdb_admin/t/mdb_app_name.pl @@ -0,0 +1,77 @@ + +# Copyright (c) 2024-2024, MDB, Mother Russia + +# Minimal test testing streaming replication +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize primary node +my $node_primary = PostgreSQL::Test::Cluster->new('primary'); +$node_primary->init(); +$node_primary->start; + +# Create some content on primary and check its presence in standby nodes +$node_primary->safe_psql('postgres', + " + CREATE DATABASE regress; + CREATE ROLE mdb_admin; + CREATE ROLE mdb_reg_lh_app_name; + GRANT mdb_admin to mdb_reg_lh_app_name; + GRANT pg_signal_backend to mdb_reg_lh_app_name; + CREATE TABLE mdb_app_name_t(i int); +"); + +my $main_sess = $node_primary->background_psql('postgres'); + +$main_sess->query_safe( + q( +SET application_name TO 'SU'; +BEGIN; +INSERT INTO mdb_app_name_t VALUES(0); +)); + + +my $res_pid = $node_primary->safe_psql('regress', + " + SELECT pid FROM pg_stat_activity WHERE application_name = 'SU'; +"); + +print "pid is $res_pid\n"; + +ok(1); + + +my ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_app_name; + SELECT pg_terminate_backend($res_pid); +"); + +# print ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1, "\n"); + +ok($res_reg_lh_1 != 0, "should fail for non-MDB"); +like($stderr_reg_lh_1, qr/Only roles with the SUPERUSER attribute may terminate processes of roles with the SUPERUSER attribute./, "matches"); + +# should succeed +$main_sess->query_safe(qq[COMMIT]); + +$main_sess->query_safe( + q( +SET application_name TO 'MDB'; +BEGIN; +INSERT INTO mdb_app_name_t VALUES(1); +)); + + +my ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_app_name; + SELECT pg_terminate_backend($res_pid); +"); + +ok($res_reg_lh_2 == 0, "should success for MDB"); + +done_testing(); From 6617f48ee22eb46590654fd4cd169dbe2f45e530 Mon Sep 17 00:00:00 2001 From: reshke Date: Sun, 18 Jan 2026 14:12:21 +0000 Subject: [PATCH 47/54] v6 of bt_page_items pretty-print --- contrib/pageinspect/Makefile | 3 +- contrib/pageinspect/btreefuncs.c | 142 ++++++++++++++++-- contrib/pageinspect/expected/btree.out | 25 +++ .../pageinspect/pageinspect--1.12--1.13.sql | 21 +++ contrib/pageinspect/pageinspect.control | 2 +- contrib/pageinspect/sql/btree.sql | 4 + 6 files changed, 185 insertions(+), 12 deletions(-) create mode 100644 contrib/pageinspect/pageinspect--1.12--1.13.sql diff --git a/contrib/pageinspect/Makefile b/contrib/pageinspect/Makefile index 95e030b3969..9dee7653310 100644 --- a/contrib/pageinspect/Makefile +++ b/contrib/pageinspect/Makefile @@ -13,7 +13,8 @@ OBJS = \ rawpage.o EXTENSION = pageinspect -DATA = pageinspect--1.11--1.12.sql pageinspect--1.10--1.11.sql \ +DATA = pageinspect--1.12--1.13.sql \ + pageinspect--1.11--1.12.sql pageinspect--1.10--1.11.sql \ pageinspect--1.9--1.10.sql pageinspect--1.8--1.9.sql \ pageinspect--1.7--1.8.sql pageinspect--1.6--1.7.sql \ pageinspect--1.5.sql pageinspect--1.5--1.6.sql \ diff --git a/contrib/pageinspect/btreefuncs.c b/contrib/pageinspect/btreefuncs.c index 9cdc8e182b4..94dd8641686 100644 --- a/contrib/pageinspect/btreefuncs.c +++ b/contrib/pageinspect/btreefuncs.c @@ -29,6 +29,7 @@ #include "access/nbtree.h" #include "access/relation.h" +#include "access/tupdesc.h" #include "catalog/namespace.h" #include "catalog/pg_am.h" #include "catalog/pg_type.h" @@ -38,6 +39,8 @@ #include "utils/array.h" #include "utils/builtins.h" #include "utils/rel.h" +#include "utils/ruleutils.h" +#include "utils/lsyscache.h" #include "utils/varlena.h" PG_FUNCTION_INFO_V1(bt_metap); @@ -95,6 +98,8 @@ typedef struct ua_page_items bool leafpage; bool rightmost; TupleDesc tupd; + Relation indexRel; + bool pretty_print; } ua_page_items; @@ -548,17 +553,118 @@ bt_page_print_tuples(ua_page_items *uargs) if (dlen < 0 || dlen > INDEX_SIZE_MASK) elog(ERROR, "invalid tuple length %d for tuple at offset number %u", dlen, offset); - dump = palloc0(dlen * 3 + 1); - datacstring = dump; - for (off = 0; off < dlen; off++) + + if (!uargs->pretty_print) { - if (off > 0) - *dump++ = ' '; - sprintf(dump, "%02x", *(ptr + off) & 0xff); - dump += 2; + /* Old-style, print hex bytes */ + dump = palloc0(dlen * 3 + 1); + datacstring = dump; + for (off = 0; off < dlen; off++) + { + if (off > 0) + *dump++ = ' '; + sprintf(dump, "%02x", *(ptr + off) & 0xff); + dump += 2; + } + values[j++] = CStringGetTextDatum(datacstring); + pfree(datacstring); + } + else + { + /* Do pretty-print, akin to record_out() */ + StringInfoData buf; + TupleDesc tupdesc; + + Datum itup_values[INDEX_MAX_KEYS]; + bool itup_isnull[INDEX_MAX_KEYS]; + char *index_columns; + + /* + * Included attributes are added when dealing with leaf pages, discarded + * for non-leaf pages as these include only data for key attributes. + */ + int printflags = RULE_INDEXDEF_PRETTY; + if (P_ISLEAF((BTPageOpaque) PageGetSpecialPointer(page))) + { + tupdesc = RelationGetDescr(uargs->indexRel); + } + else + { + tupdesc = CreateTupleDescCopy(RelationGetDescr(uargs->indexRel)); + tupdesc->natts = IndexRelationGetNumberOfKeyAttributes(uargs->indexRel); + printflags |= RULE_INDEXDEF_KEYS_ONLY; + } + + index_columns = pg_get_indexdef_columns_extended(RelationGetRelid(uargs->indexRel), + printflags); + + + index_deform_tuple(itup, tupdesc, + itup_values, itup_isnull); + + + initStringInfo(&buf); + appendStringInfo(&buf, "(%s)=(", index_columns); + + for (int i = 0; i < tupdesc->natts; i++) + { + char *value; + char *tmp; + bool nq = false; + + if (itup_isnull[i]) + value = "null"; + else + { + Oid foutoid; + bool typisvarlena; + Oid typoid; + + typoid = TupleDescAttr(tupdesc, i)->atttypid; + getTypeOutputInfo(typoid, &foutoid, &typisvarlena); + value = OidOutputFunctionCall(foutoid, itup_values[i]); + } + + if (i == IndexRelationGetNumberOfKeyAttributes(uargs->indexRel)) + appendStringInfoString(&buf, ") INCLUDE ("); + else if (i > 0) + appendStringInfoString(&buf, ", "); + + /* Check whether we need double quotes for this value */ + nq = (value[0] == '\0'); /* force quotes for empty string */ + for (tmp = value; *tmp; tmp++) + { + char ch = *tmp; + + if (ch == '"' || ch == '\\' || + ch == '(' || ch == ')' || ch == ',' || + isspace((unsigned char) ch)) + { + nq = true; + break; + } + } + + /* And emit the string */ + if (nq) + appendStringInfoCharMacro(&buf, '"'); + for (tmp = value; *tmp; tmp++) + { + char ch = *tmp; + + if (ch == '"' || ch == '\\') + appendStringInfoCharMacro(&buf, ch); + appendStringInfoCharMacro(&buf, ch); + } + if (nq) + appendStringInfoCharMacro(&buf, '"'); + } + + appendStringInfoChar(&buf, ')'); + + values[j++] = CStringGetTextDatum(buf.data); + pfree(buf.data); } - values[j++] = CStringGetTextDatum(datacstring); - pfree(datacstring); /* * We need to work around the BTreeTupleIsPivot() !heapkeyspace limitation @@ -630,6 +736,11 @@ bt_page_items_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version) FuncCallContext *fctx; MemoryContext mctx; ua_page_items *uargs; + bool pretty_print = false; + + if (PG_NARGS() >= 3) { + pretty_print = PG_GETARG_BOOL(2); + } if (!superuser()) ereport(ERROR, @@ -667,7 +778,6 @@ bt_page_items_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version) memcpy(uargs->page, BufferGetPage(buffer), BLCKSZ); UnlockReleaseBuffer(buffer); - relation_close(rel, AccessShareLock); uargs->offset = FirstOffsetNumber; @@ -683,6 +793,8 @@ bt_page_items_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version) } uargs->leafpage = P_ISLEAF(opaque); uargs->rightmost = P_RIGHTMOST(opaque); + uargs->pretty_print = pretty_print; + uargs->indexRel = rel; /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupleDesc) != TYPEFUNC_COMPOSITE) @@ -706,6 +818,9 @@ bt_page_items_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version) SRF_RETURN_NEXT(fctx, result); } + if (uargs->indexRel) + relation_close(uargs->indexRel, AccessShareLock); + SRF_RETURN_DONE(fctx); } @@ -800,6 +915,10 @@ bt_page_items_bytea(PG_FUNCTION_ARGS) uargs->leafpage = P_ISLEAF(opaque); uargs->rightmost = P_RIGHTMOST(opaque); + uargs->pretty_print = false; + /* For bytea function, we cannot do pretty-print */ + uargs->indexRel = NULL; + /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupleDesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "return type must be a row type"); @@ -822,6 +941,9 @@ bt_page_items_bytea(PG_FUNCTION_ARGS) SRF_RETURN_NEXT(fctx, result); } + if (uargs->indexRel) + relation_close(uargs->indexRel, AccessShareLock); + SRF_RETURN_DONE(fctx); } diff --git a/contrib/pageinspect/expected/btree.out b/contrib/pageinspect/expected/btree.out index 0aa5d73322f..f407a17121b 100644 --- a/contrib/pageinspect/expected/btree.out +++ b/contrib/pageinspect/expected/btree.out @@ -1,6 +1,7 @@ CREATE TABLE test1 (a int8, b int4range); INSERT INTO test1 VALUES (72057594037927937, '[0,1)'); CREATE INDEX test1_a_idx ON test1 USING btree (a); +CREATE INDEX test1_a_b_idx ON test1 USING btree (a, b); \x SELECT * FROM bt_metap('test1_a_idx'); -[ RECORD 1 ]-------------+------- @@ -147,6 +148,30 @@ btpo_flags | 1 SELECT * FROM bt_multi_page_stats('test2_col1_idx', 7, 2); ERROR: block number 7 is out of range DROP TABLE test2; +SELECT * FROM bt_page_items('test1_a_idx', 1, true); +-[ RECORD 1 ]----------------------- +itemoffset | 1 +ctid | (0,1) +itemlen | 16 +nulls | f +vars | f +data | (a)=(72057594037927937) +dead | f +htid | (0,1) +tids | + +SELECT * FROM bt_page_items('test1_a_b_idx', 1, true); +-[ RECORD 1 ]----------------------------------- +itemoffset | 1 +ctid | (0,1) +itemlen | 32 +nulls | f +vars | t +data | (a, b)=(72057594037927937, "[0,1)") +dead | f +htid | (0,1) +tids | + SELECT * FROM bt_page_items('test1_a_idx', -1); ERROR: invalid block number -1 SELECT * FROM bt_page_items('test1_a_idx', 0); diff --git a/contrib/pageinspect/pageinspect--1.12--1.13.sql b/contrib/pageinspect/pageinspect--1.12--1.13.sql new file mode 100644 index 00000000000..9917d13ba83 --- /dev/null +++ b/contrib/pageinspect/pageinspect--1.12--1.13.sql @@ -0,0 +1,21 @@ +/* contrib/pageinspect/pageinspect--1.12--1.13.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION pageinspect UPDATE TO '1.13'" to load this file. \quit + + +DROP FUNCTION bt_page_items(text, int8); +CREATE FUNCTION bt_page_items(IN relname text, IN blkno int8, + IN pretty_print boolean DEFAULT FALSE, + OUT itemoffset smallint, + OUT ctid tid, + OUT itemlen smallint, + OUT nulls bool, + OUT vars bool, + OUT data text, + OUT dead boolean, + OUT htid tid, + OUT tids tid[]) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'bt_page_items_1_9' +LANGUAGE C STRICT PARALLEL SAFE; diff --git a/contrib/pageinspect/pageinspect.control b/contrib/pageinspect/pageinspect.control index b2804e9b128..cfc87feac03 100644 --- a/contrib/pageinspect/pageinspect.control +++ b/contrib/pageinspect/pageinspect.control @@ -1,5 +1,5 @@ # pageinspect extension comment = 'inspect the contents of database pages at a low level' -default_version = '1.12' +default_version = '1.13' module_pathname = '$libdir/pageinspect' relocatable = true diff --git a/contrib/pageinspect/sql/btree.sql b/contrib/pageinspect/sql/btree.sql index 102ebdefe3c..36f100656a4 100644 --- a/contrib/pageinspect/sql/btree.sql +++ b/contrib/pageinspect/sql/btree.sql @@ -1,6 +1,7 @@ CREATE TABLE test1 (a int8, b int4range); INSERT INTO test1 VALUES (72057594037927937, '[0,1)'); CREATE INDEX test1_a_idx ON test1 USING btree (a); +CREATE INDEX test1_a_b_idx ON test1 USING btree (a, b); \x @@ -22,6 +23,9 @@ SELECT * FROM bt_multi_page_stats('test2_col1_idx', 3, 2); SELECT * FROM bt_multi_page_stats('test2_col1_idx', 7, 2); DROP TABLE test2; +SELECT * FROM bt_page_items('test1_a_idx', 1, true); +SELECT * FROM bt_page_items('test1_a_b_idx', 1, true); + SELECT * FROM bt_page_items('test1_a_idx', -1); SELECT * FROM bt_page_items('test1_a_idx', 0); SELECT * FROM bt_page_items('test1_a_idx', 1); From 658a853adc303eb64765793d4ffcc7327a33ff1b Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Tue, 10 Feb 2026 12:47:32 +0500 Subject: [PATCH 48/54] Add archive_mode=shared for coordinated WAL archiving Introduce a new archive_mode setting "shared" to prevent WAL history loss during standby promotion in HA streaming replication setups. In shared mode, the primary proactively sends archival status updates to standbys via the replication protocol. The standby creates .ready files for received WAL segments but defers marking them as .done until the primary confirms archival. This prevents WAL from being recycled before it's safely archived, addressing a critical gap in PITR continuity during failover. Key implementation details: - Primary periodically sends last archived WAL segment via new PqReplMsg_ArchiveStatusReport ('a') message - Standby marks all segments <= reported segment as .done using alphanumeric comparison on segment part (timeline-safe) - Archiver skips during recovery in shared mode, activates on promotion - Cascading replication: each standby coordinates with immediate upstream - Startup check rejects archive_mode=on during recovery This "push" design (primary sends status) is more efficient than "pull" (standby queries per-segment), avoiding directory scans and stat() calls. Based on Heikki Linnakangas's 2014 design and Greenplum's production implementation, modernized for PostgreSQL 19. Includes TAP tests covering basic synchronization, promotion, cascading replication, and multiple standbys scenarios. --- doc/src/sgml/config.sgml | 36 ++- doc/src/sgml/high-availability.sgml | 72 ++++-- src/backend/access/transam/xlog.c | 1 + src/backend/postmaster/pgarch.c | 17 +- src/backend/replication/walreceiver.c | 147 +++++++++++- src/backend/replication/walsender.c | 94 ++++++++ src/include/access/xlog.h | 3 +- src/include/libpq/protocol.h | 111 +++++++++ src/test/recovery/t/050_archive_shared.pl | 270 ++++++++++++++++++++++ 9 files changed, 712 insertions(+), 39 deletions(-) create mode 100644 src/include/libpq/protocol.h create mode 100644 src/test/recovery/t/050_archive_shared.pl diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index f675523e8c8..9f735dc95f5 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3687,14 +3687,36 @@ include_dir 'conf.d' are sent to archive storage by setting or . In addition to off, - to disable, there are two modes: on, and - always. During normal operation, there is no - difference between the two modes, but when set to always - the WAL archiver is enabled also during archive recovery or standby - mode. In always mode, all files restored from the archive - or streamed with streaming replication will be archived (again). See - for details. + to disable, there are three modes: on, shared, + and always. During normal operation as a primary, there is no + difference between the three modes, but they differ during archive recovery or + standby mode: + + + + on: Archives WAL only when running as a primary. + + + + + shared: Coordinates archiving between primary and standby. + The standby defers WAL archival and deletion until the primary confirms + archival via streaming replication. This prevents WAL history loss during + standby promotion in high availability setups. Upon promotion, the standby + automatically starts archiving any remaining unarchived WAL. This mode works + with cascading replication, where each standby coordinates with its immediate + upstream server. See for details. + + + + + always: Archives all WAL independently, even during recovery. + All files restored from the archive or streamed with streaming physical + replication will be archived (again), regardless of their source. + + + archive_mode is a separate setting from archive_command and diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index 93e762e4bb1..02aead19d41 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -1377,35 +1377,61 @@ synchronous_standby_names = 'ANY 2 (s1, s2, s3)' - When continuous WAL archiving is used in a standby, there are two - different scenarios: the WAL archive can be shared between the primary - and the standby, or the standby can have its own WAL archive. When - the standby has its own WAL archive, set archive_mode + When continuous WAL archiving is used in a standby, there are three + different scenarios: the standby can have its own independent WAL archive, + the WAL archive can be shared between the primary and standby, or archiving + can be coordinated between them. + + + + For an independent archive, set archive_mode to always, and the standby will call the archive command for every WAL segment it receives, whether it's by restoring - from the archive or by streaming replication. The shared archive can - be handled similarly, but the archive_command or archive_library must - test if the file being archived exists already, and if the existing file - has identical contents. This requires more care in the - archive_command or archive_library, as it must - be careful to not overwrite an existing file with different contents, - but return success if the exactly same file is archived twice. And - all that must be done free of race conditions, if two servers attempt - to archive the same file at the same time. + from the archive or by streaming replication. + + + + For a shared archive where both primary and standby can write, use + always mode as well, but the archive_command + or archive_library must test if the file being archived + exists already, and if the existing file has identical contents. This requires + more care in the archive_command or archive_library, + as it must be careful to not overwrite an existing file with different contents, + but return success if the exactly same file is archived twice. And all that must + be done free of race conditions, if two servers attempt to archive the same file + at the same time. + + + + For coordinated archiving in high availability setups, use + archive_mode=shared. In this mode, only + the primary archives WAL segments. The standby creates .ready + files for received segments but defers actual archiving. The primary periodically + sends archival status updates to the standby via streaming replication, informing + it which segments have been archived. The standby then marks these as archived + and allows them to be recycled. Upon promotion, the standby automatically starts + archiving any remaining WAL segments that weren't confirmed as archived by the + former primary. This prevents WAL history loss during failover while avoiding + the complexity of coordinating concurrent archiving. This mode works with cascading + replication, where each standby coordinates with its immediate upstream server. If archive_mode is set to on, the - archiver is not enabled during recovery or standby mode. If the standby - server is promoted, it will start archiving after the promotion, but - will not archive any WAL or timeline history files that - it did not generate itself. To get a complete - series of WAL files in the archive, you must ensure that all WAL is - archived, before it reaches the standby. This is inherently true with - file-based log shipping, as the standby can only restore files that - are found in the archive, but not if streaming replication is enabled. - When a server is not in recovery mode, there is no difference between - on and always modes. + archiver is not enabled during recovery or standby mode, and this setting + cannot be used on a standby. If a standby with archive_mode + set to on is promoted, it will start archiving after the + promotion, but will not archive any WAL or timeline history files that it did + not generate itself. To get a complete series of WAL files in the archive, you + must ensure that all WAL is archived before it reaches the standby. This is + inherently true with file-based log shipping, as the standby can only restore + files that are found in the archive, but not if streaming replication is enabled. + + + + When a server is not in recovery mode, on, + shared, and always modes all behave + identically, archiving completed WAL segments. diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 03d26e24d2c..bd6bac683fd 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -196,6 +196,7 @@ const struct config_enum_entry archive_mode_options[] = { {"always", ARCHIVE_MODE_ALWAYS, false}, {"on", ARCHIVE_MODE_ON, false}, {"off", ARCHIVE_MODE_OFF, false}, + {"shared", ARCHIVE_MODE_SHARED, false}, {"true", ARCHIVE_MODE_ON, true}, {"false", ARCHIVE_MODE_OFF, true}, {"yes", ARCHIVE_MODE_ON, true}, diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index 46af3495644..1255b8dd348 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -372,6 +372,15 @@ pgarch_ArchiverCopyLoop(void) { char xlog[MAX_XFN_CHARS + 1]; + /* + * In shared archive mode during recovery, the archiver doesn't archive + * files. The primary is responsible for archiving, and the walreceiver + * marks files as .done when the primary confirms archival. After + * promotion, the archiver starts working normally. + */ + if (XLogArchiveMode == ARCHIVE_MODE_SHARED && RecoveryInProgress()) + return; + /* force directory scan in the first call to pgarch_readyXlog() */ arch_files->arch_files_size = 0; @@ -457,10 +466,10 @@ pgarch_ArchiverCopyLoop(void) continue; } - if (pgarch_archiveXlog(xlog)) - { - /* successful */ - pgarch_archiveDone(xlog); + if (pgarch_archiveXlog(xlog)) + { + /* successful */ + pgarch_archiveDone(xlog); /* * Tell the cumulative stats system about the WAL file that we diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index feff7094351..03b838a43ac 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -62,6 +62,7 @@ #include "common/ip.h" #include "funcapi.h" #include "libpq/pqformat.h" +#include "libpq/protocol.h" #include "libpq/pqsignal.h" #include "miscadmin.h" #include "pgstat.h" @@ -134,6 +135,11 @@ static TimestampTz wakeup[NUM_WALRCV_WAKEUPS]; static StringInfoData reply_message; static StringInfoData incoming_message; +/* Last archived WAL segment file reported by the primary */ +static char primary_last_archived[MAX_XFN_CHARS + 1]; +static TimeLineID primary_last_archived_tli = 0; +static XLogSegNo primary_last_archived_segno = 0; + /* Prototypes for private functions */ static void WalRcvFetchTimeLineHistoryFiles(TimeLineID first, TimeLineID last); static void WalRcvWaitForStartPosition(XLogRecPtr *startpoint, TimeLineID *startpointTLI); @@ -147,6 +153,7 @@ static void XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli); static void XLogWalRcvSendReply(bool force, bool requestReply); static void XLogWalRcvSendHSFeedback(bool immed); static void ProcessWalSndrMessage(XLogRecPtr walEnd, TimestampTz sendTime); +static void ProcessArchivalReport(void); static void WalRcvComputeNextWakeup(WalRcvWakeupReason reason, TimestampTz now); /* @@ -890,6 +897,30 @@ XLogWalRcvProcessMsg(unsigned char type, char *buf, Size len, TimeLineID tli) XLogWalRcvSendReply(true, false); break; } + case PqReplMsg_ArchiveStatusReport: + { + /* Check that the filename looks valid */ + if (len >= sizeof(primary_last_archived)) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg_internal("invalid archival report message with length %d", + (int) len))); + + memcpy(primary_last_archived, buf, len); + primary_last_archived[len] = '\0'; + + /* Verify it contains only valid characters */ + if (strspn(buf, VALID_XFN_CHARS) != len) + { + primary_last_archived[0] = '\0'; + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg_internal("unexpected character in primary's last archived filename"))); + } + + ProcessArchivalReport(); + break; + } default: ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), @@ -1068,12 +1099,39 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli) /* * Create .done file forcibly to prevent the streamed segment from being - * archived later. + * archived later, unless archive_mode is 'always' or 'shared'. + * + * In 'always' mode, the standby archives independently. + * + * In 'shared' mode, we optimize by checking if this segment is already + * covered by the last archival report from the primary. If so, create + * .done directly. Otherwise, create .ready and wait for the next report. */ - if (XLogArchiveMode != ARCHIVE_MODE_ALWAYS) - XLogArchiveForceDone(xlogfname); - else + if (XLogArchiveMode == ARCHIVE_MODE_ALWAYS) + { XLogArchiveNotify(xlogfname); + } + else if (XLogArchiveMode == ARCHIVE_MODE_SHARED) + { + /* + * In shared mode, check if this segment is already archived on primary. + * If we're on the same timeline and this segment is <= last archived, + * mark it .done immediately. Otherwise create .ready. + */ + if (primary_last_archived_tli == recvFileTLI && + recvSegNo <= primary_last_archived_segno) + { + XLogArchiveForceDone(xlogfname); + } + else + { + XLogArchiveNotify(xlogfname); + } + } + else + { + XLogArchiveForceDone(xlogfname); + } recvFile = -1; } @@ -1250,6 +1308,87 @@ XLogWalRcvSendHSFeedback(bool immed) primary_has_standby_xmin = false; } +/* + * Process archival report from primary. + * + * The primary sends us the last WAL segment it has archived. We scan the + * archive_status directory for .ready files and mark segments on the same + * timeline as .done if they're <= the reported segment. + */ +static void +ProcessArchivalReport(void) +{ + TimeLineID reported_tli; + XLogSegNo reported_segno; + DIR *status_dir; + struct dirent *status_de; + char status_path[MAXPGPATH]; + + elog(DEBUG2, "received archival report from primary: %s", + primary_last_archived); + + /* Parse the reported WAL filename */ + if (!IsXLogFileName(primary_last_archived)) + { + elog(DEBUG2, "invalid WAL filename in archival report: %s", + primary_last_archived); + return; + } + + XLogFromFileName(primary_last_archived, &reported_tli, &reported_segno, + wal_segment_size); + + /* Remember the last archived segment for XLogWalRcvClose() */ + primary_last_archived_tli = reported_tli; + primary_last_archived_segno = reported_segno; + + /* Scan archive_status directory for .ready files */ + snprintf(status_path, MAXPGPATH, XLOGDIR "/archive_status"); + status_dir = AllocateDir(status_path); + if (status_dir == NULL) + { + elog(DEBUG2, "could not open archive_status directory: %m"); + return; + } + + while ((status_de = ReadDir(status_dir, status_path)) != NULL) + { + char *ready_suffix; + char walfile[MAXPGPATH]; + TimeLineID file_tli; + XLogSegNo file_segno; + /* Look for .ready files only */ + ready_suffix = strstr(status_de->d_name, ".ready"); + if (ready_suffix == NULL || ready_suffix[6] != '\0') + continue; + + /* Extract WAL filename (remove .ready suffix) */ + strlcpy(walfile, status_de->d_name, ready_suffix - status_de->d_name + 1); + + /* Parse the WAL filename */ + if (!IsXLogFileName(walfile)) + continue; + + XLogFromFileName(walfile, &file_tli, &file_segno, wal_segment_size); + + /* + * Mark as .done if it's on the same timeline and not after the + * reported segment. We only process the reported timeline to avoid + * marking segments from parent or future timelines prematurely. + * XXX: Process possible TLI switches happened between status reports. + * For now, leave segments on previous TLIs to archive_command. + */ + if (file_tli == reported_tli && file_segno <= reported_segno) + { + XLogArchiveForceDone(walfile); + elog(DEBUG3, "marked WAL segment %s as archived (primary archived up to %s)", + walfile, primary_last_archived); + } + } + + FreeDir(status_dir); +} + /* * Update shared memory status upon receiving a message from primary. * diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 36ea981addc..7b4aa17b8bc 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -64,6 +64,7 @@ #include "commands/defrem.h" #include "funcapi.h" #include "libpq/libpq.h" +#include "libpq/protocol.h" #include "libpq/pqformat.h" #include "miscadmin.h" #include "nodes/replnodes.h" @@ -181,6 +182,17 @@ static TimestampTz last_reply_timestamp = 0; /* Have we sent a heartbeat message asking for reply, since last reply? */ static bool waiting_for_ping_response = false; +/* + * Last archived WAL file. This is fetched from pgstat periodically and sent + * to the standby. last_archival_report_timestamp tracks when we last sent + * the report to avoid excessive pgstat access. + */ +static char last_archived_wal[MAX_XFN_CHARS + 1]; +static TimestampTz last_archival_report_timestamp = 0; + +/* Interval for sending archival reports (10 seconds) */ +#define ARCHIVAL_REPORT_INTERVAL 10000 + /* * While streaming WAL in Copy mode, streamingDoneSending is set to true * after we have sent CopyDone. We should not send any more CopyData messages @@ -279,6 +291,7 @@ static void StartLogicalReplication(StartReplicationCmd *cmd); static void ProcessStandbyMessage(void); static void ProcessStandbyReplyMessage(void); static void ProcessStandbyHSFeedbackMessage(void); +static void WalSndArchivalReport(void); static void ProcessRepliesIfAny(void); static void ProcessPendingWrites(void); static void WalSndKeepalive(bool requestReply, XLogRecPtr writePtr); @@ -2434,6 +2447,84 @@ ProcessStandbyHSFeedbackMessage(void) } } +/* + * Send archival status report to standby. + * + * This is called periodically during physical replication to inform the + * standby about the last WAL segment archived by the primary. The standby + * can then mark segments up to that point as .done, allowing them to be + * recycled. This prevents WAL loss during standby promotion. + */ +static void +WalSndArchivalReport(void) +{ + PgStat_ArchiverStats *archiver_stats; + TimestampTz now; + char *last_archived; + + /* Only send reports when archive_mode=shared */ + if (XLogArchiveMode != ARCHIVE_MODE_SHARED) + return; + + /* Only send reports during physical streaming replication, not during backup */ + if (MyWalSnd->kind != REPLICATION_KIND_PHYSICAL) + return; + if (MyWalSnd->state != WALSNDSTATE_CATCHUP && + MyWalSnd->state != WALSNDSTATE_STREAMING) + return; + + /* + * Don't send to temporary replication slots (used by pg_basebackup). + * Connections without slots (regular standbys) are OK. + */ + if (MyReplicationSlot != NULL && + MyReplicationSlot->data.persistency == RS_TEMPORARY) + return; + + now = GetCurrentTimestamp(); + + /* + * Send report at most once per ARCHIVAL_REPORT_INTERVAL (10 seconds). + * This avoids excessive pgstat access. + */ + if (now < TimestampTzPlusMilliseconds(last_archival_report_timestamp, + ARCHIVAL_REPORT_INTERVAL)) + return; + last_archival_report_timestamp = now; + /* + * Get archiver statistics. We use non-blocking access to avoid delaying + * replication if stats collector is slow. If stats are unavailable or + * stale, we'll just try again at the next interval. + */ + archiver_stats = pgstat_fetch_stat_archiver(); + if (archiver_stats == NULL) + return; + + last_archived = archiver_stats->last_archived_wal; + /* + * Only send a report if the last archived WAL has changed. This is both + * an optimization and ensures we don't send empty reports on startup. + */ + if (strcmp(last_archived, last_archived_wal) == 0) + return; + + /* Only send reports for WAL segments, not backup history files or other archived files */ + if (!IsXLogFileName(last_archived)) + return; + + elog(DEBUG2, "sending archival report: %s", last_archived); + + /* Remember what we sent */ + strlcpy(last_archived_wal, last_archived, sizeof(last_archived_wal)); + + /* Construct the message... */ + resetStringInfo(&output_message); + pq_sendbyte(&output_message, PqReplMsg_ArchiveStatusReport); + pq_sendbytes(&output_message, last_archived, strlen(last_archived)); + /* ... and send it wrapped in CopyData */ + pq_putmessage_noblock(PqMsg_CopyData, output_message.data, output_message.len); +} + /* * Compute how long send/receive loops should sleep. * @@ -3837,6 +3928,9 @@ WalSndKeepaliveIfNecessary(void) if (pq_flush_if_writable() != 0) WalSndShutdown(); } + + /* Send archival status report if needed */ + WalSndArchivalReport(); } /* diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 6fd5b3dfa66..1b39c235902 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -59,7 +59,8 @@ typedef enum ArchiveMode { ARCHIVE_MODE_OFF = 0, /* disabled */ ARCHIVE_MODE_ON, /* enabled while server is running normally */ - ARCHIVE_MODE_ALWAYS /* enabled always (even during recovery) */ + ARCHIVE_MODE_ALWAYS, /* enabled always (even during recovery) */ + ARCHIVE_MODE_SHARED, /* shared archive between primary and standby */ } ArchiveMode; extern PGDLLIMPORT int XLogArchiveMode; diff --git a/src/include/libpq/protocol.h b/src/include/libpq/protocol.h new file mode 100644 index 00000000000..925c7568ea1 --- /dev/null +++ b/src/include/libpq/protocol.h @@ -0,0 +1,111 @@ +/*------------------------------------------------------------------------- + * + * protocol.h + * Definitions of the request/response codes for the wire protocol. + * + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/libpq/protocol.h + * + *------------------------------------------------------------------------- + */ +#ifndef PROTOCOL_H +#define PROTOCOL_H + +/* These are the request codes sent by the frontend. */ + +#define PqMsg_Bind 'B' +#define PqMsg_Close 'C' +#define PqMsg_Describe 'D' +#define PqMsg_Execute 'E' +#define PqMsg_FunctionCall 'F' +#define PqMsg_Flush 'H' +#define PqMsg_Parse 'P' +#define PqMsg_Query 'Q' +#define PqMsg_Sync 'S' +#define PqMsg_Terminate 'X' +#define PqMsg_CopyFail 'f' +#define PqMsg_GSSResponse 'p' +#define PqMsg_PasswordMessage 'p' +#define PqMsg_SASLInitialResponse 'p' +#define PqMsg_SASLResponse 'p' + + +/* These are the response codes sent by the backend. */ + +#define PqMsg_ParseComplete '1' +#define PqMsg_BindComplete '2' +#define PqMsg_CloseComplete '3' +#define PqMsg_NotificationResponse 'A' +#define PqMsg_CommandComplete 'C' +#define PqMsg_DataRow 'D' +#define PqMsg_ErrorResponse 'E' +#define PqMsg_CopyInResponse 'G' +#define PqMsg_CopyOutResponse 'H' +#define PqMsg_EmptyQueryResponse 'I' +#define PqMsg_BackendKeyData 'K' +#define PqMsg_NoticeResponse 'N' +#define PqMsg_AuthenticationRequest 'R' +#define PqMsg_ParameterStatus 'S' +#define PqMsg_RowDescription 'T' +#define PqMsg_FunctionCallResponse 'V' +#define PqMsg_CopyBothResponse 'W' +#define PqMsg_ReadyForQuery 'Z' +#define PqMsg_NoData 'n' +#define PqMsg_PortalSuspended 's' +#define PqMsg_ParameterDescription 't' +#define PqMsg_NegotiateProtocolVersion 'v' + + +/* These are the codes sent by both the frontend and backend. */ + +#define PqMsg_CopyDone 'c' +#define PqMsg_CopyData 'd' + + +/* These are the codes sent by parallel workers to leader processes. */ +#define PqMsg_Progress 'P' + + +/* Replication codes sent by the primary (wrapped in CopyData messages). */ + +#define PqReplMsg_ArchiveStatusReport 'a' +#define PqReplMsg_Keepalive 'k' +#define PqReplMsg_PrimaryStatusUpdate 's' +#define PqReplMsg_WALData 'w' + + +/* Replication codes sent by the standby (wrapped in CopyData messages). */ + +#define PqReplMsg_HotStandbyFeedback 'h' +#define PqReplMsg_PrimaryStatusRequest 'p' +#define PqReplMsg_StandbyStatusUpdate 'r' + + +/* Codes used for backups via COPY OUT (wrapped in CopyData messages). */ + +#define PqBackupMsg_Manifest 'm' +#define PqBackupMsg_NewArchive 'n' +#define PqBackupMsg_ProgressReport 'p' + + +/* These are the authentication request codes sent by the backend. */ + +#define AUTH_REQ_OK 0 /* User is authenticated */ +#define AUTH_REQ_KRB4 1 /* Kerberos V4. Not supported any more. */ +#define AUTH_REQ_KRB5 2 /* Kerberos V5. Not supported any more. */ +#define AUTH_REQ_PASSWORD 3 /* Password */ +#define AUTH_REQ_CRYPT 4 /* crypt password. Not supported any more. */ +#define AUTH_REQ_MD5 5 /* md5 password */ +/* 6 is available. It was used for SCM creds, not supported any more. */ +#define AUTH_REQ_GSS 7 /* GSSAPI without wrap() */ +#define AUTH_REQ_GSS_CONT 8 /* Continue GSS exchanges */ +#define AUTH_REQ_SSPI 9 /* SSPI negotiate without wrap() */ +#define AUTH_REQ_SASL 10 /* Begin SASL authentication */ +#define AUTH_REQ_SASL_CONT 11 /* Continue SASL authentication */ +#define AUTH_REQ_SASL_FIN 12 /* Final SASL message */ +#define AUTH_REQ_MAX AUTH_REQ_SASL_FIN /* maximum AUTH_REQ_* value */ + +#endif /* PROTOCOL_H */ diff --git a/src/test/recovery/t/050_archive_shared.pl b/src/test/recovery/t/050_archive_shared.pl new file mode 100644 index 00000000000..397b71ad79d --- /dev/null +++ b/src/test/recovery/t/050_archive_shared.pl @@ -0,0 +1,270 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Test archive_mode=shared for coordinated WAL archiving between primary and standby +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use File::Path qw(rmtree); + +# Initialize primary node with archiving +my $archive_dir = PostgreSQL::Test::Utils::tempdir(); +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(has_archiving => 1, allows_streaming => 1); +$primary->append_conf('postgresql.conf', " +archive_mode = shared +archive_command = 'cp %p \"$archive_dir\"/%f' +wal_keep_size = 128MB +"); +$primary->start; + +# Create a test table and generate some WAL +$primary->safe_psql('postgres', 'CREATE TABLE test_table (id int, data text);'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(1, 500) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(501, 1000) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for archiver to archive segments +$primary->poll_query_until('postgres', + "SELECT archived_count > 0 FROM pg_stat_archiver") + or die "Timed out waiting for archiver to start"; + +my $archived_count = () = glob("$archive_dir/*"); +ok($archived_count > 0, "primary has archived WAL files to shared archive"); +note("Primary archived $archived_count files"); + +# Take backup for standby +my $backup_name = 'standby_backup'; +$primary->backup($backup_name); + +# Exclude possible race condition when backup WAL is last archived +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(501, 1000) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Set up standby with archive_mode=shared +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, $backup_name, has_streaming => 1); +$standby->append_conf('postgresql.conf', " +archive_mode = shared +archive_command = 'cp %p \"$archive_dir\"/%f' +wal_receiver_status_interval = 1s +"); +$standby->start; + +# Wait for standby to catch up +$primary->wait_for_catchup($standby); + +# Generate more WAL on primary (these are new segments not yet archived) +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(1001, 1500) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(1501, 2000) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for standby to receive the new WAL +$primary->wait_for_catchup($standby); + +# Check that standby has .ready or .done files for the newly received segments. +# Normally they should be .ready (not yet archived by primary), but in rare cases +# the archiver could be very fast and an archive report sent immediately, creating +# .done files instead. Both are correct behavior - the key is that files exist. +my $standby_archive_status = $standby->data_dir . '/pg_wal/archive_status'; +my $status_count = 0; +if (opendir(my $dh, $standby_archive_status)) +{ + my @files = grep { /\.(ready|done)$/ } readdir($dh); + $status_count = scalar(@files); + my $ready_count = scalar(grep { /\.ready$/ } @files); + my $done_count = scalar(grep { /\.done$/ } @files); + note("Standby has $ready_count .ready files and $done_count .done files"); + closedir($dh); +} +cmp_ok($status_count, '>', 0, "standby creates archive status files for received WAL"); + +# Generate more WAL and wait for archiving on primary +my $initial_archived = $primary->safe_psql('postgres', 'SELECT archived_count FROM pg_stat_archiver'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'more-data' || i FROM generate_series(2001, 2500) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'more-data2' || i FROM generate_series(2501, 3000) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for primary to archive the new segments +$primary->poll_query_until('postgres', + "SELECT archived_count > $initial_archived FROM pg_stat_archiver") + or die "Timed out waiting for primary to archive new segments"; + +# Wait for standby to catch up (archive status is sent during replication) +$primary->wait_for_catchup($standby); + +# Wait for primary to send archival status updates and standby to process them +# The standby should mark segments as .done after receiving archive status from primary +my $done_count = 0; +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $done_count = 0; + if (opendir(my $dh, $standby_archive_status)) + { + $done_count = scalar(grep { /\.done$/ } readdir($dh)); + closedir($dh); + } + last if $done_count > 0; + sleep(1); +} +ok($done_count > 0, "standby marked segments as .done after primary's archival report"); +note("Standby has $done_count .done files"); + +############################################################################### +# Test 2: Standby promotion - verify archiver activates +############################################################################### + +# Before promotion, verify archiver is not running on standby (shared mode during recovery) +# In shared mode, the standby's archiver should not be archiving during recovery +my $archived_before = $standby->safe_psql('postgres', + "SELECT archived_count FROM pg_stat_archiver"); +is($archived_before, '0', + "archiver not active on standby before promotion (archived_count=0)"); + +# Verify standby is still in recovery before promoting +my $in_recovery = $standby->safe_psql('postgres', "SELECT pg_is_in_recovery();"); +is($in_recovery, 't', "standby is in recovery before promotion"); + +# Promote the standby +$standby->promote; +$standby->poll_query_until('postgres', "SELECT NOT pg_is_in_recovery();"); + +# Generate WAL on new primary (former standby) +$standby->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'post-promotion' || i FROM generate_series(2001, 2500) i;"); +$standby->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for archiver to activate and archive the new WAL +# Check pg_stat_archiver to verify archiving is happening +$standby->poll_query_until('postgres', + "SELECT archived_count > 0 FROM pg_stat_archiver") + or die "Timed out waiting for promoted standby to start archiving"; +pass("promoted standby started archiving"); + +# Verify data integrity +my $count = $standby->safe_psql('postgres', 'SELECT COUNT(*) FROM test_table;'); +ok($count >= 2500, "promoted standby has all data (got $count rows)"); + +############################################################################### +# Test 3: Cascading replication +############################################################################### + +# Take a backup from the promoted standby (now the new primary) +my $promoted_backup = 'promoted_backup'; +$standby->backup($promoted_backup); + +# Set up second-level standby (cascading from first standby, now promoted) +my $standby2 = PostgreSQL::Test::Cluster->new('standby2'); +$standby2->init_from_backup($standby, $promoted_backup, has_streaming => 1); +$standby2->append_conf('postgresql.conf', " +archive_mode = shared +archive_command = 'cp %p \"$archive_dir\"/%f' +wal_receiver_status_interval = 1s +"); +$standby2->start; + +# Generate WAL on promoted standby (now primary for standby2) +my $cascading_archived_before = $standby->safe_psql('postgres', 'SELECT archived_count FROM pg_stat_archiver'); +$standby->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'cascading' || i FROM generate_series(2501, 3000) i;"); +$standby->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for the promoted standby (acting as primary) to archive the new segment +$standby->poll_query_until('postgres', + "SELECT archived_count > $cascading_archived_before FROM pg_stat_archiver") + or die "Timed out waiting for primary to archive segment in cascading test"; + +# Wait for cascading standby to catch up +$standby->wait_for_catchup($standby2); + +# Wait for cascading standby to receive archive status and mark segments as .done +my $standby2_archive_status = $standby2->data_dir . '/pg_wal/archive_status'; +my $standby2_done_count = 0; +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $standby2_done_count = 0; + if (opendir(my $dh, $standby2_archive_status)) + { + $standby2_done_count = scalar(grep { /\.done$/ } readdir($dh)); + closedir($dh); + } + last if $standby2_done_count > 0; + sleep(1); +} +ok($standby2_done_count > 0, "cascading standby marks segments as .done"); +note("Cascading standby has $standby2_done_count .done files"); + +# Verify cascading standby has all data +my $standby2_count = $standby2->safe_psql('postgres', 'SELECT COUNT(*) FROM test_table;'); +ok($standby2_count >= 3000, "cascading standby has all data (got $standby2_count rows)"); + +############################################################################### +# Test 4: Multiple standbys from same primary +############################################################################### + +# Create third standby from promoted standby (current primary) +my $standby3 = PostgreSQL::Test::Cluster->new('standby3'); +my $backup2 = 'multi_standby_backup'; +$standby->backup($backup2); +$standby3->init_from_backup($standby, $backup2, has_streaming => 1); +$standby3->append_conf('postgresql.conf', " +archive_mode = shared +archive_command = 'cp %p \"$archive_dir\"/%f' +wal_receiver_status_interval = 1s +"); +$standby3->start; + +# Generate WAL and ensure both standbys receive it +my $standby_archived_before = $standby->safe_psql('postgres', 'SELECT archived_count FROM pg_stat_archiver'); +$standby->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'multi' || i FROM generate_series(3001, 3500) i;"); +$standby->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for the promoted standby (acting as primary) to archive the new segment +$standby->poll_query_until('postgres', + "SELECT archived_count > $standby_archived_before FROM pg_stat_archiver") + or die "Timed out waiting for primary to archive segment in multi-standby test"; + +$standby->wait_for_catchup($standby2); +$standby->wait_for_catchup($standby3); + +# Verify both standbys eventually mark segments as .done +my $standby3_archive_status = $standby3->data_dir . '/pg_wal/archive_status'; + +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $standby2_done_count = 0; + if (opendir(my $dh, $standby2_archive_status)) + { + $standby2_done_count = scalar(grep { /\.done$/ } readdir($dh)); + closedir($dh); + } + last if $standby2_done_count > 0; + sleep(1); +} + +my $standby3_done_count = 0; +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $standby3_done_count = 0; + if (opendir(my $dh, $standby3_archive_status)) + { + $standby3_done_count = scalar(grep { /\.done$/ } readdir($dh)); + closedir($dh); + } + last if $standby3_done_count > 0; + sleep(1); +} + +ok($standby2_done_count > 0, "standby2 marks segments as .done"); +ok($standby3_done_count > 0, "standby3 marks segments as .done"); +note("standby2 has $standby2_done_count .done files, standby3 has $standby3_done_count .done files"); + +# Verify both standbys have all data +$standby2_count = $standby2->safe_psql('postgres', 'SELECT COUNT(*) FROM test_table;'); +my $standby3_count = $standby3->safe_psql('postgres', 'SELECT COUNT(*) FROM test_table;'); +ok($standby2_count >= 3500, "standby2 has all data (got $standby2_count rows)"); +ok($standby3_count >= 3500, "standby3 has all data (got $standby3_count rows)"); + +done_testing(); From c1f1e5569c0a8df0ba508c4d17480478d71c5ab5 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Tue, 10 Feb 2026 16:45:10 +0500 Subject: [PATCH 49/54] Mark ancestor timeline WAL segments as archived When standby receives archive status report, check if .ready files belong to ancestor timelines before the switch point and mark them as .done if already archived by primary. --- src/backend/replication/walreceiver.c | 55 ++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 03b838a43ac..75b29206f5b 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -1117,6 +1117,11 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli) * In shared mode, check if this segment is already archived on primary. * If we're on the same timeline and this segment is <= last archived, * mark it .done immediately. Otherwise create .ready. + * + * We don't check ancestor timeline cases here to avoid reading timeline + * history files on every segment close. ProcessArchivalReport() will + * handle marking ancestor timeline segments as .done when it scans + * the archive_status directory. */ if (primary_last_archived_tli == recvFileTLI && recvSegNo <= primary_last_archived_segno) @@ -1323,6 +1328,7 @@ ProcessArchivalReport(void) DIR *status_dir; struct dirent *status_de; char status_path[MAXPGPATH]; + List *tli_history = NIL; elog(DEBUG2, "received archival report from primary: %s", primary_last_archived); @@ -1372,18 +1378,57 @@ ProcessArchivalReport(void) XLogFromFileName(walfile, &file_tli, &file_segno, wal_segment_size); /* - * Mark as .done if it's on the same timeline and not after the - * reported segment. We only process the reported timeline to avoid - * marking segments from parent or future timelines prematurely. - * XXX: Process possible TLI switches happened between status reports. - * For now, leave segments on previous TLIs to archive_command. + * Mark as .done if: + * 1. Same timeline and segment <= reported segment, OR + * 2. Ancestor timeline and segment is before the timeline switch point + * + * For ancestor timelines: if primary archived segment X on timeline T, + * then all segments on ancestor timelines before the switch to T must + * have been archived (they're required to reach timeline T). */ if (file_tli == reported_tli && file_segno <= reported_segno) { + /* Same timeline, segment already archived */ XLogArchiveForceDone(walfile); elog(DEBUG3, "marked WAL segment %s as archived (primary archived up to %s)", walfile, primary_last_archived); } + else if (file_tli != reported_tli) + { + /* + * Different timeline - check if it's an ancestor and if this + * segment is before the timeline switch point. Only read timeline + * history if we haven't already (lazy loading). + * + * Note: Timelines form a tree structure, not a linear sequence, + * so we can't use < or > to compare them. + */ + if (tli_history == NIL) + tli_history = readTimeLineHistory(reported_tli); + + if (tliInHistory(file_tli, tli_history)) + { + XLogRecPtr switchpoint; + XLogSegNo switchpoint_segno; + + /* Get the point where we switched away from this timeline */ + switchpoint = tliSwitchPoint(file_tli, tli_history, NULL); + + /* + * If the segment is at or before the switch point, it must have + * been archived (it's required to reach the reported timeline). + * The segment containing the switch point belongs to the old + * timeline up to the switch point and should be archived. + */ + XLByteToSeg(switchpoint, switchpoint_segno, wal_segment_size); + if (file_segno <= switchpoint_segno) + { + XLogArchiveForceDone(walfile); + elog(DEBUG3, "marked ancestor timeline segment %s as archived (before switch to timeline %u)", + walfile, reported_tli); + } + } + } } FreeDir(status_dir); From c03a014ae8dc51de9e9badff3887cf59d3fa4b6f Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Wed, 11 Feb 2026 18:17:25 +0500 Subject: [PATCH 50/54] Optimize ProcessArchivalReport to avoid directory scans When archive status reports arrive sequentially on the same timeline, directly generate expected WAL filenames and mark them as archived instead of scanning the entire archive_status directory. This optimization reduces overhead in the common case where the primary continuously archives segments. Directory scan is still used when: - Timeline changes (to handle ancestor timelines) - First report received - Non-sequential reports XLogArchiveForceDone() handles all cases internally (checking if .done exists, if .ready exists, or creating .done if neither exists), so no pre-check is needed. --- src/backend/replication/walreceiver.c | 196 +++++++++++++++++--------- 1 file changed, 132 insertions(+), 64 deletions(-) diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 75b29206f5b..adcde3917ed 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -140,6 +140,14 @@ static char primary_last_archived[MAX_XFN_CHARS + 1]; static TimeLineID primary_last_archived_tli = 0; static XLogSegNo primary_last_archived_segno = 0; +/* + * Last segment we successfully marked as .done. Used to optimize + * ProcessArchivalReport() by generating expected filenames instead + * of scanning the archive_status directory. + */ +static TimeLineID last_processed_tli = 0; +static XLogSegNo last_processed_segno = 0; + /* Prototypes for private functions */ static void WalRcvFetchTimeLineHistoryFiles(TimeLineID first, TimeLineID last); static void WalRcvWaitForStartPosition(XLogRecPtr *startpoint, TimeLineID *startpointTLI); @@ -1325,10 +1333,9 @@ ProcessArchivalReport(void) { TimeLineID reported_tli; XLogSegNo reported_segno; - DIR *status_dir; - struct dirent *status_de; char status_path[MAXPGPATH]; - List *tli_history = NIL; + bool use_direct_check = false; + XLogSegNo start_segno; elog(DEBUG2, "received archival report from primary: %s", primary_last_archived); @@ -1348,90 +1355,151 @@ ProcessArchivalReport(void) primary_last_archived_tli = reported_tli; primary_last_archived_segno = reported_segno; - /* Scan archive_status directory for .ready files */ - snprintf(status_path, MAXPGPATH, XLOGDIR "/archive_status"); - status_dir = AllocateDir(status_path); - if (status_dir == NULL) + /* + * Optimization: If the new report is on the same timeline as the last + * processed segment and moves forward, we can directly check for .ready + * files for segments between last_processed_segno and reported_segno + * instead of scanning the entire archive_status directory. + * + * Fall back to directory scan if: + * - Timeline changed (need to handle ancestor timelines) + * - This is the first report (last_processed_tli == 0) + * - Reported segment is not ahead (nothing new to process) + */ + if (last_processed_tli == reported_tli && + last_processed_tli != 0 && + reported_segno > last_processed_segno) { - elog(DEBUG2, "could not open archive_status directory: %m"); - return; + use_direct_check = true; + start_segno = last_processed_segno + 1; } - while ((status_de = ReadDir(status_dir, status_path)) != NULL) + if (use_direct_check) { - char *ready_suffix; - char walfile[MAXPGPATH]; - TimeLineID file_tli; - XLogSegNo file_segno; - /* Look for .ready files only */ - ready_suffix = strstr(status_de->d_name, ".ready"); - if (ready_suffix == NULL || ready_suffix[6] != '\0') - continue; - - /* Extract WAL filename (remove .ready suffix) */ - strlcpy(walfile, status_de->d_name, ready_suffix - status_de->d_name + 1); - - /* Parse the WAL filename */ - if (!IsXLogFileName(walfile)) - continue; - - XLogFromFileName(walfile, &file_tli, &file_segno, wal_segment_size); - /* - * Mark as .done if: - * 1. Same timeline and segment <= reported segment, OR - * 2. Ancestor timeline and segment is before the timeline switch point - * - * For ancestor timelines: if primary archived segment X on timeline T, - * then all segments on ancestor timelines before the switch to T must - * have been archived (they're required to reach timeline T). + * Direct check: generate filenames for expected segments. + * XLogArchiveForceDone() will handle the case where .ready doesn't + * exist or .done already exists, so no need to stat() first. */ - if (file_tli == reported_tli && file_segno <= reported_segno) + XLogSegNo segno; + + for (segno = start_segno; segno <= reported_segno; segno++) { - /* Same timeline, segment already archived */ + char walfile[MAXFNAMELEN]; + + /* Generate WAL filename and mark as archived */ + XLogFileName(walfile, reported_tli, segno, wal_segment_size); XLogArchiveForceDone(walfile); elog(DEBUG3, "marked WAL segment %s as archived (primary archived up to %s)", walfile, primary_last_archived); + + /* Track the last segment we processed */ + last_processed_tli = reported_tli; + last_processed_segno = segno; + } + } + else + { + /* + * Directory scan: needed when timeline changed or first report. + * This handles both same-timeline and ancestor-timeline cases. + */ + DIR *status_dir; + struct dirent *status_de; + List *tli_history = NIL; + + snprintf(status_path, MAXPGPATH, XLOGDIR "/archive_status"); + status_dir = AllocateDir(status_path); + if (status_dir == NULL) + { + elog(DEBUG2, "could not open archive_status directory: %m"); + return; } - else if (file_tli != reported_tli) + + while ((status_de = ReadDir(status_dir, status_path)) != NULL) { + char *ready_suffix; + char walfile[MAXPGPATH]; + TimeLineID file_tli; + XLogSegNo file_segno; + + /* Look for .ready files only */ + ready_suffix = strstr(status_de->d_name, ".ready"); + if (ready_suffix == NULL || ready_suffix[6] != '\0') + continue; + + /* Extract WAL filename (remove .ready suffix) */ + strlcpy(walfile, status_de->d_name, ready_suffix - status_de->d_name + 1); + + /* Parse the WAL filename */ + if (!IsXLogFileName(walfile)) + continue; + + XLogFromFileName(walfile, &file_tli, &file_segno, wal_segment_size); + /* - * Different timeline - check if it's an ancestor and if this - * segment is before the timeline switch point. Only read timeline - * history if we haven't already (lazy loading). + * Mark as .done if: + * 1. Same timeline and segment <= reported segment, OR + * 2. Ancestor timeline and segment is before the timeline switch point * - * Note: Timelines form a tree structure, not a linear sequence, - * so we can't use < or > to compare them. + * For ancestor timelines: if primary archived segment X on timeline T, + * then all segments on ancestor timelines before the switch to T must + * have been archived (they're required to reach timeline T). */ - if (tli_history == NIL) - tli_history = readTimeLineHistory(reported_tli); - - if (tliInHistory(file_tli, tli_history)) + if (file_tli == reported_tli && file_segno <= reported_segno) + { + /* Same timeline, segment already archived */ + XLogArchiveForceDone(walfile); + elog(DEBUG3, "marked WAL segment %s as archived (primary archived up to %s)", + walfile, primary_last_archived); + } + else if (file_tli != reported_tli) { - XLogRecPtr switchpoint; - XLogSegNo switchpoint_segno; - - /* Get the point where we switched away from this timeline */ - switchpoint = tliSwitchPoint(file_tli, tli_history, NULL); - /* - * If the segment is at or before the switch point, it must have - * been archived (it's required to reach the reported timeline). - * The segment containing the switch point belongs to the old - * timeline up to the switch point and should be archived. + * Different timeline - check if it's an ancestor and if this + * segment is before the timeline switch point. Only read timeline + * history if we haven't already (lazy loading). + * + * Note: Timelines form a tree structure, not a linear sequence, + * so we can't use < or > to compare them. */ - XLByteToSeg(switchpoint, switchpoint_segno, wal_segment_size); - if (file_segno <= switchpoint_segno) + if (tli_history == NIL) + tli_history = readTimeLineHistory(reported_tli); + + if (tliInHistory(file_tli, tli_history)) { - XLogArchiveForceDone(walfile); - elog(DEBUG3, "marked ancestor timeline segment %s as archived (before switch to timeline %u)", - walfile, reported_tli); + XLogRecPtr switchpoint; + XLogSegNo switchpoint_segno; + + /* Get the point where we switched away from this timeline */ + switchpoint = tliSwitchPoint(file_tli, tli_history, NULL); + + /* + * If the segment is at or before the switch point, it must have + * been archived (it's required to reach the reported timeline). + * The segment containing the switch point belongs to the old + * timeline up to the switch point and should be archived. + */ + XLByteToSeg(switchpoint, switchpoint_segno, wal_segment_size); + if (file_segno <= switchpoint_segno) + { + XLogArchiveForceDone(walfile); + elog(DEBUG3, "marked ancestor timeline segment %s as archived (before switch to timeline %u)", + walfile, reported_tli); + } } } } - } - FreeDir(status_dir); + FreeDir(status_dir); + + /* + * After a full directory scan following a timeline change, update + * our tracking to the newly reported position for future optimizations. + */ + last_processed_tli = reported_tli; + last_processed_segno = reported_segno; + } } /* From 0a4deccaf4c19cf1a2df7ee995cf13e36ef89744 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Thu, 12 Feb 2026 22:25:09 +0500 Subject: [PATCH 51/54] Fuse shared archive with ycmdb.shared_archive --- src/backend/access/transam/xlog.c | 1 + src/backend/postmaster/pgarch.c | 2 +- src/backend/replication/walreceiver.c | 2 +- src/backend/replication/walsender.c | 4 ++-- src/backend/utils/misc/guc_tables.c | 12 ++++++++++++ src/include/access/xlog.h | 9 +++++++++ 6 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index bd6bac683fd..dd362b89950 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -121,6 +121,7 @@ int wal_keep_size_mb = 0; int XLOGbuffers = -1; int XLogArchiveTimeout = 0; int XLogArchiveMode = ARCHIVE_MODE_OFF; +bool ycmdb_shared_archive = false; /* makes archive_mode=on act as shared */ char *XLogArchiveCommand = NULL; bool EnableHotStandby = false; bool fullPageWrites = true; diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index 1255b8dd348..a0a87702421 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -378,7 +378,7 @@ pgarch_ArchiverCopyLoop(void) * marks files as .done when the primary confirms archival. After * promotion, the archiver starts working normally. */ - if (XLogArchiveMode == ARCHIVE_MODE_SHARED && RecoveryInProgress()) + if (EffectiveArchiveModeIsShared() && RecoveryInProgress()) return; /* force directory scan in the first call to pgarch_readyXlog() */ diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index adcde3917ed..cf1acd4f93f 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -1119,7 +1119,7 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli) { XLogArchiveNotify(xlogfname); } - else if (XLogArchiveMode == ARCHIVE_MODE_SHARED) + else if (EffectiveArchiveModeIsShared()) { /* * In shared mode, check if this segment is already archived on primary. diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 7b4aa17b8bc..96a1043aeee 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -2462,8 +2462,8 @@ WalSndArchivalReport(void) TimestampTz now; char *last_archived; - /* Only send reports when archive_mode=shared */ - if (XLogArchiveMode != ARCHIVE_MODE_SHARED) + /* Only send reports when shared archive is active */ + if (!EffectiveArchiveModeIsShared()) return; /* Only send reports during physical streaming replication, not during backup */ diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index ba6e7ef993c..eb2faeb563f 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -30,6 +30,7 @@ #include "access/gin.h" #include "access/toast_compression.h" #include "access/twophase.h" +#include "access/xlog.h" #include "access/xlog_internal.h" #include "access/xlogprefetcher.h" #include "access/xlogrecovery.h" @@ -1191,6 +1192,17 @@ struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"ycmdb.shared_archive", PGC_POSTMASTER, WAL_ARCHIVING, + gettext_noop("Makes archive_mode=on behave as shared (for managed service compatibility)."), + gettext_noop("When true, archive_mode=on is treated as archive_mode=shared. Does not affect archive_mode=off or archive_mode=always. Used when control plane cannot configure archive_mode=shared directly."), + GUC_NOT_IN_SAMPLE + }, + &ycmdb_shared_archive, + false, + NULL, NULL, NULL + }, + { {"wal_init_zero", PGC_SUSET, WAL_SETTINGS, gettext_noop("Writes zeroes to new WAL files before first use."), diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 1b39c235902..48187aa1013 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -63,6 +63,15 @@ typedef enum ArchiveMode ARCHIVE_MODE_SHARED, /* shared archive between primary and standby */ } ArchiveMode; extern PGDLLIMPORT int XLogArchiveMode; +extern PGDLLIMPORT bool ycmdb_shared_archive; + +/* + * True when shared archive behavior is active: either archive_mode=shared + * or archive_mode=on with ycmdb.shared_archive=true (managed service). + */ +#define EffectiveArchiveModeIsShared() \ + (XLogArchiveMode == ARCHIVE_MODE_SHARED || \ + (XLogArchiveMode == ARCHIVE_MODE_ON && ycmdb_shared_archive)) /* WAL levels */ typedef enum WalLevel From 7201355ab40904c2e9549a5cef3195e1a4076480 Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 14 Apr 2026 11:16:54 +0300 Subject: [PATCH 52/54] [PATCH] REASSIGN OWNED: ignore subscriptions in other databases XXX do we also need DROP OWNED to work this way? Probably not. Author: Alvaro Herrera Discussion: https://postgr.es/m/CALdSSPhjONb+EftRD=J1pqajkB+pjT0=tbMJs16C6q9+xT8NNg@mail.gmail.com --- src/backend/catalog/pg_shdepend.c | 12 +++++++++++- src/backend/utils/cache/lsyscache.c | 22 ++++++++++++++++++++++ src/include/utils/lsyscache.h | 1 + 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/backend/catalog/pg_shdepend.c b/src/backend/catalog/pg_shdepend.c index ea41d9b7502..b2e8a409a7b 100644 --- a/src/backend/catalog/pg_shdepend.c +++ b/src/backend/catalog/pg_shdepend.c @@ -68,6 +68,7 @@ #include "utils/fmgroids.h" #include "utils/memutils.h" #include "utils/syscache.h" +#include "utils/lsyscache.h" typedef enum { @@ -1535,7 +1536,16 @@ shdepReassignOwned(List *roleids, Oid newrole) continue; /* - * The various ALTER OWNER routines tend to leak memory in + * Subscriptions are linked to specific databases, even though + * they are nominally shared objects. Skip those that aren't + * in this database. + */ + if (sdepForm->classid == SubscriptionRelationId && + get_subscription_database(sdepForm->objid) != MyDatabaseId) + continue; + + /* + * The various DDL routines called here tend to leak memory in * CurrentMemoryContext. That's not a problem when they're only * called once per command; but in this usage where we might be * touching many objects, it can amount to a serious memory leak. diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index 60978f9415b..9cec2d6ed08 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -3672,3 +3672,25 @@ get_subscription_name(Oid subid, bool missing_ok) return subname; } + +/* + * Return the OID of the database the given subscription is in. + */ +Oid +get_subscription_database(Oid subid) +{ + HeapTuple tup; + Form_pg_subscription subform; + Oid subdbid; + + tup = SearchSysCache1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid)); + if (!HeapTupleIsValid(tup)) + elog(ERROR, "cache lookup failed for subscription %u", subid); + + subform = (Form_pg_subscription) GETSTRUCT(tup); + subdbid = subform->subdbid; + + ReleaseSysCache(tup); + + return subdbid; +} diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h index 4f5418b9728..52f7ec63c80 100644 --- a/src/include/utils/lsyscache.h +++ b/src/include/utils/lsyscache.h @@ -202,6 +202,7 @@ extern Oid get_publication_oid(const char *pubname, bool missing_ok); extern char *get_publication_name(Oid pubid, bool missing_ok); extern Oid get_subscription_oid(const char *subname, bool missing_ok); extern char *get_subscription_name(Oid subid, bool missing_ok); +extern Oid get_subscription_database(Oid subid); #define type_is_array(typid) (get_element_type(typid) != InvalidOid) /* type_is_array_domain accepts both plain arrays and domains over arrays */ From 9427963f6c940f3100a86c5f1d8f9012c24ac78b Mon Sep 17 00:00:00 2001 From: Roman Khapov Date: Sun, 19 Apr 2026 18:50:00 +0000 Subject: [PATCH 53/54] Introduce transaction_timeout This commit adds timeout that is expected to be used as a prevention of long-running queries. Any session within the transaction will be terminated after spanning longer than this timeout. However, this timeout is not applied to prepared transactions. Only transactions with user connections are affected. Don't deal with transaction timeout in PostgresMain(). Instead, release transaction timeout activated by StartTransaction() in CommitTransaction()/AbortTransaction()/PrepareTransaction(). Deal with both enabling and disabling transaction timeout in assign_transaction_timeout(). Also, remove potentially flaky timeouts-long isolation test, which has no guarantees to pass on slow/busy machines. Discussion: https://postgr.es/m/20240215230856.pc6k57tqxt7fhldm%40awork3.anarazel.de Discussion: https://postgr.es/m/CAAhFRxiQsRs2Eq5kCo9nXE3HTugsAAJdSQSmxncivebAxdmBjQ%40mail.gmail.com Author: Andrey Borodin Author: Japin Li Author: Junwang Zhao Reviewed-by: Andres Freund Reviewed-by: Fujii Masao Reviewed-by: bt23nguyent Reviewed-by: Yuhang Qiu Backport-by: rkhapov Reviewed-by: reshke ===== Cherry-pick source: 51efe38cb92f4b15b68811bcce9ab878fbc71ea5 bf82f43790a675dd1b9522a7799357e61e7aa635 --- doc/src/sgml/config.sgml | 36 +++++++++ src/backend/access/transam/xact.c | 16 ++++ src/backend/postmaster/autovacuum.c | 2 + src/backend/storage/lmgr/proc.c | 1 + src/backend/tcop/postgres.c | 37 ++++++++- src/backend/utils/errcodes.txt | 1 + src/backend/utils/init/globals.c | 1 + src/backend/utils/init/postinit.c | 10 +++ src/backend/utils/misc/guc_tables.c | 11 +++ src/backend/utils/misc/postgresql.conf.sample | 5 +- src/bin/pg_dump/pg_backup_archiver.c | 1 + src/bin/pg_dump/pg_dump.c | 2 + src/bin/pg_rewind/libpq_source.c | 1 + src/include/miscadmin.h | 1 + src/include/storage/proc.h | 1 + src/include/utils/guc_hooks.h | 1 + src/include/utils/timeout.h | 1 + src/test/isolation/Makefile | 3 + src/test/isolation/expected/timeouts-long.out | 69 ++++++++++++++++ src/test/isolation/expected/timeouts.out | 79 ++++++++++++++++++- src/test/isolation/specs/timeouts.spec | 40 +++++++++- 21 files changed, 312 insertions(+), 7 deletions(-) create mode 100644 src/test/isolation/expected/timeouts-long.out diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 9f735dc95f5..cccd933be37 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -9242,6 +9242,42 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; + + transaction_timeout (integer) + + transaction_timeout configuration parameter + + + + + Terminate any session that spans longer than the specified amount of + time in the transaction. The limit applies both to explicit transactions + (started with BEGIN) and to an implicitly started + transaction corresponding to a single statement. + If this value is specified without units, it is taken as milliseconds. + A value of zero (the default) disables the timeout. + + + + If transaction_timeout is shorter or equal to + idle_in_transaction_session_timeout or statement_timeout + transaction_timeout will invalidate the longer timeout. + + + + Setting transaction_timeout in + postgresql.conf is not recommended because it would + affect all sessions. + + + + + Prepared transactions are not subject to this timeout. + + + + + lock_timeout (integer) diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 95fa6ce039a..62ad896e73b 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -2150,6 +2150,10 @@ StartTransaction(void) */ s->state = TRANS_INPROGRESS; + /* Schedule transaction timeout */ + if (TransactionTimeout > 0) + enable_timeout_after(TRANSACTION_TIMEOUT, TransactionTimeout); + ShowTransactionState("StartTransaction"); } @@ -2269,6 +2273,10 @@ CommitTransaction(void) s->state = TRANS_COMMIT; s->parallelModeLevel = 0; + /* Disable transaction timeout */ + if (TransactionTimeout > 0) + disable_timeout(TRANSACTION_TIMEOUT, false); + if (!is_parallel_worker) { /* @@ -2542,6 +2550,10 @@ PrepareTransaction(void) */ s->state = TRANS_PREPARE; + /* Disable transaction timeout */ + if (TransactionTimeout > 0) + disable_timeout(TRANSACTION_TIMEOUT, false); + prepared_at = GetCurrentTimestamp(); /* @@ -2714,6 +2726,10 @@ AbortTransaction(void) /* Prevent cancel/die interrupt while cleaning up */ HOLD_INTERRUPTS(); + /* Disable transaction timeout */ + if (TransactionTimeout > 0) + disable_timeout(TRANSACTION_TIMEOUT, false); + /* Make sure we have a valid memory context and resource owner */ AtAbort_Memory(); AtAbort_ResourceOwner(); diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 246ac2bec88..ecc7d8bfd2e 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -600,6 +600,7 @@ AutoVacLauncherMain(int argc, char *argv[]) * regular maintenance from being executed. */ SetConfigOption("statement_timeout", "0", PGC_SUSET, PGC_S_OVERRIDE); + SetConfigOption("transaction_timeout", "0", PGC_SUSET, PGC_S_OVERRIDE); SetConfigOption("lock_timeout", "0", PGC_SUSET, PGC_S_OVERRIDE); SetConfigOption("idle_in_transaction_session_timeout", "0", PGC_SUSET, PGC_S_OVERRIDE); @@ -1618,6 +1619,7 @@ AutoVacWorkerMain(int argc, char *argv[]) * regular maintenance from being executed. */ SetConfigOption("statement_timeout", "0", PGC_SUSET, PGC_S_OVERRIDE); + SetConfigOption("transaction_timeout", "0", PGC_SUSET, PGC_S_OVERRIDE); SetConfigOption("lock_timeout", "0", PGC_SUSET, PGC_S_OVERRIDE); SetConfigOption("idle_in_transaction_session_timeout", "0", PGC_SUSET, PGC_S_OVERRIDE); diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index 4b1772f94c0..4a27dafe17d 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -59,6 +59,7 @@ int DeadlockTimeout = 1000; int StatementTimeout = 0; int LockTimeout = 0; int IdleInTransactionSessionTimeout = 0; +int TransactionTimeout = 0; int IdleSessionTimeout = 0; bool log_lock_waits = false; diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 95df771a41a..126fcbb2d17 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -3448,6 +3448,17 @@ ProcessInterrupts(void) IdleInTransactionSessionTimeoutPending = false; } + if (TransactionTimeoutPending) + { + /* As above, ignore the signal if the GUC has been reset to zero. */ + if (TransactionTimeout > 0) + ereport(FATAL, + (errcode(ERRCODE_TRANSACTION_TIMEOUT), + errmsg("terminating connection due to transaction timeout"))); + else + TransactionTimeoutPending = false; + } + if (IdleSessionTimeoutPending) { /* As above, ignore the signal if the GUC has been reset to zero. */ @@ -3723,6 +3734,23 @@ assign_restrict_nonsystem_relation_kind(const char *newval, void *extra) restrict_nonsystem_relation_kind = *flags; } +/* GUC assign hook for transaction_timeout */ +void +assign_transaction_timeout(int newval, void *extra) +{ + if (IsTransactionState()) + { + /* + * If transaction_timeout GUC has changes within the transaction block + * enable or disable the timer correspondingly. + */ + if (newval > 0 && !get_timeout_active(TRANSACTION_TIMEOUT)) + enable_timeout_after(TRANSACTION_TIMEOUT, newval); + else if (newval <= 0 && get_timeout_active(TRANSACTION_TIMEOUT)) + disable_timeout(TRANSACTION_TIMEOUT, false); + } +} + /* * set_debug_options --- apply "-d N" command line option * @@ -4569,7 +4597,8 @@ PostgresMain(const char *dbname, const char *username) pgstat_report_activity(STATE_IDLEINTRANSACTION_ABORTED, NULL); /* Start the idle-in-transaction timer */ - if (IdleInTransactionSessionTimeout > 0) + if (IdleInTransactionSessionTimeout > 0 + && (IdleInTransactionSessionTimeout < TransactionTimeout || TransactionTimeout == 0)) { idle_in_transaction_timeout_enabled = true; enable_timeout_after(IDLE_IN_TRANSACTION_SESSION_TIMEOUT, @@ -4582,7 +4611,8 @@ PostgresMain(const char *dbname, const char *username) pgstat_report_activity(STATE_IDLEINTRANSACTION, NULL); /* Start the idle-in-transaction timer */ - if (IdleInTransactionSessionTimeout > 0) + if (IdleInTransactionSessionTimeout > 0 + && (IdleInTransactionSessionTimeout < TransactionTimeout || TransactionTimeout == 0)) { idle_in_transaction_timeout_enabled = true; enable_timeout_after(IDLE_IN_TRANSACTION_SESSION_TIMEOUT, @@ -5198,7 +5228,8 @@ enable_statement_timeout(void) /* must be within an xact */ Assert(xact_started); - if (StatementTimeout > 0) + if (StatementTimeout > 0 + && (StatementTimeout < TransactionTimeout || TransactionTimeout == 0)) { if (!get_timeout_active(STATEMENT_TIMEOUT)) enable_timeout_after(STATEMENT_TIMEOUT, StatementTimeout); diff --git a/src/backend/utils/errcodes.txt b/src/backend/utils/errcodes.txt index 3d244af130a..dcc55d7afa1 100644 --- a/src/backend/utils/errcodes.txt +++ b/src/backend/utils/errcodes.txt @@ -252,6 +252,7 @@ Section: Class 25 - Invalid Transaction State 25P01 E ERRCODE_NO_ACTIVE_SQL_TRANSACTION no_active_sql_transaction 25P02 E ERRCODE_IN_FAILED_SQL_TRANSACTION in_failed_sql_transaction 25P03 E ERRCODE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT idle_in_transaction_session_timeout +25P04 E ERRCODE_TRANSACTION_TIMEOUT transaction_timeout Section: Class 26 - Invalid SQL Statement Name diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c index 19edc3d8674..241cdfede35 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -33,6 +33,7 @@ volatile sig_atomic_t ProcDiePending = false; volatile sig_atomic_t CheckClientConnectionPending = false; volatile sig_atomic_t ClientConnectionLost = false; volatile sig_atomic_t IdleInTransactionSessionTimeoutPending = false; +volatile sig_atomic_t TransactionTimeoutPending = false; volatile sig_atomic_t IdleSessionTimeoutPending = false; volatile sig_atomic_t ProcSignalBarrierPending = false; volatile sig_atomic_t LogMemoryContextPending = false; diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 3c7e9620129..3dfbfbc9daa 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -75,6 +75,7 @@ static void ShutdownPostgres(int code, Datum arg); static void StatementTimeoutHandler(void); static void LockTimeoutHandler(void); static void IdleInTransactionSessionTimeoutHandler(void); +static void TransactionTimeoutHandler(void); static void IdleSessionTimeoutHandler(void); static void IdleStatsUpdateTimeoutHandler(void); static void ClientCheckTimeoutHandler(void); @@ -766,6 +767,7 @@ InitPostgres(const char *in_dbname, Oid dboid, RegisterTimeout(LOCK_TIMEOUT, LockTimeoutHandler); RegisterTimeout(IDLE_IN_TRANSACTION_SESSION_TIMEOUT, IdleInTransactionSessionTimeoutHandler); + RegisterTimeout(TRANSACTION_TIMEOUT, TransactionTimeoutHandler); RegisterTimeout(IDLE_SESSION_TIMEOUT, IdleSessionTimeoutHandler); RegisterTimeout(CLIENT_CONNECTION_CHECK_TIMEOUT, ClientCheckTimeoutHandler); RegisterTimeout(IDLE_STATS_UPDATE_TIMEOUT, @@ -1401,6 +1403,14 @@ LockTimeoutHandler(void) kill(MyProcPid, SIGINT); } +static void +TransactionTimeoutHandler(void) +{ + TransactionTimeoutPending = true; + InterruptPending = true; + SetLatch(MyLatch); +} + static void IdleInTransactionSessionTimeoutHandler(void) { diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index eb2faeb563f..00a4081eb15 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -2575,6 +2575,17 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"transaction_timeout", PGC_USERSET, CLIENT_CONN_STATEMENT, + gettext_noop("Sets the maximum allowed time in a transaction with a session (not a prepared transaction)."), + gettext_noop("A value of 0 turns off the timeout."), + GUC_UNIT_MS + }, + &TransactionTimeout, + 0, 0, INT_MAX, + NULL, assign_transaction_timeout, NULL + }, + { {"idle_session_timeout", PGC_USERSET, CLIENT_CONN_STATEMENT, gettext_noop("Sets the maximum allowed idle time between queries, when not in a transaction."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index c92bdfeef81..cd35cb52e01 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -702,8 +702,9 @@ #default_transaction_read_only = off #default_transaction_deferrable = off #session_replication_role = 'origin' -#statement_timeout = 0 # in milliseconds, 0 is disabled -#lock_timeout = 0 # in milliseconds, 0 is disabled +#statement_timeout = 0 # in milliseconds, 0 is disabled +#transaction_timeout = 0 # in milliseconds, 0 is disabled +#lock_timeout = 0 # in milliseconds, 0 is disabled #idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled #idle_session_timeout = 0 # in milliseconds, 0 is disabled #vacuum_freeze_table_age = 150000000 diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 8e87d5d0f8a..217df8c513f 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -3190,6 +3190,7 @@ _doSetFixedOutputState(ArchiveHandle *AH) ahprintf(AH, "SET statement_timeout = 0;\n"); ahprintf(AH, "SET lock_timeout = 0;\n"); ahprintf(AH, "SET idle_in_transaction_session_timeout = 0;\n"); + ahprintf(AH, "SET transaction_timeout = 0;\n"); /* Select the correct character set encoding */ ahprintf(AH, "SET client_encoding = '%s';\n", diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 09943d457c0..4191cd4f864 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -1263,6 +1263,8 @@ setup_connection(Archive *AH, const char *dumpencoding, ExecuteSqlStatement(AH, "SET lock_timeout = 0"); if (AH->remoteVersion >= 90600) ExecuteSqlStatement(AH, "SET idle_in_transaction_session_timeout = 0"); + if (AH->remoteVersion >= 140000) + ExecuteSqlStatement(AH, "SET transaction_timeout = 0"); /* * Quote all identifiers, if requested. diff --git a/src/bin/pg_rewind/libpq_source.c b/src/bin/pg_rewind/libpq_source.c index 0d8e9ee2d1a..6537bee4a00 100644 --- a/src/bin/pg_rewind/libpq_source.c +++ b/src/bin/pg_rewind/libpq_source.c @@ -117,6 +117,7 @@ init_libpq_conn(PGconn *conn) run_simple_command(conn, "SET statement_timeout = 0"); run_simple_command(conn, "SET lock_timeout = 0"); run_simple_command(conn, "SET idle_in_transaction_session_timeout = 0"); + run_simple_command(conn, "SET transaction_timeout = 0"); /* * we don't intend to do any updates, put the connection in read-only mode diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index adfea00fd11..9d8d0215d07 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -91,6 +91,7 @@ extern PGDLLIMPORT volatile sig_atomic_t InterruptPending; extern PGDLLIMPORT volatile sig_atomic_t QueryCancelPending; extern PGDLLIMPORT volatile sig_atomic_t ProcDiePending; extern PGDLLIMPORT volatile sig_atomic_t IdleInTransactionSessionTimeoutPending; +extern PGDLLIMPORT volatile sig_atomic_t TransactionTimeoutPending; extern PGDLLIMPORT volatile sig_atomic_t IdleSessionTimeoutPending; extern PGDLLIMPORT volatile sig_atomic_t ProcSignalBarrierPending; extern PGDLLIMPORT volatile sig_atomic_t LogMemoryContextPending; diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index e16ab4a2e58..19da8390d64 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -428,6 +428,7 @@ extern PGDLLIMPORT int DeadlockTimeout; extern PGDLLIMPORT int StatementTimeout; extern PGDLLIMPORT int LockTimeout; extern PGDLLIMPORT int IdleInTransactionSessionTimeout; +extern PGDLLIMPORT int TransactionTimeout; extern PGDLLIMPORT int IdleSessionTimeout; extern PGDLLIMPORT bool log_lock_waits; diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 499d379d7d0..a912c032349 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -157,6 +157,7 @@ extern void assign_timezone_abbreviations(const char *newval, void *extra); extern bool check_transaction_deferrable(bool *newval, void **extra, GucSource source); extern bool check_transaction_isolation(int *newval, void **extra, GucSource source); extern bool check_transaction_read_only(bool *newval, void **extra, GucSource source); +extern void assign_transaction_timeout(int newval, void *extra); extern const char *show_unix_socket_permissions(void); extern bool check_wal_buffers(int *newval, void **extra, GucSource source); extern bool check_wal_consistency_checking(char **newval, void **extra, diff --git a/src/include/utils/timeout.h b/src/include/utils/timeout.h index e561a1cde92..fe5447f8174 100644 --- a/src/include/utils/timeout.h +++ b/src/include/utils/timeout.h @@ -31,6 +31,7 @@ typedef enum TimeoutId STANDBY_TIMEOUT, STANDBY_LOCK_TIMEOUT, IDLE_IN_TRANSACTION_SESSION_TIMEOUT, + TRANSACTION_TIMEOUT, IDLE_SESSION_TIMEOUT, IDLE_STATS_UPDATE_TIMEOUT, CLIENT_CONNECTION_CHECK_TIMEOUT, diff --git a/src/test/isolation/Makefile b/src/test/isolation/Makefile index b8738b7c1be..9e93fc3f153 100644 --- a/src/test/isolation/Makefile +++ b/src/test/isolation/Makefile @@ -78,3 +78,6 @@ installcheck-prepared-txns: all temp-install check-prepared-txns: all temp-install $(pg_isolation_regress_check) --schedule=$(srcdir)/isolation_schedule prepared-transactions prepared-transactions-cic + +check-timeouts: all temp-install + $(pg_isolation_regress_check) timeouts timeouts-long diff --git a/src/test/isolation/expected/timeouts-long.out b/src/test/isolation/expected/timeouts-long.out new file mode 100644 index 00000000000..26a6672c051 --- /dev/null +++ b/src/test/isolation/expected/timeouts-long.out @@ -0,0 +1,69 @@ +Parsed test spec with 3 sessions + +starting permutation: s7_begin s7_sleep s7_commit_and_chain s7_sleep s7_check s7_abort +step s7_begin: + BEGIN ISOLATION LEVEL READ COMMITTED; + SET transaction_timeout = '1s'; + +step s7_sleep: SELECT pg_sleep(0.6); +pg_sleep +-------- + +(1 row) + +step s7_commit_and_chain: COMMIT AND CHAIN; +step s7_sleep: SELECT pg_sleep(0.6); +pg_sleep +-------- + +(1 row) + +step s7_check: SELECT count(*) FROM pg_stat_activity WHERE application_name = 'isolation/timeouts/s7'; +count +----- + 0 +(1 row) + +step s7_abort: ABORT; + +starting permutation: s8_begin s8_sleep s8_select_1 s8_check checker_sleep checker_sleep s8_check +step s8_begin: + BEGIN ISOLATION LEVEL READ COMMITTED; + SET transaction_timeout = '900ms'; + +step s8_sleep: SELECT pg_sleep(0.6); +pg_sleep +-------- + +(1 row) + +step s8_select_1: SELECT 1; +?column? +-------- + 1 +(1 row) + +step s8_check: SELECT count(*) FROM pg_stat_activity WHERE application_name = 'isolation/timeouts/s8'; +count +----- + 0 +(1 row) + +step checker_sleep: SELECT pg_sleep(0.3); +pg_sleep +-------- + +(1 row) + +step checker_sleep: SELECT pg_sleep(0.3); +pg_sleep +-------- + +(1 row) + +step s8_check: SELECT count(*) FROM pg_stat_activity WHERE application_name = 'isolation/timeouts/s8'; +count +----- + 0 +(1 row) + diff --git a/src/test/isolation/expected/timeouts.out b/src/test/isolation/expected/timeouts.out index 9328676f1cc..81a0016375b 100644 --- a/src/test/isolation/expected/timeouts.out +++ b/src/test/isolation/expected/timeouts.out @@ -1,4 +1,4 @@ -Parsed test spec with 2 sessions +Parsed test spec with 7 sessions starting permutation: rdtbl sto locktbl step rdtbl: SELECT * FROM accounts; @@ -79,3 +79,80 @@ step slto: SET lock_timeout = '10s'; SET statement_timeout = '10ms'; step update: DELETE FROM accounts WHERE accountid = 'checking'; step update: <... completed> ERROR: canceling statement due to statement timeout + +starting permutation: stto s3_begin s3_sleep s3_check s3_abort +step stto: SET statement_timeout = '10ms'; SET transaction_timeout = '1s'; +step s3_begin: BEGIN ISOLATION LEVEL READ COMMITTED; +step s3_sleep: SELECT pg_sleep(0.1); +ERROR: canceling statement due to statement timeout +step s3_check: SELECT count(*) FROM pg_stat_activity WHERE application_name = 'isolation/timeouts/s3'; +count +----- + 1 +(1 row) + +step s3_abort: ABORT; + +starting permutation: tsto s3_begin checker_sleep s3_check +step tsto: SET statement_timeout = '1s'; SET transaction_timeout = '10ms'; +step s3_begin: BEGIN ISOLATION LEVEL READ COMMITTED; +step checker_sleep: SELECT pg_sleep(0.1); +pg_sleep +-------- + +(1 row) + +step s3_check: SELECT count(*) FROM pg_stat_activity WHERE application_name = 'isolation/timeouts/s3'; +count +----- + 0 +(1 row) + + +starting permutation: itto s4_begin checker_sleep s4_check +step itto: SET idle_in_transaction_session_timeout = '10ms'; SET transaction_timeout = '1s'; +step s4_begin: BEGIN ISOLATION LEVEL READ COMMITTED; +step checker_sleep: SELECT pg_sleep(0.1); +pg_sleep +-------- + +(1 row) + +step s4_check: SELECT count(*) FROM pg_stat_activity WHERE application_name = 'isolation/timeouts/s4'; +count +----- + 0 +(1 row) + + +starting permutation: tito s5_begin checker_sleep s5_check +step tito: SET idle_in_transaction_session_timeout = '1s'; SET transaction_timeout = '10ms'; +step s5_begin: BEGIN ISOLATION LEVEL READ COMMITTED; +step checker_sleep: SELECT pg_sleep(0.1); +pg_sleep +-------- + +(1 row) + +step s5_check: SELECT count(*) FROM pg_stat_activity WHERE application_name = 'isolation/timeouts/s5'; +count +----- + 0 +(1 row) + + +starting permutation: s6_begin s6_tt checker_sleep s6_check +step s6_begin: BEGIN ISOLATION LEVEL READ COMMITTED; +step s6_tt: SET statement_timeout = '1s'; SET transaction_timeout = '10ms'; +step checker_sleep: SELECT pg_sleep(0.1); +pg_sleep +-------- + +(1 row) + +step s6_check: SELECT count(*) FROM pg_stat_activity WHERE application_name = 'isolation/timeouts/s6'; +count +----- + 0 +(1 row) + diff --git a/src/test/isolation/specs/timeouts.spec b/src/test/isolation/specs/timeouts.spec index c747b4ae28d..c2cc5d8d37b 100644 --- a/src/test/isolation/specs/timeouts.spec +++ b/src/test/isolation/specs/timeouts.spec @@ -1,4 +1,4 @@ -# Simple tests for statement_timeout and lock_timeout features +# Simple tests for statement_timeout, lock_timeout and transaction_timeout features setup { @@ -27,6 +27,33 @@ step locktbl { LOCK TABLE accounts; } step update { DELETE FROM accounts WHERE accountid = 'checking'; } teardown { ABORT; } +session s3 +step s3_begin { BEGIN ISOLATION LEVEL READ COMMITTED; } +step stto { SET statement_timeout = '10ms'; SET transaction_timeout = '1s'; } +step tsto { SET statement_timeout = '1s'; SET transaction_timeout = '10ms'; } +step s3_sleep { SELECT pg_sleep(0.1); } +step s3_abort { ABORT; } + +session s4 +step s4_begin { BEGIN ISOLATION LEVEL READ COMMITTED; } +step itto { SET idle_in_transaction_session_timeout = '10ms'; SET transaction_timeout = '1s'; } + +session s5 +step s5_begin { BEGIN ISOLATION LEVEL READ COMMITTED; } +step tito { SET idle_in_transaction_session_timeout = '1s'; SET transaction_timeout = '10ms'; } + +session s6 +step s6_begin { BEGIN ISOLATION LEVEL READ COMMITTED; } +step s6_tt { SET statement_timeout = '1s'; SET transaction_timeout = '10ms'; } + +session checker +step checker_sleep { SELECT pg_sleep(0.1); } +step s3_check { SELECT count(*) FROM pg_stat_activity WHERE application_name = 'isolation/timeouts/s3'; } +step s4_check { SELECT count(*) FROM pg_stat_activity WHERE application_name = 'isolation/timeouts/s4'; } +step s5_check { SELECT count(*) FROM pg_stat_activity WHERE application_name = 'isolation/timeouts/s5'; } +step s6_check { SELECT count(*) FROM pg_stat_activity WHERE application_name = 'isolation/timeouts/s6'; } + + # It's possible that the isolation tester will not observe the final # steps as "waiting", thanks to the relatively short timeouts we use. # We can ensure consistent test output by marking those steps with (*). @@ -47,3 +74,14 @@ permutation wrtbl lto update(*) permutation wrtbl lsto update(*) # statement timeout expires first, row-level lock permutation wrtbl slto update(*) + +# statement timeout expires first +permutation stto s3_begin s3_sleep s3_check s3_abort +# transaction timeout expires first, session s3 FATAL-out +permutation tsto s3_begin checker_sleep s3_check +# idle in transaction timeout expires first, session s4 FATAL-out +permutation itto s4_begin checker_sleep s4_check +# transaction timeout expires first, session s5 FATAL-out +permutation tito s5_begin checker_sleep s5_check +# transaction timeout can be schedule amid transaction, session s6 FATAL-out +permutation s6_begin s6_tt checker_sleep s6_check \ No newline at end of file From c44f07404bac69c51f71c6c036d3ed078062f1b8 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Fri, 17 Apr 2026 23:06:08 +0500 Subject: [PATCH 54/54] Fix two bugs in archive_mode=shared on standby 1. Checkpoint on standby deletes WAL with .ready status. XLogArchiveCheckDone() treated archive_mode=shared like archive_mode=on during recovery, returning true unconditionally and allowing checkpoint to remove WAL segments that the primary had not yet archived. Fix: exclude shared mode from the early-return path, same as "always". 2. Walsender never sends archival status reports after archiving is restored. WalSndArchivalReport() calls pgstat_fetch_stat_archiver() whose result is cached per-session (PGSTAT_FETCH_CONSISTENCY_CACHE by default). The walsender has no transaction boundaries that would clear the cache, so last_archived_wal remained "" forever, and strcmp() suppressed all reports. Fix: call pgstat_clear_snapshot() before fetching archiver stats. Add TAP tests in 051_archive_shared_checkpoint.pl that reproduce both bugs, and extend 050_archive_shared.pl with checkpoint/restore scenarios. --- src/backend/access/transam/xlogarchive.c | 14 +- src/backend/replication/walsender.c | 9 +- src/test/recovery/t/050_archive_shared.pl | 138 ++++++++++++ .../t/051_archive_shared_checkpoint.pl | 211 ++++++++++++++++++ 4 files changed, 365 insertions(+), 7 deletions(-) create mode 100644 src/test/recovery/t/051_archive_shared_checkpoint.pl diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c index 524e80adb1c..72c82e0902b 100644 --- a/src/backend/access/transam/xlogarchive.c +++ b/src/backend/access/transam/xlogarchive.c @@ -574,16 +574,22 @@ XLogArchiveCheckDone(const char *xlog) /* * During archive recovery, the file is deletable if archive_mode is not - * "always". + * "always" or "shared". + * + * In "shared" mode the standby does not archive independently; instead it + * waits for the primary to report successful archival, at which point the + * walreceiver converts the .ready file to .done. We must therefore fall + * through to the .done/.ready check below so that checkpoint cannot + * delete a segment whose .ready file has not yet become .done. */ - if (!XLogArchivingAlways() && + if (!XLogArchivingAlways() && !EffectiveArchiveModeIsShared() && GetRecoveryState() == RECOVERY_STATE_ARCHIVE) return true; /* * At this point of the logic, note that we are either a primary with - * archive_mode set to "on" or "always", or a standby with archive_mode - * set to "always". + * archive_mode set to "on" or "always", a standby with archive_mode set + * to "always", or a standby with archive_mode set to "shared". */ /* First check for .done --- this means archiver is done with it */ diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 96a1043aeee..864a0ac3acb 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -2492,10 +2492,13 @@ WalSndArchivalReport(void) return; last_archival_report_timestamp = now; /* - * Get archiver statistics. We use non-blocking access to avoid delaying - * replication if stats collector is slow. If stats are unavailable or - * stale, we'll just try again at the next interval. + * Get archiver statistics. The pgstat snapshot is cached per-session and + * is only invalidated at transaction boundaries. The walsender runs + * without transaction boundaries, so we must clear the snapshot explicitly + * to avoid reading stale data (e.g. last_archived_wal stuck at its initial + * empty value even after the archiver has archived new segments). */ + pgstat_clear_snapshot(); archiver_stats = pgstat_fetch_stat_archiver(); if (archiver_stats == NULL) return; diff --git a/src/test/recovery/t/050_archive_shared.pl b/src/test/recovery/t/050_archive_shared.pl index 397b71ad79d..fdcdbb0657b 100644 --- a/src/test/recovery/t/050_archive_shared.pl +++ b/src/test/recovery/t/050_archive_shared.pl @@ -267,4 +267,142 @@ ok($standby2_count >= 3500, "standby2 has all data (got $standby2_count rows)"); ok($standby3_count >= 3500, "standby3 has all data (got $standby3_count rows)"); +############################################################################### +# Test 5: checkpoint on standby must NOT delete WAL that has .ready status +# +# In archive_mode=shared, the standby relies on archival reports from the +# primary to know when a segment is safe to delete. Segments not yet +# confirmed as archived have .ready files. A checkpoint (CreateRestartPoint) +# must not remove those WAL files because they may be needed for recovery +# after a standby promotion if the primary never archived them. +# +# Root cause: XLogArchiveCheckDone() treats archive_mode=shared the same as +# archive_mode=on during recovery, bypassing the .ready/.done check. +############################################################################### + +note("Test 5: checkpoint must not delete WAL with .ready on standby"); + +my $archive_dir5 = PostgreSQL::Test::Utils::tempdir(); +my $primary5 = PostgreSQL::Test::Cluster->new('primary5'); +$primary5->init(has_archiving => 1, allows_streaming => 1); +$primary5->append_conf( + 'postgresql.conf', qq{ +archive_mode = shared +archive_command = 'cp %p "$archive_dir5/%f"' +}); +$primary5->start; +$primary5->safe_psql('postgres', 'CREATE TABLE t5 (i int);'); + +# Ensure WAL activity exists in the current segment before switching. +# pg_switch_wal() is a no-op when called at the very start of a segment, +# so we write a row first to guarantee there is WAL to switch away from. +$primary5->safe_psql('postgres', 'INSERT INTO t5 VALUES (0);'); +$primary5->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for archiver to archive the switched segment +$primary5->poll_query_until('postgres', + 'SELECT archived_count > 0 FROM pg_stat_archiver') + or die "primary5: archiver did not start"; + +# Create standby without wal_keep_size so checkpoint is free to recycle segments +# backup() returns an empty list (bare "return"), so the backup name must be +# stored separately before passing it to init_from_backup. +$primary5->backup('backup5'); +my $standby5 = PostgreSQL::Test::Cluster->new('standby5'); +$standby5->init_from_backup($primary5, 'backup5', has_streaming => 1); +$standby5->append_conf( + 'postgresql.conf', qq{ +archive_mode = shared +archive_command = 'cp %p "$archive_dir5/%f"' +wal_receiver_status_interval = 1s +}); +$standby5->start; +$primary5->wait_for_catchup($standby5); + +# Break archiving on primary: new segments received by standby will get .ready +$primary5->adjust_conf('postgresql.conf', 'archive_command', "'/bin/false'"); +$primary5->reload; + +# Generate several complete WAL segments. After the standby replays all of +# them its redo pointer is well past the first few, making those candidates +# for checkpoint removal. +for (1 .. 6) +{ + $primary5->safe_psql('postgres', + 'INSERT INTO t5 SELECT generate_series(1,1000);'); + $primary5->safe_psql('postgres', 'SELECT pg_switch_wal();'); +} +$primary5->wait_for_catchup($standby5); + +# Collect every WAL segment that has a .ready file on the standby +my $status_dir5 = $standby5->data_dir . '/pg_wal/archive_status'; +my @ready5; +if (opendir(my $dh, $status_dir5)) +{ + @ready5 = map { s/\.ready$//r } grep { /\.ready$/ } readdir($dh); + closedir($dh); +} +my $n_ready5 = scalar @ready5; +note("Before checkpoint: $n_ready5 WAL files with .ready"); +cmp_ok($n_ready5, '>', 0, "standby has .ready WAL files before checkpoint"); + +# Trigger CreateRestartPoint (the standby equivalent of CHECKPOINT). +# It must not remove WAL files that carry a .ready status. +$standby5->safe_psql('postgres', 'CHECKPOINT'); + +my $wal_dir5 = $standby5->data_dir . '/pg_wal'; +my $deleted5 = 0; +for my $f (@ready5) +{ + unless (-f "$wal_dir5/$f") + { + $deleted5++; + diag("BUG: $f had .ready but checkpoint deleted it from standby"); + } +} +is($deleted5, 0, + "checkpoint does not delete WAL with .ready (not yet archived by primary)"); + +############################################################################### +# Test 6: after archiving is restored on primary, standby .ready -> .done +# +# When archive_command is broken for a while and then fixed, the primary will +# archive the previously-failed segments. The walsender sends an archival +# status report to the standby which then converts .ready to .done. +# This verifies the end-to-end recovery of the mechanism after an outage. +############################################################################### + +note("Test 6: .ready files become .done after archiving restored on primary"); + +# Capture archived_count before restoring so we can detect new archival +my $archived_before5 = + $primary5->safe_psql('postgres', 'SELECT archived_count FROM pg_stat_archiver'); + +# Restore archiving +$primary5->adjust_conf('postgresql.conf', 'archive_command', + qq{'cp %p "$archive_dir5/%f"'}); +$primary5->reload; + +# Wait for primary to archive the segments that failed during the outage +$primary5->poll_query_until('postgres', + "SELECT archived_count > $archived_before5 FROM pg_stat_archiver") + or die "primary5: archiver did not catch up after archive_command restored"; + +# The walsender sends archival status reports every ~10 s. Wait up to +# timeout_default seconds for every .ready file to transition to .done. +my $remaining5 = $n_ready5; +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $remaining5 = 0; + if (opendir(my $dh, $status_dir5)) + { + $remaining5 = scalar(grep { /\.ready$/ } readdir($dh)); + closedir($dh); + } + last if $remaining5 == 0; + sleep(1); +} +is($remaining5, 0, + "all .ready files become .done after archiving restored on primary"); + done_testing(); diff --git a/src/test/recovery/t/051_archive_shared_checkpoint.pl b/src/test/recovery/t/051_archive_shared_checkpoint.pl new file mode 100644 index 00000000000..278fc2f8b70 --- /dev/null +++ b/src/test/recovery/t/051_archive_shared_checkpoint.pl @@ -0,0 +1,211 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Tests for archive_mode=shared correctness on standbys: +# +# 1. Checkpoint on standby must NOT remove WAL segments that have a .ready +# status file (i.e. not yet archived by the primary). With the bug, +# XLogArchiveCheckDone() returns true unconditionally during recovery for +# any mode that is not "always", so checkpoint deletes these segments. +# +# 2. After archiving is broken on the primary and then restored, .ready files +# on the standby must eventually transition to .done (primary sends archival +# status reports to the standby via the walsender). + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Use 1 MB WAL segments so we can generate many segments cheaply. +my $wal_segsize = 1; + +# An archive command that always fails (but is recognized by the archiver as a +# real failure, not a missing command). Mirrors the approach in +# 020_archive_status.pl to stay portable. +my $broken_command = + $PostgreSQL::Test::Utils::windows_os + ? q{copy "%p_does_not_exist" "%f_does_not_exist"} + : q{cp "%p_does_not_exist" "%f_does_not_exist"}; + +my $archive_dir = PostgreSQL::Test::Utils::tempdir(); +my $good_command = + $PostgreSQL::Test::Utils::windows_os + ? qq{copy "%p" "$archive_dir\\%f"} + : qq{cp %p "$archive_dir/%f"}; + +############################################################################### +# Set up primary with archive_mode=shared and BROKEN archiving so that every +# WAL segment received by the standby gets a .ready file. +############################################################################### + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init( + has_archiving => 1, + allows_streaming => 1, + extra => [ '--wal-segsize' => $wal_segsize ]); +$primary->append_conf('postgresql.conf', qq{ +archive_mode = shared +archive_command = '$broken_command' +wal_keep_size = 0 +}); +$primary->start; + +my $backup_name = 'standby_backup'; +$primary->backup($backup_name); + +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, $backup_name, has_streaming => 1); +$standby->append_conf('postgresql.conf', qq{ +archive_mode = shared +archive_command = '$good_command' +wal_receiver_status_interval = 1s +wal_keep_size = 0 +}); +$standby->start; + +$primary->wait_for_catchup($standby); + +############################################################################### +# Generate WAL while archiving is broken. +# The walreceiver will create .ready files for every received segment. +############################################################################### + +$primary->safe_psql('postgres', 'CREATE TABLE t (x int)'); + +# Switch WAL several times to create clearly-identifiable old segments. +# We capture the name of the first switched-away segment; it is the primary +# candidate that checkpoint would delete. +my $target_seg = $primary->safe_psql('postgres', + q{SELECT pg_walfile_name(pg_current_wal_lsn())}); + +for my $i (1..5) +{ + $primary->safe_psql('postgres', + "INSERT INTO t SELECT generate_series(1,500)"); + $primary->safe_psql('postgres', 'SELECT pg_switch_wal()'); +} + +# Wait for the archiver to register failures so we are sure archiving is +# truly broken (not just slow). +$primary->poll_query_until('postgres', + q{SELECT failed_count > 0 FROM pg_stat_archiver}) + or die "Timed out waiting for archiver to fail"; + +# Issue a CHECKPOINT on the primary so that the standby can form a +# restartpoint whose redo LSN is past $target_seg. +$primary->safe_psql('postgres', 'CHECKPOINT'); + +# Wait for the standby to replay everything up to that checkpoint. +$primary->wait_for_catchup($standby); + +my $standby_wal_dir = $standby->data_dir . '/pg_wal'; +my $standby_status_dir = "$standby_wal_dir/archive_status"; + +# The target segment must already be visible on the standby as .ready. +my $target_ready = "$standby_status_dir/$target_seg.ready"; +ok(-f $target_ready, + "standby has .ready file for segment $target_seg (not archived by primary)"); + +# The WAL file itself must also be present. +ok(-f "$standby_wal_dir/$target_seg", + "WAL segment $target_seg exists in standby pg_wal before CHECKPOINT"); + +############################################################################### +# Test 1: CHECKPOINT (restartpoint) on standby must not remove .ready segments +############################################################################### + +# This triggers CreateRestartPoint, which calls RemoveOldXlogFiles. +# With the bug, XLogArchiveCheckDone returns true for every segment in +# archive_mode=shared during recovery, so $target_seg would be deleted. +$standby->safe_psql('postgres', 'CHECKPOINT'); + +ok(-f "$standby_wal_dir/$target_seg", + "WAL segment $target_seg still exists after CHECKPOINT on standby " + . "(not deleted despite .ready status)"); + +ok(-f $target_ready, + ".ready file for $target_seg still present after CHECKPOINT on standby"); + +############################################################################### +# Test 2: Restoring archiving on primary causes .ready -> .done on standby +# +# This part is independent of Test 1: we generate fresh WAL (with archiving +# still broken) so the standby accumulates new .ready files, then restore +# archiving and verify those files become .done. +############################################################################### + +# Generate a few more segments so the standby definitely has fresh .ready files +# regardless of what checkpoint may have done above. +for my $i (1..3) +{ + $primary->safe_psql('postgres', + "INSERT INTO t SELECT generate_series(1,200)"); + $primary->safe_psql('postgres', 'SELECT pg_switch_wal()'); +} +$primary->wait_for_catchup($standby); + +# Collect all current .ready files on the standby. +my @ready_segs; +if (opendir(my $dh, $standby_status_dir)) +{ + @ready_segs = + map { (my $s = $_) =~ s/\.ready$//; $s } + grep { /\.ready$/ } readdir($dh); + closedir($dh); +} +note("Standby has " + . scalar(@ready_segs) + . " .ready segments before archiving is restored"); +cmp_ok(scalar(@ready_segs), '>', 0, + "standby has fresh .ready files for newly received unarchived segments"); + +# Restore archiving on the primary. +$primary->safe_psql('postgres', qq{ + ALTER SYSTEM SET archive_command TO '$good_command'; + SELECT pg_reload_conf(); +}); + +# Wait until primary has archived at least one segment. +$primary->poll_query_until('postgres', + q{SELECT archived_count > 0 FROM pg_stat_archiver}) + or die "Timed out waiting for primary to start archiving after restore"; + +# Generate one more WAL switch so the walsender picks up the updated +# last_archived_wal and sends a fresh archival report to the standby. +# (The walsender only sends when last_archived_wal changes and every +# ARCHIVAL_REPORT_INTERVAL = 10 s at most.) +$primary->safe_psql('postgres', 'SELECT pg_switch_wal()'); +$primary->wait_for_catchup($standby); + +# Poll until all previously-.ready segments have become .done. +# Allow up to the framework default timeout (usually 120 s); the walsender +# reports every 10 s so convergence should happen well within that. +my $remaining_ready = scalar(@ready_segs); +for my $i (1 .. $PostgreSQL::Test::Utils::timeout_default) +{ + $remaining_ready = 0; + if (opendir(my $dh, $standby_status_dir)) + { + # Count only the segments that were .ready before archiving was restored + for my $seg (@ready_segs) + { + $remaining_ready++ if -f "$standby_status_dir/$seg.ready"; + } + closedir($dh); + } + last if $remaining_ready == 0; + sleep(1); +} + +is($remaining_ready, 0, + "all .ready files on standby transitioned to .done " + . "after archiving restored on primary"); + +# Sanity-check: the WAL files are still present (they weren't deleted by +# checkpoint while .ready, nor disappeared otherwise). +my @still_missing = grep { !-f "$standby_wal_dir/$_" } @ready_segs; +is(scalar(@still_missing), 0, + "WAL segments were not lost while waiting for archival reports"); + +done_testing();